src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 5386 2006-09-06 21:56:21Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/md-emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55
  56 #if defined(ENABLE_THREADS)
  57 # include "threads/native/lock.h"
  58 #endif
  59
  60 #include "vm/builtin.h"
  61 #include "vm/exceptions.h"
  62 #include "vm/global.h"
  63 #include "vm/loader.h"
  64 #include "vm/options.h"
  65 #include "vm/statistics.h"
  66 #include "vm/stringlocal.h"
  67 #include "vm/vm.h"
  68 #include "vm/jit/asmpart.h"
  69 #include "vm/jit/codegen-common.h"
  70 #include "vm/jit/dseg.h"
  71 #include "vm/jit/emit.h"
  72 #include "vm/jit/jit.h"
  73 #include "vm/jit/methodheader.h"
  74 #include "vm/jit/parse.h"
  75 #include "vm/jit/patcher.h"
  76 #include "vm/jit/reg.h"
  77 #include "vm/jit/replace.h"
  78
  79 #if defined(ENABLE_LSRA)
  80 # include "vm/jit/allocator/lsra.h"
  81 #endif
  82
  83
  84 /* codegen *********************************************************************
  85
  86    Generates machine code.
  87
  88 *******************************************************************************/
  89
  90 bool codegen(jitdata *jd)
  91 {
  92         methodinfo         *m;
  93         codeinfo           *code;
  94         codegendata        *cd;
  95         registerdata       *rd;
  96         s4                  len, s1, s2, s3, d, disp;
  97         u2                  currentline;
  98         ptrint              a;
  99         s4                  fieldtype;
 100         stackptr            src;
 101         varinfo            *var;
 102         basicblock         *bptr;
 103         instruction        *iptr;
 104         exceptiontable     *ex;
 105         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 106         unresolved_method  *um;
 107         builtintable_entry *bte;
 108         methoddesc         *md;
 109         rplpoint           *replacementpoint;
 110
 111         /* get required compiler data */
 112
 113         m    = jd->m;
 114         code = jd->code;
 115         cd   = jd->cd;
 116         rd   = jd->rd;
 117
 118         /* prevent compiler warnings */
 119
 120         d = 0;
 121         lm = NULL;
 122         bte = NULL;
 123
 124         {
 125         s4 i, p, t, l;
 126         s4 savedregs_num;
 127
 128         savedregs_num = 0;
 129
 130         /* space to save used callee saved registers */
 131
 132         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 133         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 134
 135         cd->stackframesize = rd->memuse + savedregs_num;
 136
 137 #if defined(ENABLE_THREADS)
 138         /* space to save argument of monitor_enter */
 139
 140         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 141                 cd->stackframesize++;
 142 #endif
 143
 144         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 145            native code e.g. libc or jni (alignment problems with
 146            movaps). */
 147
 148         if (!jd->isleafmethod || opt_verbosecall)
 149                 cd->stackframesize |= 0x1;
 150
 151         /* create method header */
 152
 153         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
 154         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
 155
 156 #if defined(ENABLE_THREADS)
 157         /* IsSync contains the offset relative to the stack pointer for the
 158            argument of monitor_exit used in the exception handler. Since the
 159            offset could be zero and give a wrong meaning of the flag it is
 160            offset by one.
 161         */
 162
 163         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 164                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);       /* IsSync          */
 165         else
 166 #endif
 167                 (void) dseg_adds4(cd, 0);                          /* IsSync          */
 168
 169         (void) dseg_adds4(cd, jd->isleafmethod);               /* IsLeaf          */
 170         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave         */
 171         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave         */
 172
 173         (void) dseg_addlinenumbertablesize(cd);
 174
 175         (void) dseg_adds4(cd, cd->exceptiontablelength);       /* ExTableSize     */
 176
 177         /* create exception table */
 178
 179         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 180                 dseg_addtarget(cd, ex->start);
 181                 dseg_addtarget(cd, ex->end);
 182                 dseg_addtarget(cd, ex->handler);
 183                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 184         }
 185
 186         /* generate method profiling code */
 187
 188         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 189                 /* count frequency */
 190
 191                 M_MOV_IMM(code, REG_ITMP3);
 192                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 193
 194                 PROFILE_CYCLE_START;
 195         }
 196
 197         /* create stack frame (if necessary) */
 198
 199         if (cd->stackframesize)
 200                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 201
 202         /* save used callee saved registers */
 203
 204         p = cd->stackframesize;
 205         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 206                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 207         }
 208         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 209                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 210         }
 211
 212         /* take arguments out of register or stack frame */
 213
 214         md = m->parseddesc;
 215
 216         for (p = 0, l = 0; p < md->paramcount; p++) {
 217                 t = md->paramtypes[p].type;
 218                 var = &(rd->locals[l][t]);
 219                 l++;
 220                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 221                         l++;
 222                 if (var->type < 0)
 223                         continue;
 224                 s1 = md->params[p].regoff;
 225                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 226                         s2 = rd->argintregs[s1];
 227                         if (!md->params[p].inmemory) {           /* register arguments    */
 228                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 229                                         M_INTMOVE(s2, var->regoff);
 230
 231                                 } else {                             /* reg arg -> spilled    */
 232                                     M_LST(s2, REG_SP, var->regoff * 8);
 233                                 }
 234
 235                         } else {                                 /* stack arguments       */
 236                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 237                                         /* + 8 for return address */
 238                                         M_LLD(var->regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 239
 240                                 } else {                             /* stack arg -> spilled  */
 241                                         var->regoff = cd->stackframesize + s1 + 1;
 242                                 }
 243                         }
 244
 245                 } else {                                     /* floating args         */
 246                         if (!md->params[p].inmemory) {           /* register arguments    */
 247                                 s2 = rd->argfltregs[s1];
 248                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 249                                         M_FLTMOVE(s2, var->regoff);
 250
 251                                 } else {                                         /* reg arg -> spilled    */
 252                                         M_DST(s2, REG_SP, var->regoff * 8);
 253                                 }
 254
 255                         } else {                                 /* stack arguments       */
 256                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 257                                         M_DLD(var->regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 258
 259                                 } else {
 260                                         var->regoff = cd->stackframesize + s1 + 1;
 261                                 }
 262                         }
 263                 }
 264         }  /* end for */
 265
 266         /* save monitorenter argument */
 267
 268 #if defined(ENABLE_THREADS)
 269         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 270                 /* stack offset for monitor argument */
 271
 272                 s1 = rd->memuse;
 273
 274                 if (opt_verbosecall) {
 275                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 276
 277                         for (p = 0; p < INT_ARG_CNT; p++)
 278                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 279
 280                         for (p = 0; p < FLT_ARG_CNT; p++)
 281                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 282
 283                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 284                 }
 285
 286                 /* decide which monitor enter function to call */
 287
 288                 if (m->flags & ACC_STATIC) {
 289                         M_MOV_IMM(&m->class->object.header, REG_A0);
 290                 }
 291                 else {
 292                         M_TEST(REG_A0);
 293                         M_BEQ(0);
 294                         codegen_add_nullpointerexception_ref(cd);
 295                 }
 296
 297                 M_AST(REG_A0, REG_SP, s1 * 8);
 298                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 299                 M_CALL(REG_ITMP1);
 300
 301                 if (opt_verbosecall) {
 302                         for (p = 0; p < INT_ARG_CNT; p++)
 303                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 304
 305                         for (p = 0; p < FLT_ARG_CNT; p++)
 306                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 307
 308                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 309                 }
 310         }
 311 #endif
 312
 313 #if !defined(NDEBUG)
 314         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 315                 emit_verbosecall_enter(jd);
 316 #endif /* !defined(NDEBUG) */
 317
 318         }
 319
 320         /* end of header generation */
 321
 322         replacementpoint = jd->code->rplpoints;
 323
 324         /* walk through all basic blocks */
 325
 326         for (bptr = jd->new_basicblocks; bptr != NULL; bptr = bptr->next) {
 327
 328                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 329
 330                 if (bptr->flags >= BBREACHED) {
 331
 332                         /* branch resolving */
 333
 334                         branchref *bref;
 335                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 336                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 337                                                                   bref->branchpos,
 338                                                                   bptr->mpc);
 339                         }
 340
 341                 /* handle replacement points */
 342
 343 #if 0
 344                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 345                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 346
 347                         replacementpoint++;
 348
 349                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 350                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 351                 }
 352 #endif
 353
 354                 /* copy interface registers to their destination */
 355
 356                 len = bptr->indepth;
 357                 MCODECHECK(512);
 358
 359                 /* generate basicblock profiling code */
 360
 361                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 362                         /* count frequency */
 363
 364                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 365                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 366
 367                         /* if this is an exception handler, start profiling again */
 368
 369                         if (bptr->type == BBTYPE_EXH)
 370                                 PROFILE_CYCLE_START;
 371                 }
 372
 373 #if defined(ENABLE_LSRA)
 374                 if (opt_lsra) {
 375                         while (len) {
 376                                 len--;
 377                                 src = bptr->invars[len];
 378                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 379                                         if (bptr->type == BBTYPE_SBR) {
 380 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 381                                                 if (!(src->flags & INMEMORY))
 382                                                         d = src->regoff;
 383                                                 else
 384                                                         d = REG_ITMP1;
 385                                                 M_POP(d);
 386                                                 emit_store(jd, NULL, src, d);
 387
 388                                         } else if (bptr->type == BBTYPE_EXH) {
 389 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 390                                                 if (!(src->flags & INMEMORY))
 391                                                         d= src->regoff;
 392                                                 else
 393                                                         d=REG_ITMP1;
 394                                                 M_INTMOVE(REG_ITMP1, d);
 395                                                 emit_store(jd, NULL, src, d);
 396                                         }
 397                                 }
 398                         }
 399
 400                 } else {
 401 #endif
 402
 403                 while (len) {
 404                         len--;
 405                         src = bptr->invars[len];
 406                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 407                                 if (bptr->type == BBTYPE_SBR) {
 408                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 409                                         M_POP(d);
 410                                         emit_store(jd, NULL, src, d);
 411
 412                                 } else if (bptr->type == BBTYPE_EXH) {
 413                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 414                                         M_INTMOVE(REG_ITMP1, d);
 415                                         emit_store(jd, NULL, src, d);
 416                                 }
 417
 418                         } else {
 419                                 d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 420                                 if ((src->varkind != STACKVAR)) {
 421                                         s2 = src->type;
 422                                         if (IS_FLT_DBL_TYPE(s2)) {
 423                                                 s1 = rd->interfaces[len][s2].regoff;
 424
 425                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 426                                                         M_FLTMOVE(s1, d);
 427                                                 else
 428                                                         M_DLD(d, REG_SP, s1 * 8);
 429
 430                                                 emit_store(jd, NULL, src, d);
 431
 432                                         } else {
 433                                                 s1 = rd->interfaces[len][s2].regoff;
 434
 435                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 436                                                         M_INTMOVE(s1, d);
 437                                                 else
 438                                                         M_LLD(d, REG_SP, s1 * 8);
 439
 440                                                 emit_store(jd, NULL, src, d);
 441                                         }
 442                                 }
 443                         }
 444                 }
 445 #if defined(ENABLE_LSRA)
 446                 }
 447 #endif
 448                 /* walk through all instructions */
 449
 450                 len = bptr->icount;
 451                 currentline = 0;
 452
 453                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 454                         if (iptr->line != currentline) {
 455                                 dseg_addlinenumber(cd, iptr->line);
 456                                 currentline = iptr->line;
 457                         }
 458
 459                         MCODECHECK(1024);                         /* 1KB should be enough */
 460
 461                         switch (iptr->opc) {
 462                         case ICMD_INLINE_START: /* internal ICMDs                         */
 463                         case ICMD_INLINE_END:
 464                                 break;
 465
 466                         case ICMD_NOP:    /* ...  ==> ...                                 */
 467                                 break;
 468
 469                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 470
 471                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 472                                 M_TEST(s1);
 473                                 M_BEQ(0);
 474                                 codegen_add_nullpointerexception_ref(cd);
 475                                 break;
 476
 477                 /* constant operations ************************************************/
 478
 479                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 480
 481                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 482                         ICONST(d, iptr->sx.val.i);
 483                         emit_store_dst(jd, iptr, d);
 484                         break;
 485
 486                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 487
 488                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 489                         LCONST(d, iptr->sx.val.l);
 490                         emit_store_dst(jd, iptr, d);
 491                         break;
 492
 493                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 494
 495                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 496                         disp = dseg_addfloat(cd, iptr->sx.val.f);
 497                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 498                         emit_store_dst(jd, iptr, d);
 499                         break;
 500
 501                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 502
 503                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 504                         disp = dseg_adddouble(cd, iptr->sx.val.d);
 505                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 506                         emit_store_dst(jd, iptr, d);
 507                         break;
 508
 509                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 510
 511                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 512
 513                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 514 /*                              PROFILE_CYCLE_STOP; */
 515
 516                                 codegen_addpatchref(cd, PATCHER_aconst,
 517                                                                         iptr->sx.val.c.ref, 0);
 518
 519                                 if (opt_showdisassemble) {
 520                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 521                                 }
 522
 523 /*                              PROFILE_CYCLE_START; */
 524
 525                                 M_MOV_IMM(NULL, d);
 526
 527                         } else {
 528                                 if (iptr->sx.val.anyptr == 0)
 529                                         M_CLR(d);
 530                                 else
 531                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 532                         }
 533                         emit_store_dst(jd, iptr, d);
 534                         break;
 535
 536
 537                 /* load/store operations **********************************************/
 538
 539                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 540                                       /* s1.localindex = local variable                         */
 541
 542                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 543                         if ((iptr->dst.var->varkind == LOCALVAR) &&
 544                                 (iptr->dst.var->varnum == iptr->s1.localindex))
 545                                 break;
 546                         var = &(rd->locals[iptr->s1.localindex][iptr->opc - ICMD_ILOAD]);
 547                         if (var->flags & INMEMORY)
 548                                 M_ILD(d, REG_SP, var->regoff * 8);
 549                         else
 550                                 M_INTMOVE(var->regoff, d);
 551                         emit_store_dst(jd, iptr, d);
 552                         break;
 553
 554                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 555                 case ICMD_ALOAD:      /* op1 = local variable                         */
 556
 557                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 558                         if ((iptr->dst.var->varkind == LOCALVAR) &&
 559                                 (iptr->dst.var->varnum == iptr->s1.localindex))
 560                                 break;
 561                         var = &(rd->locals[iptr->s1.localindex][iptr->opc - ICMD_ILOAD]);
 562                         if (var->flags & INMEMORY)
 563                                 M_LLD(d, REG_SP, var->regoff * 8);
 564                         else
 565                                 M_INTMOVE(var->regoff, d);
 566                         emit_store_dst(jd, iptr, d);
 567                         break;
 568
 569                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 570                                       /* s1.localindex = local variable                         */
 571
 572                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 573                         if ((iptr->dst.var->varkind == LOCALVAR) &&
 574                             (iptr->dst.var->varnum == iptr->s1.localindex))
 575                                 break;
 576                         var = &(rd->locals[iptr->s1.localindex][iptr->opc - ICMD_ILOAD]);
 577                         if (var->flags & INMEMORY)
 578                                 M_FLD(d, REG_SP, var->regoff * 8);
 579                         else
 580                                 M_FLTMOVE(var->regoff, d);
 581                         emit_store_dst(jd, iptr, d);
 582                         break;
 583
 584                 case ICMD_DLOAD:      /* ...  ==> ..., content of local variable      */
 585                                       /* s1.localindex = local variable                         */
 586
 587                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 588                         if ((iptr->dst.var->varkind == LOCALVAR) &&
 589                             (iptr->dst.var->varnum == iptr->s1.localindex))
 590                                 break;
 591                         var = &(rd->locals[iptr->s1.localindex][iptr->opc - ICMD_ILOAD]);
 592                         if (var->flags & INMEMORY)
 593                                 M_DLD(d, REG_SP, var->regoff * 8);
 594                         else
 595                                 M_FLTMOVE(var->regoff, d);
 596                         emit_store_dst(jd, iptr, d);
 597                         break;
 598
 599                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 600                                       /* dst.localindex = local variable                         */
 601
 602                         if ((iptr->s1.var->varkind == LOCALVAR) && (iptr->s1.var->varnum == iptr->dst.localindex))
 603                                 break;
 604                         var = &(rd->locals[iptr->dst.localindex][iptr->opc - ICMD_ISTORE]);
 605                         if (var->flags & INMEMORY) {
 606                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 607                                 M_IST(s1, REG_SP, var->regoff * 8);
 608                         } else {
 609                                 s1 = emit_load_s1(jd, iptr, var->regoff);
 610                                 M_INTMOVE(s1, var->regoff);
 611                         }
 612                         break;
 613
 614                 case ICMD_LSTORE:     /* ..., value  ==> ...                          */
 615                 case ICMD_ASTORE:     /* op1 = local variable                         */
 616
 617                         if ((iptr->s1.var->varkind == LOCALVAR) && (iptr->s1.var->varnum == iptr->dst.localindex))
 618                                 break;
 619                         var = &(rd->locals[iptr->dst.localindex][iptr->opc - ICMD_ISTORE]);
 620                         if (var->flags & INMEMORY) {
 621                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 622                                 M_LST(s1, REG_SP, var->regoff * 8);
 623                         } else {
 624                                 s1 = emit_load_s1(jd, iptr, var->regoff);
 625                                 M_INTMOVE(s1, var->regoff);
 626                         }
 627                         break;
 628
 629                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 630                                       /* dst.localindex = local variable                         */
 631
 632                         if ((iptr->s1.var->varkind == LOCALVAR) &&
 633                             (iptr->s1.var->varnum == iptr->dst.localindex)) {
 634                                 break;
 635                         }
 636                         var = &(rd->locals[iptr->dst.localindex][iptr->opc - ICMD_ISTORE]);
 637                         if (var->flags & INMEMORY) {
 638                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 639                                 M_FST(s1, REG_SP, var->regoff * 8);
 640                         } else {
 641                                 s1 = emit_load_s1(jd, iptr, var->regoff);
 642                                 M_FLTMOVE(s1, var->regoff);
 643                         }
 644                         break;
 645
 646                 case ICMD_DSTORE:     /* ..., value  ==> ...                          */
 647                                       /* dst.localindex = local variable                         */
 648
 649                         if ((iptr->s1.var->varkind == LOCALVAR) &&
 650                             (iptr->s1.var->varnum == iptr->dst.localindex)) {
 651                                 break;
 652                         }
 653                         var = &(rd->locals[iptr->dst.localindex][iptr->opc - ICMD_ISTORE]);
 654                         if (var->flags & INMEMORY) {
 655                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 656                                 M_DST(s1, REG_SP, var->regoff * 8);
 657                         } else {
 658                                 s1 = emit_load_s1(jd, iptr, var->regoff);
 659                                 M_FLTMOVE(s1, var->regoff);
 660                         }
 661                         break;
 662
 663
 664                 /* pop/dup/swap operations ********************************************/
 665
 666                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 667
 668                 case ICMD_POP:        /* ..., value  ==> ...                          */
 669                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 670                         break;
 671
 672                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 673
 674                         M_COPY(iptr->s1.var, iptr->dst.var);
 675                         break;
 676
 677                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 678
 679                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[2+2]);
 680                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[2+1]);
 681                         M_COPY(iptr->dst.dupslots[2+2], iptr->dst.dupslots[2+0]);
 682                         break;
 683
 684                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 685
 686                         M_COPY(iptr->dst.dupslots[  2], iptr->dst.dupslots[3+3]);
 687                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[3+2]);
 688                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[3+1]);
 689                         M_COPY(iptr->dst.dupslots[3+3], iptr->dst.dupslots[3+0]);
 690                         break;
 691
 692                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 693
 694                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[2+1]);
 695                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[2+0]);
 696                         break;
 697
 698                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 699
 700                         M_COPY(iptr->dst.dupslots[  2], iptr->dst.dupslots[3+4]);
 701                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[3+3]);
 702                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[3+2]);
 703                         M_COPY(iptr->dst.dupslots[3+4], iptr->dst.dupslots[3+1]);
 704                         M_COPY(iptr->dst.dupslots[3+3], iptr->dst.dupslots[3+0]);
 705                         break;
 706
 707                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 708
 709                         M_COPY(iptr->dst.dupslots[  3], iptr->dst.dupslots[4+5]);
 710                         M_COPY(iptr->dst.dupslots[  2], iptr->dst.dupslots[4+4]);
 711                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[4+3]);
 712                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[4+2]);
 713                         M_COPY(iptr->dst.dupslots[4+5], iptr->dst.dupslots[4+1]);
 714                         M_COPY(iptr->dst.dupslots[4+4], iptr->dst.dupslots[4+0]);
 715                         break;
 716
 717                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 718
 719                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[2+0]);
 720                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[2+1]);
 721                         break;
 722
 723
 724                 /* integer operations *************************************************/
 725
 726                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 727
 728                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 729                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 730                         M_INTMOVE(s1, d);
 731                         M_INEG(d);
 732                         emit_store_dst(jd, iptr, d);
 733                         break;
 734
 735                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 736
 737                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 738                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 739                         M_INTMOVE(s1, d);
 740                         M_LNEG(d);
 741                         emit_store_dst(jd, iptr, d);
 742                         break;
 743
 744                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 745
 746                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 747                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 748                         M_ISEXT(s1, d);
 749                         emit_store_dst(jd, iptr, d);
 750                         break;
 751
 752                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 753
 754                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 755                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 756                         M_IMOV(s1, d);
 757                         emit_store_dst(jd, iptr, d);
 758                         break;
 759
 760                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 761
 762                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 763                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 764                         M_BSEXT(s1, d);
 765                         emit_store_dst(jd, iptr, d);
 766                         break;
 767
 768                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 769
 770                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 771                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 772                         M_CZEXT(s1, d);
 773                         emit_store_dst(jd, iptr, d);
 774                         break;
 775
 776                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 777
 778                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 779                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 780                         M_SSEXT(s1, d);
 781                         emit_store_dst(jd, iptr, d);
 782                         break;
 783
 784
 785                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 786
 787                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 788                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 789                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 790                         if (s2 == d)
 791                                 M_IADD(s1, d);
 792                         else {
 793                                 M_INTMOVE(s1, d);
 794                                 M_IADD(s2, d);
 795                         }
 796                         emit_store_dst(jd, iptr, d);
 797                         break;
 798
 799                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 800                                       /* sx.val.i = constant                             */
 801
 802                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 803                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 804                         M_INTMOVE(s1, d);
 805                         M_IADD_IMM(iptr->sx.val.i, d);
 806                         emit_store_dst(jd, iptr, d);
 807                         break;
 808
 809                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 810
 811                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 812                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 813                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 814                         if (s2 == d)
 815                                 M_LADD(s1, d);
 816                         else {
 817                                 M_INTMOVE(s1, d);
 818                                 M_LADD(s2, d);
 819                         }
 820                         emit_store_dst(jd, iptr, d);
 821                         break;
 822
 823                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 824                                       /* sx.val.l = constant                             */
 825
 826                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 827                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 828                         M_INTMOVE(s1, d);
 829                         if (IS_IMM32(iptr->sx.val.l))
 830                                 M_LADD_IMM(iptr->sx.val.l, d);
 831                         else {
 832                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 833                                 M_LADD(REG_ITMP2, d);
 834                         }
 835                         emit_store_dst(jd, iptr, d);
 836                         break;
 837
 838                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 839
 840                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 841                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 842                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 843                         if (s2 == d) {
 844                                 M_INTMOVE(s1, REG_ITMP1);
 845                                 M_ISUB(s2, REG_ITMP1);
 846                                 M_INTMOVE(REG_ITMP1, d);
 847                         } else {
 848                                 M_INTMOVE(s1, d);
 849                                 M_ISUB(s2, d);
 850                         }
 851                         emit_store_dst(jd, iptr, d);
 852                         break;
 853
 854                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 855                                       /* sx.val.i = constant                             */
 856
 857                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 858                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 859                         M_INTMOVE(s1, d);
 860                         M_ISUB_IMM(iptr->sx.val.i, d);
 861                         emit_store_dst(jd, iptr, d);
 862                         break;
 863
 864                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 865
 866                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 867                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 868                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 869                         if (s2 == d) {
 870                                 M_INTMOVE(s1, REG_ITMP1);
 871                                 M_LSUB(s2, REG_ITMP1);
 872                                 M_INTMOVE(REG_ITMP1, d);
 873                         } else {
 874                                 M_INTMOVE(s1, d);
 875                                 M_LSUB(s2, d);
 876                         }
 877                         emit_store_dst(jd, iptr, d);
 878                         break;
 879
 880                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 881                                       /* sx.val.l = constant                             */
 882
 883                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 884                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 885                         M_INTMOVE(s1, d);
 886                         if (IS_IMM32(iptr->sx.val.l))
 887                                 M_LSUB_IMM(iptr->sx.val.l, d);
 888                         else {
 889                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 890                                 M_LSUB(REG_ITMP2, d);
 891                         }
 892                         emit_store_dst(jd, iptr, d);
 893                         break;
 894
 895                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 896
 897                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 898                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 899                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 900                         if (s2 == d)
 901                                 M_IMUL(s1, d);
 902                         else {
 903                                 M_INTMOVE(s1, d);
 904                                 M_IMUL(s2, d);
 905                         }
 906                         emit_store_dst(jd, iptr, d);
 907                         break;
 908
 909                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 910                                       /* sx.val.i = constant                             */
 911
 912                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 913                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 914                         if (iptr->sx.val.i == 2) {
 915                                 M_INTMOVE(s1, d);
 916                                 M_ISLL_IMM(1, d);
 917                         } else
 918                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 919                         emit_store_dst(jd, iptr, d);
 920                         break;
 921
 922                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 923
 924                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 925                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 926                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 927                         if (s2 == d)
 928                                 M_LMUL(s1, d);
 929                         else {
 930                                 M_INTMOVE(s1, d);
 931                                 M_LMUL(s2, d);
 932                         }
 933                         emit_store_dst(jd, iptr, d);
 934                         break;
 935
 936                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 937                                       /* sx.val.l = constant                             */
 938
 939                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 940                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 941                         if (IS_IMM32(iptr->sx.val.l))
 942                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 943                         else {
 944                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 945                                 M_INTMOVE(s1, d);
 946                                 M_LMUL(REG_ITMP2, d);
 947                         }
 948                         emit_store_dst(jd, iptr, d);
 949                         break;
 950
 951                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 952
 953                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 954                 if (iptr->s1.var->flags & INMEMORY)
 955                                 M_ILD(RAX, REG_SP, iptr->s1.var->regoff * 8);
 956                         else
 957                                 M_INTMOVE(iptr->s1.var->regoff, RAX);
 958
 959                         if (iptr->sx.s23.s2.var->flags & INMEMORY)
 960                                 M_ILD(REG_ITMP3, REG_SP, iptr->sx.s23.s2.var->regoff * 8);
 961                         else
 962                                 M_INTMOVE(iptr->sx.s23.s2.var->regoff, REG_ITMP3);
 963
 964                         if (checknull) {
 965                                 M_ITEST(REG_ITMP3);
 966                                 M_BEQ(0);
 967                                 codegen_add_arithmeticexception_ref(cd);
 968                         }
 969
 970                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 971                         emit_jcc(cd, CC_NE, 4 + 6);
 972                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 973                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
 974
 975                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 976                         emit_cltd(cd);
 977                         emit_idivl_reg(cd, REG_ITMP3);
 978
 979                         if (iptr->dst.var->flags & INMEMORY) {
 980                                 emit_mov_reg_membase(cd, RAX, REG_SP, iptr->dst.var->regoff * 8);
 981                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 982
 983                         } else {
 984                                 M_INTMOVE(RAX, iptr->dst.var->regoff);
 985
 986                                 if (iptr->dst.var->regoff != RDX) {
 987                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 988                                 }
 989                         }
 990                         break;
 991
 992                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 993
 994                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 995                         if (iptr->s1.var->flags & INMEMORY)
 996                                 M_ILD(RAX, REG_SP, iptr->s1.var->regoff * 8);
 997                         else
 998                                 M_INTMOVE(iptr->s1.var->regoff, RAX);
 999
1000                         if (iptr->sx.s23.s2.var->flags & INMEMORY)
1001                                 M_ILD(REG_ITMP3, REG_SP, iptr->sx.s23.s2.var->regoff * 8);
1002                         else
1003                                 M_INTMOVE(iptr->sx.s23.s2.var->regoff, REG_ITMP3);
1004
1005                         if (checknull) {
1006                                 M_ITEST(REG_ITMP3);
1007                                 M_BEQ(0);
1008                                 codegen_add_arithmeticexception_ref(cd);
1009                         }
1010
1011                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1012
1013                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1014                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
1015
1016
1017                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1018                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1019                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
1020
1021                         emit_cltd(cd);
1022                         emit_idivl_reg(cd, REG_ITMP3);
1023
1024                         if (iptr->dst.var->flags & INMEMORY) {
1025                                 emit_mov_reg_membase(cd, RDX, REG_SP, iptr->dst.var->regoff * 8);
1026                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1027
1028                         } else {
1029                                 M_INTMOVE(RDX, iptr->dst.var->regoff);
1030
1031                                 if (iptr->dst.var->regoff != RDX) {
1032                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1033                                 }
1034                         }
1035                         break;
1036
1037                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1038                                       /* sx.val.i = constant                             */
1039
1040                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1041                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1042                         M_INTMOVE(s1, REG_ITMP1);
1043                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1044                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
1045                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1046                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
1047                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1048                         emit_store_dst(jd, iptr, d);
1049                         break;
1050
1051                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1052                                       /* sx.val.i = constant                             */
1053
1054                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1055                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1056                         M_INTMOVE(s1, REG_ITMP1);
1057                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1058                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
1059                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1060                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
1061                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1062                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1063                         emit_store_dst(jd, iptr, d);
1064                         break;
1065
1066
1067                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1068
1069                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1070
1071                 if (iptr->s1.var->flags & INMEMORY)
1072                                 M_LLD(RAX, REG_SP, iptr->s1.var->regoff * 8);
1073                         else
1074                                 M_INTMOVE(iptr->s1.var->regoff, RAX);
1075
1076                         if (iptr->sx.s23.s2.var->flags & INMEMORY)
1077                                 M_LLD(REG_ITMP3, REG_SP, iptr->sx.s23.s2.var->regoff * 8);
1078                         else
1079                                 M_INTMOVE(iptr->sx.s23.s2.var->regoff, REG_ITMP3);
1080
1081                         if (checknull) {
1082                                 M_TEST(REG_ITMP3);
1083                                 M_BEQ(0);
1084                                 codegen_add_arithmeticexception_ref(cd);
1085                         }
1086
1087                         /* check as described in jvm spec */
1088                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1089                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
1090                         M_BNE(4 + 6);
1091                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1092                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1093
1094                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1095                         emit_cqto(cd);
1096                         emit_idiv_reg(cd, REG_ITMP3);
1097
1098                         if (iptr->dst.var->flags & INMEMORY) {
1099                                 M_LST(RAX, REG_SP, iptr->dst.var->regoff * 8);
1100                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1101
1102                         } else {
1103                                 M_INTMOVE(RAX, iptr->dst.var->regoff);
1104
1105                                 if (iptr->dst.var->regoff != RDX) {
1106                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1107                                 }
1108                         }
1109                         break;
1110
1111                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1112
1113                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1114                         if (iptr->s1.var->flags & INMEMORY)
1115                                 M_LLD(REG_ITMP1, REG_SP, iptr->s1.var->regoff * 8);
1116                         else
1117                                 M_INTMOVE(iptr->s1.var->regoff, REG_ITMP1);
1118
1119                         if (iptr->sx.s23.s2.var->flags & INMEMORY)
1120                                 M_LLD(REG_ITMP3, REG_SP, iptr->sx.s23.s2.var->regoff * 8);
1121                         else
1122                                 M_INTMOVE(iptr->sx.s23.s2.var->regoff, REG_ITMP3);
1123
1124                         if (checknull) {
1125                                 M_ITEST(REG_ITMP3);
1126                                 M_BEQ(0);
1127                                 codegen_add_arithmeticexception_ref(cd);
1128                         }
1129
1130                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1131
1132                         /* check as described in jvm spec */
1133                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1134                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
1135                         M_BNE(3 + 4 + 6);
1136
1137 #if 0
1138                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1139 #endif
1140                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
1141                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1142                         M_BEQ(2 + 3);                                          /* 6 bytes */
1143
1144                         emit_cqto(cd);
1145                         emit_idiv_reg(cd, REG_ITMP3);
1146
1147                         if (iptr->dst.var->flags & INMEMORY) {
1148                                 M_LST(RDX, REG_SP, iptr->dst.var->regoff * 8);
1149                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1150
1151                         } else {
1152                                 M_INTMOVE(RDX, iptr->dst.var->regoff);
1153
1154                                 if (iptr->dst.var->regoff != RDX) {
1155                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1156                                 }
1157                         }
1158                         break;
1159
1160                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1161                                       /* sx.val.i = constant                             */
1162
1163                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1164                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1165                         M_INTMOVE(s1, REG_ITMP1);
1166                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1167                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
1168                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1169                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
1170                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1171                         emit_store_dst(jd, iptr, d);
1172                         break;
1173
1174                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1175                                       /* sx.val.l = constant                             */
1176
1177                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1178                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1179                         M_INTMOVE(s1, REG_ITMP1);
1180                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1181                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
1182                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1183                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
1184                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1185                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1186                         emit_store_dst(jd, iptr, d);
1187                         break;
1188
1189                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1190
1191                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1192                         emit_ishift(cd, SHIFT_SHL, iptr->sx.s23.s2.var, iptr);
1193                         break;
1194
1195                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1196                                       /* sx.val.i = constant                             */
1197
1198                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1199                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1200                         M_INTMOVE(s1, d);
1201                         M_ISLL_IMM(iptr->sx.val.i, d);
1202                         emit_store_dst(jd, iptr, d);
1203                         break;
1204
1205                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1206
1207                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1208                         emit_ishift(cd, SHIFT_SAR, iptr->sx.s23.s2.var, iptr);
1209                         break;
1210
1211                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1212                                       /* sx.val.i = constant                             */
1213
1214                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1215                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1216                         M_INTMOVE(s1, d);
1217                         M_ISRA_IMM(iptr->sx.val.i, d);
1218                         emit_store_dst(jd, iptr, d);
1219                         break;
1220
1221                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1222
1223                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1224                         emit_ishift(cd, SHIFT_SHR, iptr->sx.s23.s2.var, iptr);
1225                         break;
1226
1227                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1228                                       /* sx.val.i = constant                             */
1229
1230                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1231                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1232                         M_INTMOVE(s1, d);
1233                         M_ISRL_IMM(iptr->sx.val.i, d);
1234                         emit_store_dst(jd, iptr, d);
1235                         break;
1236
1237                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1238
1239                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1240                         emit_lshift(cd, SHIFT_SHL, iptr->sx.s23.s2.var, iptr);
1241                         break;
1242
1243         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1244                                           /* sx.val.i = constant                             */
1245
1246                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1247                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1248                         M_INTMOVE(s1, d);
1249                         M_LSLL_IMM(iptr->sx.val.i, d);
1250                         emit_store_dst(jd, iptr, d);
1251                         break;
1252
1253                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1254
1255                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1256                         emit_lshift(cd, SHIFT_SAR, iptr->sx.s23.s2.var, iptr);
1257                         break;
1258
1259                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1260                                       /* sx.val.i = constant                             */
1261
1262                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1263                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1264                         M_INTMOVE(s1, d);
1265                         M_LSRA_IMM(iptr->sx.val.i, d);
1266                         emit_store_dst(jd, iptr, d);
1267                         break;
1268
1269                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1270
1271                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1272                         emit_lshift(cd, SHIFT_SHR, iptr->sx.s23.s2.var, iptr);
1273                         break;
1274
1275                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1276                                       /* sx.val.l = constant                             */
1277
1278                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1279                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1280                         M_INTMOVE(s1, d);
1281                         M_LSRL_IMM(iptr->sx.val.i, d);
1282                         emit_store_dst(jd, iptr, d);
1283                         break;
1284
1285                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1286
1287                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1288                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1289                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1290                         if (s2 == d)
1291                                 M_IAND(s1, d);
1292                         else {
1293                                 M_INTMOVE(s1, d);
1294                                 M_IAND(s2, d);
1295                         }
1296                         emit_store_dst(jd, iptr, d);
1297                         break;
1298
1299                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1300                                       /* sx.val.i = constant                             */
1301
1302                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1303                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1304                         M_INTMOVE(s1, d);
1305                         M_IAND_IMM(iptr->sx.val.i, d);
1306                         emit_store_dst(jd, iptr, d);
1307                         break;
1308
1309                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1310
1311                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1312                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1313                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1314                         if (s2 == d)
1315                                 M_LAND(s1, d);
1316                         else {
1317                                 M_INTMOVE(s1, d);
1318                                 M_LAND(s2, d);
1319                         }
1320                         emit_store_dst(jd, iptr, d);
1321                         break;
1322
1323                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1324                                       /* sx.val.l = constant                             */
1325
1326                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1327                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1328                         M_INTMOVE(s1, d);
1329                         if (IS_IMM32(iptr->sx.val.l))
1330                                 M_LAND_IMM(iptr->sx.val.l, d);
1331                         else {
1332                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1333                                 M_LAND(REG_ITMP2, d);
1334                         }
1335                         emit_store_dst(jd, iptr, d);
1336                         break;
1337
1338                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1339
1340                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1341                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1342                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1343                         if (s2 == d)
1344                                 M_IOR(s1, d);
1345                         else {
1346                                 M_INTMOVE(s1, d);
1347                                 M_IOR(s2, d);
1348                         }
1349                         emit_store_dst(jd, iptr, d);
1350                         break;
1351
1352                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1353                                       /* sx.val.i = constant                             */
1354
1355                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1357                         M_INTMOVE(s1, d);
1358                         M_IOR_IMM(iptr->sx.val.i, d);
1359                         emit_store_dst(jd, iptr, d);
1360                         break;
1361
1362                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1363
1364                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1365                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1366                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1367                         if (s2 == d)
1368                                 M_LOR(s1, d);
1369                         else {
1370                                 M_INTMOVE(s1, d);
1371                                 M_LOR(s2, d);
1372                         }
1373                         emit_store_dst(jd, iptr, d);
1374                         break;
1375
1376                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1377                                       /* sx.val.l = constant                             */
1378
1379                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1380                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1381                         M_INTMOVE(s1, d);
1382                         if (IS_IMM32(iptr->sx.val.l))
1383                                 M_LOR_IMM(iptr->sx.val.l, d);
1384                         else {
1385                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1386                                 M_LOR(REG_ITMP2, d);
1387                         }
1388                         emit_store_dst(jd, iptr, d);
1389                         break;
1390
1391                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1392
1393                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1394                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1395                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1396                         if (s2 == d)
1397                                 M_IXOR(s1, d);
1398                         else {
1399                                 M_INTMOVE(s1, d);
1400                                 M_IXOR(s2, d);
1401                         }
1402                         emit_store_dst(jd, iptr, d);
1403                         break;
1404
1405                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1406                                       /* sx.val.i = constant                             */
1407
1408                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1409                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1410                         M_INTMOVE(s1, d);
1411                         M_IXOR_IMM(iptr->sx.val.i, d);
1412                         emit_store_dst(jd, iptr, d);
1413                         break;
1414
1415                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1416
1417                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1418                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1419                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1420                         if (s2 == d)
1421                                 M_LXOR(s1, d);
1422                         else {
1423                                 M_INTMOVE(s1, d);
1424                                 M_LXOR(s2, d);
1425                         }
1426                         emit_store_dst(jd, iptr, d);
1427                         break;
1428
1429                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1430                                       /* sx.val.l = constant                             */
1431
1432                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1433                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1434                         M_INTMOVE(s1, d);
1435                         if (IS_IMM32(iptr->sx.val.l))
1436                                 M_LXOR_IMM(iptr->sx.val.l, d);
1437                         else {
1438                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1439                                 M_LXOR(REG_ITMP2, d);
1440                         }
1441                         emit_store_dst(jd, iptr, d);
1442                         break;
1443
1444
1445                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1446                                       /* s1.localindex = variable, sx.val.i = constant             */
1447
1448                         var = &(rd->locals[iptr->s1.localindex][TYPE_INT]);
1449                         if (var->flags & INMEMORY) {
1450                                 s1 = REG_ITMP1;
1451                                 M_ILD(s1, REG_SP, var->regoff * 8);
1452                         } else
1453                                 s1 = var->regoff;
1454
1455                         /* Using inc and dec is not faster than add (tested with
1456                            sieve). */
1457
1458                         M_IADD_IMM(iptr->sx.val.i, s1);
1459
1460                         if (var->flags & INMEMORY)
1461                                 M_IST(s1, REG_SP, var->regoff * 8);
1462                         break;
1463
1464
1465                 /* floating operations ************************************************/
1466
1467                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1468
1469                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1470                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1471                         disp = dseg_adds4(cd, 0x80000000);
1472                         M_FLTMOVE(s1, d);
1473                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1474                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1475                         emit_store_dst(jd, iptr, d);
1476                         break;
1477
1478                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1479
1480                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1481                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1482                         disp = dseg_adds8(cd, 0x8000000000000000);
1483                         M_FLTMOVE(s1, d);
1484                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1485                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1486                         emit_store_dst(jd, iptr, d);
1487                         break;
1488
1489                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1490
1491                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1492                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1493                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1494                         if (s2 == d)
1495                                 M_FADD(s1, d);
1496                         else {
1497                                 M_FLTMOVE(s1, d);
1498                                 M_FADD(s2, d);
1499                         }
1500                         emit_store_dst(jd, iptr, d);
1501                         break;
1502
1503                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1504
1505                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1506                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1507                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1508                         if (s2 == d)
1509                                 M_DADD(s1, d);
1510                         else {
1511                                 M_FLTMOVE(s1, d);
1512                                 M_DADD(s2, d);
1513                         }
1514                         emit_store_dst(jd, iptr, d);
1515                         break;
1516
1517                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1518
1519                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1520                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1522                         if (s2 == d) {
1523                                 M_FLTMOVE(s2, REG_FTMP2);
1524                                 s2 = REG_FTMP2;
1525                         }
1526                         M_FLTMOVE(s1, d);
1527                         M_FSUB(s2, d);
1528                         emit_store_dst(jd, iptr, d);
1529                         break;
1530
1531                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1532
1533                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1534                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1535                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1536                         if (s2 == d) {
1537                                 M_FLTMOVE(s2, REG_FTMP2);
1538                                 s2 = REG_FTMP2;
1539                         }
1540                         M_FLTMOVE(s1, d);
1541                         M_DSUB(s2, d);
1542                         emit_store_dst(jd, iptr, d);
1543                         break;
1544
1545                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1546
1547                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1548                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1549                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1550                         if (s2 == d)
1551                                 M_FMUL(s1, d);
1552                         else {
1553                                 M_FLTMOVE(s1, d);
1554                                 M_FMUL(s2, d);
1555                         }
1556                         emit_store_dst(jd, iptr, d);
1557                         break;
1558
1559                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1560
1561                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1562                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1563                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1564                         if (s2 == d)
1565                                 M_DMUL(s1, d);
1566                         else {
1567                                 M_FLTMOVE(s1, d);
1568                                 M_DMUL(s2, d);
1569                         }
1570                         emit_store_dst(jd, iptr, d);
1571                         break;
1572
1573                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1574
1575                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1576                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1577                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1578                         if (s2 == d) {
1579                                 M_FLTMOVE(s2, REG_FTMP2);
1580                                 s2 = REG_FTMP2;
1581                         }
1582                         M_FLTMOVE(s1, d);
1583                         M_FDIV(s2, d);
1584                         emit_store_dst(jd, iptr, d);
1585                         break;
1586
1587                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1588
1589                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1590                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1591                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1592                         if (s2 == d) {
1593                                 M_FLTMOVE(s2, REG_FTMP2);
1594                                 s2 = REG_FTMP2;
1595                         }
1596                         M_FLTMOVE(s1, d);
1597                         M_DDIV(s2, d);
1598                         emit_store_dst(jd, iptr, d);
1599                         break;
1600
1601                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1602
1603                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1604                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1605                         M_CVTIF(s1, d);
1606                         emit_store_dst(jd, iptr, d);
1607                         break;
1608
1609                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1610
1611                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1612                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1613                         M_CVTID(s1, d);
1614                         emit_store_dst(jd, iptr, d);
1615                         break;
1616
1617                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1618
1619                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1620                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1621                         M_CVTLF(s1, d);
1622                         emit_store_dst(jd, iptr, d);
1623                         break;
1624
1625                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1626
1627                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1628                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1629                         M_CVTLD(s1, d);
1630                         emit_store_dst(jd, iptr, d);
1631                         break;
1632
1633                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1634
1635                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1636                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1637                         M_CVTFI(s1, d);
1638                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1639                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1640                                 ((REG_RESULT == d) ? 0 : 3);
1641                         M_BNE(disp);
1642                         M_FLTMOVE(s1, REG_FTMP1);
1643                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1644                         M_CALL(REG_ITMP2);
1645                         M_INTMOVE(REG_RESULT, d);
1646                         emit_store_dst(jd, iptr, d);
1647                         break;
1648
1649                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1650
1651                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1652                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1653                         M_CVTDI(s1, d);
1654                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1655                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1656                                 ((REG_RESULT == d) ? 0 : 3);
1657                         M_BNE(disp);
1658                         M_FLTMOVE(s1, REG_FTMP1);
1659                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1660                         M_CALL(REG_ITMP2);
1661                         M_INTMOVE(REG_RESULT, d);
1662                         emit_store_dst(jd, iptr, d);
1663                         break;
1664
1665                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1666
1667                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1668                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1669                         M_CVTFL(s1, d);
1670                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1671                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1672                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1673                                 ((REG_RESULT == d) ? 0 : 3);
1674                         M_BNE(disp);
1675                         M_FLTMOVE(s1, REG_FTMP1);
1676                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1677                         M_CALL(REG_ITMP2);
1678                         M_INTMOVE(REG_RESULT, d);
1679                         emit_store_dst(jd, iptr, d);
1680                         break;
1681
1682                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1683
1684                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1685                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1686                         M_CVTDL(s1, d);
1687                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1688                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1689                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1690                                 ((REG_RESULT == d) ? 0 : 3);
1691                         M_BNE(disp);
1692                         M_FLTMOVE(s1, REG_FTMP1);
1693                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1694                         M_CALL(REG_ITMP2);
1695                         M_INTMOVE(REG_RESULT, d);
1696                         emit_store_dst(jd, iptr, d);
1697                         break;
1698
1699                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1700
1701                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1702                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1703                         M_CVTFD(s1, d);
1704                         emit_store_dst(jd, iptr, d);
1705                         break;
1706
1707                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1708
1709                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1710                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1711                         M_CVTDF(s1, d);
1712                         emit_store_dst(jd, iptr, d);
1713                         break;
1714
1715                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1716                                           /* == => 0, < => 1, > => -1 */
1717
1718                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1719                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1720                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1721                         M_CLR(d);
1722                         M_MOV_IMM(1, REG_ITMP1);
1723                         M_MOV_IMM(-1, REG_ITMP2);
1724                         emit_ucomiss_reg_reg(cd, s1, s2);
1725                         M_CMOVB(REG_ITMP1, d);
1726                         M_CMOVA(REG_ITMP2, d);
1727                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1728                         emit_store_dst(jd, iptr, d);
1729                         break;
1730
1731                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1732                                           /* == => 0, < => 1, > => -1 */
1733
1734                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1735                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1736                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1737                         M_CLR(d);
1738                         M_MOV_IMM(1, REG_ITMP1);
1739                         M_MOV_IMM(-1, REG_ITMP2);
1740                         emit_ucomiss_reg_reg(cd, s1, s2);
1741                         M_CMOVB(REG_ITMP1, d);
1742                         M_CMOVA(REG_ITMP2, d);
1743                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1744                         emit_store_dst(jd, iptr, d);
1745                         break;
1746
1747                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1748                                           /* == => 0, < => 1, > => -1 */
1749
1750                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1751                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1752                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1753                         M_CLR(d);
1754                         M_MOV_IMM(1, REG_ITMP1);
1755                         M_MOV_IMM(-1, REG_ITMP2);
1756                         emit_ucomisd_reg_reg(cd, s1, s2);
1757                         M_CMOVB(REG_ITMP1, d);
1758                         M_CMOVA(REG_ITMP2, d);
1759                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1760                         emit_store_dst(jd, iptr, d);
1761                         break;
1762
1763                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1764                                           /* == => 0, < => 1, > => -1 */
1765
1766                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1767                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1768                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1769                         M_CLR(d);
1770                         M_MOV_IMM(1, REG_ITMP1);
1771                         M_MOV_IMM(-1, REG_ITMP2);
1772                         emit_ucomisd_reg_reg(cd, s1, s2);
1773                         M_CMOVB(REG_ITMP1, d);
1774                         M_CMOVA(REG_ITMP2, d);
1775                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1776                         emit_store_dst(jd, iptr, d);
1777                         break;
1778
1779
1780                 /* memory operations **************************************************/
1781
1782                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1783
1784                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1785                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1786                         gen_nullptr_check(s1);
1787                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1788                         emit_store_dst(jd, iptr, d);
1789                         break;
1790
1791                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1792
1793                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1794                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1795                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1796                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1797                                 gen_nullptr_check(s1);
1798                                 gen_bound_check;
1799                         }
1800                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1801                         emit_store_dst(jd, iptr, d);
1802                         break;
1803
1804                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1805
1806                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1807                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1808                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1809                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1810                                 gen_nullptr_check(s1);
1811                                 gen_bound_check;
1812                         }
1813                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1814                         emit_store_dst(jd, iptr, d);
1815                         break;
1816
1817                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1818
1819                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1820                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1821                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1822                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1823                                 gen_nullptr_check(s1);
1824                                 gen_bound_check;
1825                         }
1826                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1827                         emit_store_dst(jd, iptr, d);
1828                         break;
1829
1830                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1831
1832                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1833                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1834                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1835                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1836                                 gen_nullptr_check(s1);
1837                                 gen_bound_check;
1838                         }
1839                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1840                         emit_store_dst(jd, iptr, d);
1841                         break;
1842
1843                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1844
1845                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1846                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1847                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1848                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1849                                 gen_nullptr_check(s1);
1850                                 gen_bound_check;
1851                         }
1852                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1853                         emit_store_dst(jd, iptr, d);
1854                         break;
1855
1856                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1857
1858                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1859                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1860                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1861                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1862                                 gen_nullptr_check(s1);
1863                                 gen_bound_check;
1864                         }
1865                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1866                         emit_store_dst(jd, iptr, d);
1867                         break;
1868
1869                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1870
1871                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1872                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1873                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1874                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1875                                 gen_nullptr_check(s1);
1876                                 gen_bound_check;
1877                         }
1878                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1879                         emit_store_dst(jd, iptr, d);
1880                         break;
1881
1882                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1883
1884                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1885                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1886                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1887                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1888                                 gen_nullptr_check(s1);
1889                                 gen_bound_check;
1890                         }
1891                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1892                         emit_store_dst(jd, iptr, d);
1893                         break;
1894
1895
1896                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1897
1898                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1899                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1900                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1901                                 gen_nullptr_check(s1);
1902                                 gen_bound_check;
1903                         }
1904                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1905                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1906                         break;
1907
1908                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1909
1910                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1911                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1912                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1913                                 gen_nullptr_check(s1);
1914                                 gen_bound_check;
1915                         }
1916                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1917                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1918                         break;
1919
1920                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1921
1922                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1923                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1924                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1925                                 gen_nullptr_check(s1);
1926                                 gen_bound_check;
1927                         }
1928                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1929                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1930                         break;
1931
1932                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1933
1934                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1935                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1936                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1937                                 gen_nullptr_check(s1);
1938                                 gen_bound_check;
1939                         }
1940                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1941                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1942                         break;
1943
1944                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1945
1946                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1947                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1948                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1949                                 gen_nullptr_check(s1);
1950                                 gen_bound_check;
1951                         }
1952                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1953                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1954                         break;
1955
1956                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1957
1958                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1959                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1960                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1961                                 gen_nullptr_check(s1);
1962                                 gen_bound_check;
1963                         }
1964                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1965                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1966                         break;
1967
1968                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1969
1970                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1971                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1972                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1973                                 gen_nullptr_check(s1);
1974                                 gen_bound_check;
1975                         }
1976                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1977                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1978                         break;
1979
1980                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1981
1982                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1983                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1984                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1985                                 gen_nullptr_check(s1);
1986                                 gen_bound_check;
1987                         }
1988                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1989
1990                         M_MOV(s1, REG_A0);
1991                         M_MOV(s3, REG_A1);
1992                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1993                         M_CALL(REG_ITMP1);
1994                         M_TEST(REG_RESULT);
1995                         M_BEQ(0);
1996                         codegen_add_arraystoreexception_ref(cd);
1997
1998                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1999                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2000                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
2001                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2002                         break;
2003
2004
2005                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2006
2007                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2008                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2009                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2010                                 gen_nullptr_check(s1);
2011                                 gen_bound_check;
2012                         }
2013                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2014                         break;
2015
2016                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2017
2018                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2019                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2020                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2021                                 gen_nullptr_check(s1);
2022                                 gen_bound_check;
2023                         }
2024                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
2025                         break;
2026
2027                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2028
2029                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2030                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2031                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2032                                 gen_nullptr_check(s1);
2033                                 gen_bound_check;
2034                         }
2035                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2036                         break;
2037
2038                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2039
2040                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2041                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2042                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2043                                 gen_nullptr_check(s1);
2044                                 gen_bound_check;
2045                         }
2046                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
2047                         break;
2048
2049                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2050
2051                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2052                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2053                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2054                                 gen_nullptr_check(s1);
2055                                 gen_bound_check;
2056                         }
2057
2058                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
2059                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2060                         } else {
2061                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2062                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2063                         }
2064                         break;
2065
2066                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2067
2068                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2069                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2070                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2071                                 gen_nullptr_check(s1);
2072                                 gen_bound_check;
2073                         }
2074                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2075                         break;
2076
2077
2078                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2079
2080                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2081                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2082
2083                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2084                                 disp = dseg_addaddress(cd, NULL);
2085                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2086
2087                                 /* must be calculated before codegen_addpatchref */
2088
2089                                 if (opt_showdisassemble)
2090                                         disp -= PATCHER_CALL_SIZE;
2091
2092 /*                              PROFILE_CYCLE_STOP; */
2093
2094                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2095
2096                                 if (opt_showdisassemble) {
2097                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2098                                 }
2099
2100 /*                              PROFILE_CYCLE_START; */
2101                         }
2102                         else {
2103                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2104
2105                                 fieldtype = fi->type;
2106
2107                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2108                                         PROFILE_CYCLE_STOP;
2109
2110                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2111
2112                                         if (opt_showdisassemble) {
2113                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2114                                         }
2115
2116                                         PROFILE_CYCLE_START;
2117                                 }
2118
2119                                 disp = dseg_addaddress(cd, &(fi->value));
2120                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2121                         }
2122
2123                         /* This approach is much faster than moving the field
2124                            address inline into a register. */
2125
2126                         M_ALD(REG_ITMP1, RIP, disp);
2127
2128                         switch (fieldtype) {
2129                         case TYPE_INT:
2130                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2131                                 M_ILD(d, REG_ITMP1, 0);
2132                                 break;
2133                         case TYPE_LNG:
2134                         case TYPE_ADR:
2135                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2136                                 M_LLD(d, REG_ITMP1, 0);
2137                                 break;
2138                         case TYPE_FLT:
2139                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2140                                 M_FLD(d, REG_ITMP1, 0);
2141                                 break;
2142                         case TYPE_DBL:
2143                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2144                                 M_DLD(d, REG_ITMP1, 0);
2145                                 break;
2146                         }
2147                         emit_store_dst(jd, iptr, d);
2148                         break;
2149
2150                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2151
2152                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2153                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2154
2155                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2156                                 disp = dseg_addaddress(cd, NULL);
2157                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2158
2159                                 /* must be calculated before codegen_addpatchref */
2160
2161                                 if (opt_showdisassemble)
2162                                         disp -= PATCHER_CALL_SIZE;
2163
2164 /*                              PROFILE_CYCLE_STOP; */
2165
2166                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2167
2168                                 if (opt_showdisassemble) {
2169                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2170                                 }
2171
2172 /*                              PROFILE_CYCLE_START; */
2173                         }
2174                         else {
2175                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2176
2177                                 fieldtype = fi->type;
2178
2179                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2180                                         PROFILE_CYCLE_STOP;
2181
2182                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2183
2184                                         if (opt_showdisassemble) {
2185                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2186                                                 disp -= PATCHER_CALL_SIZE;
2187                                         }
2188
2189                                         PROFILE_CYCLE_START;
2190                                 }
2191
2192                                 disp = dseg_addaddress(cd, &(fi->value));
2193                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2194                         }
2195
2196                         /* This approach is much faster than moving the field
2197                            address inline into a register. */
2198
2199                         M_ALD(REG_ITMP1, RIP, disp);
2200
2201                         switch (fieldtype) {
2202                         case TYPE_INT:
2203                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2204                                 M_IST(s1, REG_ITMP1, 0);
2205                                 break;
2206                         case TYPE_LNG:
2207                         case TYPE_ADR:
2208                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2209                                 M_LST(s1, REG_ITMP1, 0);
2210                                 break;
2211                         case TYPE_FLT:
2212                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2213                                 M_FST(s1, REG_ITMP1, 0);
2214                                 break;
2215                         case TYPE_DBL:
2216                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2217                                 M_DST(s1, REG_ITMP1, 0);
2218                                 break;
2219                         }
2220                         break;
2221
2222                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2223                                           /* val = value (in current instruction)     */
2224                                           /* following NOP)                           */
2225
2226                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2227                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2228
2229                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2230                                 disp = dseg_addaddress(cd, NULL);
2231                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2232
2233                                 /* must be calculated before codegen_addpatchref */
2234
2235                                 if (opt_showdisassemble)
2236                                         disp -= PATCHER_CALL_SIZE;
2237
2238
2239 /*                              PROFILE_CYCLE_STOP; */
2240
2241                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2242
2243                                 if (opt_showdisassemble) {
2244                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2245                                 }
2246
2247 /*                              PROFILE_CYCLE_START; */
2248                         }
2249                         else {
2250                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2251
2252                                 fieldtype = fi->type;
2253
2254                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2255                                         PROFILE_CYCLE_STOP;
2256
2257                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2258
2259                                         if (opt_showdisassemble) {
2260                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2261                                         }
2262
2263                                         PROFILE_CYCLE_START;
2264                                 }
2265
2266                                 disp = dseg_addaddress(cd, &(fi->value));
2267                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2268                         }
2269
2270                         /* This approach is much faster than moving the field
2271                            address inline into a register. */
2272
2273                         M_ALD(REG_ITMP1, RIP, disp);
2274
2275                         switch (fieldtype) {
2276                         case TYPE_INT:
2277                         case TYPE_FLT:
2278                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2279                                 break;
2280                         case TYPE_LNG:
2281                         case TYPE_ADR:
2282                         case TYPE_DBL:
2283                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
2284                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2285                                 else {
2286                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2287                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
2288                                 }
2289                                 break;
2290                         }
2291                         break;
2292
2293                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2294
2295                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2296                         gen_nullptr_check(s1);
2297
2298                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2299                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2300
2301 /*                              PROFILE_CYCLE_STOP; */
2302
2303                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2304
2305                                 if (opt_showdisassemble) {
2306                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2307                                 }
2308
2309 /*                              PROFILE_CYCLE_START; */
2310
2311                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2312                                 disp = 0;
2313                         }
2314                         else {
2315                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2316
2317                                 fieldtype = fi->type;
2318                                 disp = fi->offset;
2319                         }
2320
2321                         switch (fieldtype) {
2322                         case TYPE_INT:
2323                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2324                                 M_ILD32(d, s1, disp);
2325                                 break;
2326                         case TYPE_LNG:
2327                         case TYPE_ADR:
2328                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2329                                 M_LLD32(d, s1, disp);
2330                                 break;
2331                         case TYPE_FLT:
2332                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2333                                 M_FLD32(d, s1, disp);
2334                                 break;
2335                         case TYPE_DBL:
2336                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2337                                 M_DLD32(d, s1, disp);
2338                                 break;
2339                         }
2340                         emit_store_dst(jd, iptr, d);
2341                         break;
2342
2343                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2344
2345                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2346                         gen_nullptr_check(s1);
2347
2348                         s2 = emit_load_s2(jd, iptr, REG_IFTMP);
2349
2350                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2351                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2352
2353 /*                              PROFILE_CYCLE_STOP; */
2354
2355                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2356
2357                                 if (opt_showdisassemble) {
2358                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2359                                 }
2360
2361 /*                              PROFILE_CYCLE_START; */
2362
2363                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2364                                 disp = 0;
2365                         }
2366                         else {
2367                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2368
2369                                 fieldtype = fi->type;
2370                                 disp = fi->offset;
2371                         }
2372
2373                         switch (fieldtype) {
2374                         case TYPE_INT:
2375                                 M_IST32(s2, s1, disp);
2376                                 break;
2377                         case TYPE_LNG:
2378                         case TYPE_ADR:
2379                                 M_LST32(s2, s1, disp);
2380                                 break;
2381                         case TYPE_FLT:
2382                                 M_FST32(s2, s1, disp);
2383                                 break;
2384                         case TYPE_DBL:
2385                                 M_DST32(s2, s1, disp);
2386                                 break;
2387                         }
2388                         break;
2389
2390                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2391                                           /* val = value (in current instruction)     */
2392                                           /* following NOP)                           */
2393
2394                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2395                         gen_nullptr_check(s1);
2396
2397                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2398                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2399
2400 /*                              PROFILE_CYCLE_STOP; */
2401
2402                                 codegen_addpatchref(cd, PATCHER_putfieldconst, uf, 0);
2403
2404                                 if (opt_showdisassemble) {
2405                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2406                                 }
2407
2408 /*                              PROFILE_CYCLE_START; */
2409
2410                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2411                                 disp = 0;
2412                         }
2413                         else {
2414                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2415
2416                                 fieldtype = fi->type;
2417                                 disp = fi->offset;
2418                         }
2419
2420                         switch (fieldtype) {
2421                         case TYPE_INT:
2422                         case TYPE_FLT:
2423                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2424                                 break;
2425                         case TYPE_LNG:
2426                         case TYPE_ADR:
2427                         case TYPE_DBL:
2428                                 /* XXX why no check for IS_IMM32? */
2429                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2430                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2431                                 break;
2432                         }
2433                         break;
2434
2435
2436                 /* branch operations **************************************************/
2437
2438                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2439
2440                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2441                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2442
2443                         PROFILE_CYCLE_STOP;
2444
2445 #ifdef ENABLE_VERIFIER
2446                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2447                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2448                                                                         iptr->sx.s23.s2.uc, 0);
2449
2450                                 if (opt_showdisassemble) {
2451                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2452                                 }
2453                         }
2454 #endif /* ENABLE_VERIFIER */
2455
2456                         M_CALL_IMM(0);                            /* passing exception pc */
2457                         M_POP(REG_ITMP2_XPC);
2458
2459                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2460                         M_JMP(REG_ITMP3);
2461                         break;
2462
2463                 case ICMD_GOTO:         /* ... ==> ...                                */
2464
2465                         M_JMP_IMM(0);
2466                         codegen_addreference(cd, iptr->dst.block);
2467                         break;
2468
2469                 case ICMD_JSR:          /* ... ==> ...                                */
2470
2471                         M_CALL_IMM(0);
2472                         codegen_addreference(cd, iptr->sx.s23.s3.jsrtarget.block);
2473                         break;
2474
2475                 case ICMD_RET:          /* ... ==> ...                                */
2476                                         /* s1.localindex = local variable                       */
2477
2478                         var = &(rd->locals[iptr->s1.localindex][TYPE_ADR]);
2479                         if (var->flags & INMEMORY) {
2480                                 M_ALD(REG_ITMP1, REG_SP, var->regoff * 8);
2481                                 M_JMP(REG_ITMP1);
2482                         } else
2483                                 M_JMP(var->regoff);
2484                         break;
2485
2486                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2487
2488                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2489                         M_TEST(s1);
2490                         M_BEQ(0);
2491                         codegen_addreference(cd, iptr->dst.block);
2492                         break;
2493
2494                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2495
2496                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2497                         M_TEST(s1);
2498                         M_BNE(0);
2499                         codegen_addreference(cd, iptr->dst.block);
2500                         break;
2501
2502                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2503
2504                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2505                         M_ICMP_IMM(iptr->sx.val.i, s1);
2506                         M_BEQ(0);
2507                         codegen_addreference(cd, iptr->dst.block);
2508                         break;
2509
2510                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2511
2512                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2513                         M_ICMP_IMM(iptr->sx.val.i, s1);
2514                         M_BLT(0);
2515                         codegen_addreference(cd, iptr->dst.block);
2516                         break;
2517
2518                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2519
2520                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2521                         M_ICMP_IMM(iptr->sx.val.i, s1);
2522                         M_BLE(0);
2523                         codegen_addreference(cd, iptr->dst.block);
2524                         break;
2525
2526                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2527
2528                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2529                         M_ICMP_IMM(iptr->sx.val.i, s1);
2530                         M_BNE(0);
2531                         codegen_addreference(cd, iptr->dst.block);
2532                         break;
2533
2534                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2535
2536                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2537                         M_ICMP_IMM(iptr->sx.val.i, s1);
2538                         M_BGT(0);
2539                         codegen_addreference(cd, iptr->dst.block);
2540                         break;
2541
2542                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2543
2544                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2545                         M_ICMP_IMM(iptr->sx.val.i, s1);
2546                         M_BGE(0);
2547                         codegen_addreference(cd, iptr->dst.block);
2548                         break;
2549
2550                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2551
2552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2553                         if (IS_IMM32(iptr->sx.val.l))
2554                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2555                         else {
2556                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2557                                 M_LCMP(REG_ITMP2, s1);
2558                         }
2559                         M_BEQ(0);
2560                         codegen_addreference(cd, iptr->dst.block);
2561                         break;
2562
2563                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2564
2565                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2566                         if (IS_IMM32(iptr->sx.val.l))
2567                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2568                         else {
2569                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2570                                 M_LCMP(REG_ITMP2, s1);
2571                         }
2572                         M_BLT(0);
2573                         codegen_addreference(cd, iptr->dst.block);
2574                         break;
2575
2576                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2577
2578                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2579                         if (IS_IMM32(iptr->sx.val.l))
2580                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2581                         else {
2582                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2583                                 M_LCMP(REG_ITMP2, s1);
2584                         }
2585                         M_BLE(0);
2586                         codegen_addreference(cd, iptr->dst.block);
2587                         break;
2588
2589                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2590
2591                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2592                         if (IS_IMM32(iptr->sx.val.l))
2593                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2594                         else {
2595                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2596                                 M_LCMP(REG_ITMP2, s1);
2597                         }
2598                         M_BNE(0);
2599                         codegen_addreference(cd, iptr->dst.block);
2600                         break;
2601
2602                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2603
2604                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2605                         if (IS_IMM32(iptr->sx.val.l))
2606                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2607                         else {
2608                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2609                                 M_LCMP(REG_ITMP2, s1);
2610                         }
2611                         M_BGT(0);
2612                         codegen_addreference(cd, iptr->dst.block);
2613                         break;
2614
2615                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2616
2617                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2618                         if (IS_IMM32(iptr->sx.val.l))
2619                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2620                         else {
2621                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2622                                 M_LCMP(REG_ITMP2, s1);
2623                         }
2624                         M_BGE(0);
2625                         codegen_addreference(cd, iptr->dst.block);
2626                         break;
2627
2628                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2629
2630                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2631                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2632                         M_ICMP(s2, s1);
2633                         M_BEQ(0);
2634                         codegen_addreference(cd, iptr->dst.block);
2635                         break;
2636
2637                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2638                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2639
2640                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2641                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2642                         M_LCMP(s2, s1);
2643                         M_BEQ(0);
2644                         codegen_addreference(cd, iptr->dst.block);
2645                         break;
2646
2647                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2648
2649                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2650                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2651                         M_ICMP(s2, s1);
2652                         M_BNE(0);
2653                         codegen_addreference(cd, iptr->dst.block);
2654                         break;
2655
2656                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2657                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2658
2659                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2660                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2661                         M_LCMP(s2, s1);
2662                         M_BNE(0);
2663                         codegen_addreference(cd, iptr->dst.block);
2664                         break;
2665
2666                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2667
2668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2670                         M_ICMP(s2, s1);
2671                         M_BLT(0);
2672                         codegen_addreference(cd, iptr->dst.block);
2673                         break;
2674
2675                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2676
2677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2678                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2679                         M_LCMP(s2, s1);
2680                         M_BLT(0);
2681                         codegen_addreference(cd, iptr->dst.block);
2682                         break;
2683
2684                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2685
2686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2688                         M_ICMP(s2, s1);
2689                         M_BGT(0);
2690                         codegen_addreference(cd, iptr->dst.block);
2691                         break;
2692
2693                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2694
2695                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2696                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2697                         M_LCMP(s2, s1);
2698                         M_BGT(0);
2699                         codegen_addreference(cd, iptr->dst.block);
2700                         break;
2701
2702                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2703
2704                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2705                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2706                         M_ICMP(s2, s1);
2707                         M_BLE(0);
2708                         codegen_addreference(cd, iptr->dst.block);
2709                         break;
2710
2711                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2712
2713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2715                         M_LCMP(s2, s1);
2716                         M_BLE(0);
2717                         codegen_addreference(cd, iptr->dst.block);
2718                         break;
2719
2720                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2721
2722                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2723                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2724                         M_ICMP(s2, s1);
2725                         M_BGE(0);
2726                         codegen_addreference(cd, iptr->dst.block);
2727                         break;
2728
2729                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2730
2731                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2732                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2733                         M_LCMP(s2, s1);
2734                         M_BGE(0);
2735                         codegen_addreference(cd, iptr->dst.block);
2736                         break;
2737
2738                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2739                 case ICMD_LRETURN:
2740
2741                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2742                         M_INTMOVE(s1, REG_RESULT);
2743                         goto nowperformreturn;
2744
2745                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2746
2747                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2748                         M_INTMOVE(s1, REG_RESULT);
2749
2750 #ifdef ENABLE_VERIFIER
2751                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2752                                 PROFILE_CYCLE_STOP;
2753
2754                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2755                                                                         iptr->sx.s23.s2.uc, 0);
2756
2757                                 if (opt_showdisassemble) {
2758                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2759                                 }
2760
2761                                 PROFILE_CYCLE_START;
2762                         }
2763 #endif /* ENABLE_VERIFIER */
2764                         goto nowperformreturn;
2765
2766                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2767                 case ICMD_DRETURN:
2768
2769                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2770                         M_FLTMOVE(s1, REG_FRESULT);
2771                         goto nowperformreturn;
2772
2773                 case ICMD_RETURN:      /* ...  ==> ...                                */
2774
2775 nowperformreturn:
2776                         {
2777                         s4 i, p;
2778
2779                         p = cd->stackframesize;
2780
2781 #if !defined(NDEBUG)
2782                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2783                                 emit_verbosecall_exit(jd);
2784 #endif /* !defined(NDEBUG) */
2785
2786 #if defined(ENABLE_THREADS)
2787                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2788                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2789
2790                                 /* we need to save the proper return value */
2791                                 switch (iptr->opc) {
2792                                 case ICMD_IRETURN:
2793                                 case ICMD_ARETURN:
2794                                 case ICMD_LRETURN:
2795                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2796                                         break;
2797                                 case ICMD_FRETURN:
2798                                 case ICMD_DRETURN:
2799                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2800                                         break;
2801                                 }
2802
2803                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2804                                 M_CALL(REG_ITMP1);
2805
2806                                 /* and now restore the proper return value */
2807                                 switch (iptr->opc) {
2808                                 case ICMD_IRETURN:
2809                                 case ICMD_ARETURN:
2810                                 case ICMD_LRETURN:
2811                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2812                                         break;
2813                                 case ICMD_FRETURN:
2814                                 case ICMD_DRETURN:
2815                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2816                                         break;
2817                                 }
2818                         }
2819 #endif
2820
2821                         /* restore saved registers */
2822
2823                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2824                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2825                         }
2826                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2827                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2828                         }
2829
2830                         /* deallocate stack */
2831
2832                         if (cd->stackframesize)
2833                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2834
2835                         /* generate method profiling code */
2836
2837                         PROFILE_CYCLE_STOP;
2838
2839                         M_RET;
2840                         }
2841                         break;
2842
2843
2844                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2845                         {
2846                                 s4 i, l;
2847                                 branch_target_t *table;
2848
2849                                 table = iptr->dst.table;
2850
2851                                 l = iptr->sx.s23.s2.tablelow;
2852                                 i = iptr->sx.s23.s3.tablehigh;
2853
2854                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2855                                 M_INTMOVE(s1, REG_ITMP1);
2856
2857                                 if (l != 0)
2858                                         M_ISUB_IMM(l, REG_ITMP1);
2859
2860                                 /* number of targets */
2861                                 i = i - l + 1;
2862
2863                 /* range check */
2864                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2865                                 M_BA(0);
2866
2867                                 codegen_addreference(cd, table[0].block); /* default target */
2868
2869                                 /* build jump table top down and use address of lowest entry */
2870
2871                                 table += i;
2872
2873                                 while (--i >= 0) {
2874                                         dseg_addtarget(cd, table->block);
2875                                         --table;
2876                                 }
2877
2878                                 /* length of dataseg after last dseg_addtarget is used
2879                                    by load */
2880
2881                                 M_MOV_IMM(0, REG_ITMP2);
2882                                 dseg_adddata(cd);
2883                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2884                                 M_JMP(REG_ITMP1);
2885                         }
2886                         break;
2887
2888
2889                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2890                         {
2891                                 s4 i;
2892                                 lookup_target_t *lookup;
2893
2894                                 lookup = iptr->dst.lookup;
2895
2896                                 i = iptr->sx.s23.s2.lookupcount;
2897
2898                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2899                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2900
2901                                 while (--i >= 0) {
2902                                         M_ICMP_IMM(lookup->value, s1);
2903                                         M_BEQ(0);
2904                                         codegen_addreference(cd, lookup->target.block);
2905                                         lookup++;
2906                                 }
2907
2908                                 M_JMP_IMM(0);
2909
2910                                 codegen_addreference(cd, iptr->sx.s23.s3.lookupdefault.block);
2911                         }
2912                         break;
2913
2914
2915                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2916
2917                         bte = iptr->sx.s23.s3.bte;
2918                         md  = bte->md;
2919                         goto gen_method;
2920
2921                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2922
2923                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2924                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2925                 case ICMD_INVOKEINTERFACE:
2926
2927                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2928                                 lm = NULL;
2929                                 um = iptr->sx.s23.s3.um;
2930                                 md = um->methodref->parseddesc.md;
2931                         }
2932                         else {
2933                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2934                                 um = NULL;
2935                                 md = lm->parseddesc;
2936                         }
2937
2938 gen_method:
2939                         s3 = md->paramcount;
2940
2941                         MCODECHECK((20 * s3) + 128);
2942
2943                         /* copy arguments to registers or stack location */
2944
2945                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2946                                 src = iptr->sx.s23.s2.args[s3];
2947
2948                                 if (src->varkind == ARGVAR)
2949                                         continue;
2950
2951                                 if (IS_INT_LNG_TYPE(src->type)) {
2952                                         if (!md->params[s3].inmemory) {
2953                                                 s1 = rd->argintregs[md->params[s3].regoff];
2954                                                 d = emit_load(jd, iptr, src, s1);
2955                                                 M_INTMOVE(d, s1);
2956                                         }
2957                                         else {
2958                                                 d = emit_load(jd, iptr, src, REG_ITMP1);
2959                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2960                                         }
2961                                 }
2962                                 else {
2963                                         if (!md->params[s3].inmemory) {
2964                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2965                                                 d = emit_load(jd, iptr, src, s1);
2966                                                 M_FLTMOVE(d, s1);
2967                                         }
2968                                         else {
2969                                                 d = emit_load(jd, iptr, src, REG_FTMP1);
2970
2971                                                 if (IS_2_WORD_TYPE(src->type))
2972                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2973                                                 else
2974                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2975                                         }
2976                                 }
2977                         }
2978
2979                         /* generate method profiling code */
2980
2981                         PROFILE_CYCLE_STOP;
2982
2983                         switch (iptr->opc) {
2984                         case ICMD_BUILTIN:
2985                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2986                                 M_CALL(REG_ITMP1);
2987
2988                                 if (INSTRUCTION_MUST_CHECK(iptr)) {
2989                                         M_TEST(REG_RESULT);
2990                                         M_BEQ(0);
2991                                         codegen_add_fillinstacktrace_ref(cd);
2992                                 }
2993                                 break;
2994
2995                         case ICMD_INVOKESPECIAL:
2996                                 M_TEST(REG_A0);
2997                                 M_BEQ(0);
2998                                 codegen_add_nullpointerexception_ref(cd);
2999
3000                                 /* fall through */
3001
3002                         case ICMD_INVOKESTATIC:
3003                                 if (lm == NULL) {
3004                                         disp = dseg_addaddress(cd, NULL);
3005                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
3006
3007                                         /* must be calculated before codegen_addpatchref */
3008
3009                                         if (opt_showdisassemble)
3010                                                 disp -= PATCHER_CALL_SIZE;
3011
3012                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
3013                                                                                 um, disp);
3014
3015                                         if (opt_showdisassemble) {
3016                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3017                                         }
3018
3019 /*                                      a = 0; */
3020                                 }
3021                                 else {
3022                                         disp = dseg_addaddress(cd, lm->stubroutine);
3023                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
3024
3025 /*                                      a = (ptrint) lm->stubroutine; */
3026                                 }
3027
3028 /*                              M_MOV_IMM(a, REG_ITMP2); */
3029                                 M_ALD(REG_ITMP2, RIP, disp);
3030                                 M_CALL(REG_ITMP2);
3031                                 break;
3032
3033                         case ICMD_INVOKEVIRTUAL:
3034                                 gen_nullptr_check(REG_A0);
3035
3036                                 if (lm == NULL) {
3037                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
3038
3039                                         if (opt_showdisassemble) {
3040                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3041                                         }
3042
3043                                         s1 = 0;
3044                                 }
3045                                 else
3046                                         s1 = OFFSET(vftbl_t, table[0]) +
3047                                                 sizeof(methodptr) * lm->vftblindex;
3048
3049                                 M_ALD(REG_METHODPTR, REG_A0,
3050                                           OFFSET(java_objectheader, vftbl));
3051                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
3052                                 M_CALL(REG_ITMP3);
3053                                 break;
3054
3055                         case ICMD_INVOKEINTERFACE:
3056                                 gen_nullptr_check(REG_A0);
3057
3058                                 if (lm == NULL) {
3059                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
3060
3061                                         if (opt_showdisassemble) {
3062                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3063                                         }
3064
3065                                         s1 = 0;
3066                                         s2 = 0;
3067                                 }
3068                                 else {
3069                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3070                                                 sizeof(methodptr) * lm->class->index;
3071
3072                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3073                                 }
3074
3075                                 M_ALD(REG_METHODPTR, REG_A0,
3076                                           OFFSET(java_objectheader, vftbl));
3077                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
3078                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
3079                                 M_CALL(REG_ITMP3);
3080                                 break;
3081                         }
3082
3083                         /* generate method profiling code */
3084
3085                         PROFILE_CYCLE_START;
3086
3087                         /* store return value */
3088
3089                         d = md->returntype.type;
3090
3091                         if (d != TYPE_VOID) {
3092                                 if (IS_INT_LNG_TYPE(d)) {
3093                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3094                                         M_INTMOVE(REG_RESULT, s1);
3095                                 }
3096                                 else {
3097                                         s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
3098                                         M_FLTMOVE(REG_FRESULT, s1);
3099                                 }
3100                                 emit_store_dst(jd, iptr, s1);
3101                         }
3102                         break;
3103
3104
3105                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3106
3107                                       /* val.a: (classinfo *) superclass              */
3108
3109                         /*  superclass is an interface:
3110                          *
3111                          *  OK if ((sub == NULL) ||
3112                          *         (sub->vftbl->interfacetablelength > super->index) &&
3113                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3114                          *
3115                          *  superclass is a class:
3116                          *
3117                          *  OK if ((sub == NULL) || (0
3118                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3119                          *         super->vftbl->diffval));
3120                          */
3121
3122                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
3123                                 /* object type cast-check */
3124
3125                                 classinfo *super;
3126                                 vftbl_t   *supervftbl;
3127                                 s4         superindex;
3128
3129                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3130                                         super = NULL;
3131                                         superindex = 0;
3132                                         supervftbl = NULL;
3133                                 }
3134                                 else {
3135                                         super = iptr->sx.s23.s3.c.cls;
3136                                         superindex = super->index;
3137                                         supervftbl = super->vftbl;
3138                                 }
3139
3140 #if defined(ENABLE_THREADS)
3141                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3142 #endif
3143                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3144
3145                                 /* calculate interface checkcast code size */
3146
3147                                 s2 = 3; /* mov_membase_reg */
3148                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3149
3150                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3151                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3152                                         3 /* test */ + 6 /* jcc */;
3153
3154                                 if (super == NULL)
3155                                         s2 += (opt_showdisassemble ? 5 : 0);
3156
3157                                 /* calculate class checkcast code size */
3158
3159                                 s3 = 3; /* mov_membase_reg */
3160                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3161                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3162
3163 #if 0
3164                                 if (s1 != REG_ITMP1) {
3165                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3166                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3167                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3168                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3169                                         a += 3;    /* sub */
3170
3171                                 } else
3172 #endif
3173                                         {
3174                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3175                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3176                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3177                                         }
3178
3179                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3180
3181                                 if (super == NULL)
3182                                         s3 += (opt_showdisassemble ? 5 : 0);
3183
3184                                 /* if class is not resolved, check which code to call */
3185
3186                                 if (super == NULL) {
3187                                         M_TEST(s1);
3188                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3189
3190                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3191                                                                                 iptr->sx.s23.s3.c.ref, 0);
3192
3193                                         if (opt_showdisassemble) {
3194                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3195                                         }
3196
3197                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3198                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3199                                         M_BEQ(s2 + 5);
3200                                 }
3201
3202                                 /* interface checkcast code */
3203
3204                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3205                                         if (super != NULL) {
3206                                                 M_TEST(s1);
3207                                                 M_BEQ(s2);
3208                                         }
3209
3210                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3211
3212                                         if (super == NULL) {
3213                                                 codegen_addpatchref(cd,
3214                                                                                         PATCHER_checkcast_instanceof_interface,
3215                                                                                         iptr->sx.s23.s3.c.ref,
3216                                                                                         0);
3217
3218                                                 if (opt_showdisassemble) {
3219                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3220                                                 }
3221                                         }
3222
3223                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3224                                                                                           OFFSET(vftbl_t, interfacetablelength),
3225                                                                                           REG_ITMP3);
3226                                         /* XXX TWISTI: should this be int arithmetic? */
3227                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3228                                         M_TEST(REG_ITMP3);
3229                                         M_BLE(0);
3230                                         codegen_add_classcastexception_ref(cd, s1);
3231                                         emit_mov_membase32_reg(cd, REG_ITMP2,
3232                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3233                                                                                          superindex * sizeof(methodptr*),
3234                                                                                          REG_ITMP3);
3235                                         M_TEST(REG_ITMP3);
3236                                         M_BEQ(0);
3237                                         codegen_add_classcastexception_ref(cd, s1);
3238
3239                                         if (super == NULL)
3240                                                 M_JMP_IMM(s3);
3241                                 }
3242
3243                                 /* class checkcast code */
3244
3245                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3246                                         if (super != NULL) {
3247                                                 M_TEST(s1);
3248                                                 M_BEQ(s3);
3249                                         }
3250
3251                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3252
3253                                         if (super == NULL) {
3254                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3255                                                                                         iptr->sx.s23.s3.c.ref,
3256                                                                                         0);
3257
3258                                                 if (opt_showdisassemble) {
3259                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3260                                                 }
3261                                         }
3262
3263                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3264 #if defined(ENABLE_THREADS)
3265                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3266 #endif
3267                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3268                                                                                           OFFSET(vftbl_t, baseval),
3269                                                                                           REG_ITMP2);
3270                                         /*                                      if (s1 != REG_ITMP1) { */
3271                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3272                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3273                                         /*                                                                                              REG_ITMP1); */
3274                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3275                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3276                                         /*                                                                                              REG_ITMP3); */
3277                                         /*  #if defined(ENABLE_THREADS) */
3278                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3279                                         /*  #endif */
3280                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3281
3282                                         /*                                      } else { */
3283                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3284                                                                                           OFFSET(vftbl_t, baseval),
3285                                                                                           REG_ITMP3);
3286                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3287                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3288                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3289                                         /*                                      } */
3290 #if defined(ENABLE_THREADS)
3291                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3292 #endif
3293                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3294                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3295                                         codegen_add_classcastexception_ref(cd, s1);
3296                                 }
3297
3298                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
3299                         }
3300                         else {
3301                                 /* array type cast-check */
3302
3303                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3304                                 M_INTMOVE(s1, REG_A0);
3305
3306                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3307                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3308                                                                                 iptr->sx.s23.s3.c.ref, 0);
3309
3310                                         if (opt_showdisassemble) {
3311                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3312                                         }
3313                                 }
3314
3315                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3316                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3317                                 M_CALL(REG_ITMP1);
3318
3319                                 /* s1 may have been destroyed over the function call */
3320                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3321                                 M_TEST(REG_RESULT);
3322                                 M_BEQ(0);
3323                                 codegen_add_classcastexception_ref(cd, s1);
3324
3325                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3326                         }
3327
3328                         M_INTMOVE(s1, d);
3329                         emit_store_dst(jd, iptr, d);
3330                         break;
3331
3332                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3333
3334                                       /* val.a: (classinfo *) superclass              */
3335
3336                         /*  superclass is an interface:
3337                          *
3338                          *  return (sub != NULL) &&
3339                          *         (sub->vftbl->interfacetablelength > super->index) &&
3340                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3341                          *
3342                          *  superclass is a class:
3343                          *
3344                          *  return ((sub != NULL) && (0
3345                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3346                          *          super->vftbl->diffvall));
3347                          */
3348
3349                         {
3350                         classinfo *super;
3351                         vftbl_t   *supervftbl;
3352                         s4         superindex;
3353
3354                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3355                                 super = NULL;
3356                                 superindex = 0;
3357                                 supervftbl = NULL;
3358
3359                         } else {
3360                                 super = iptr->sx.s23.s3.c.cls;
3361                                 superindex = super->index;
3362                                 supervftbl = super->vftbl;
3363                         }
3364
3365 #if defined(ENABLE_THREADS)
3366             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3367 #endif
3368
3369                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3370                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3371                         if (s1 == d) {
3372                                 M_INTMOVE(s1, REG_ITMP1);
3373                                 s1 = REG_ITMP1;
3374                         }
3375
3376                         /* calculate interface instanceof code size */
3377
3378                         s2 = 3; /* mov_membase_reg */
3379                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3380                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3381                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3382                                 3 /* test */ + 4 /* setcc */;
3383
3384                         if (!super)
3385                                 s2 += (opt_showdisassemble ? 5 : 0);
3386
3387                         /* calculate class instanceof code size */
3388
3389                         s3 = 3; /* mov_membase_reg */
3390                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3391                         s3 += 10; /* mov_imm_reg */
3392                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3393                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3394                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3395                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3396                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3397                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3398                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3399
3400                         if (!super)
3401                                 s3 += (opt_showdisassemble ? 5 : 0);
3402
3403                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3404
3405                         /* if class is not resolved, check which code to call */
3406
3407                         if (!super) {
3408                                 emit_test_reg_reg(cd, s1, s1);
3409                                 emit_jcc(cd, CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3410                                                                                          7 + 6 + s2 + 5 + s3));
3411
3412                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3413                                                                         iptr->sx.s23.s3.c.ref, 0);
3414
3415                                 if (opt_showdisassemble) {
3416                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3417                                 }
3418
3419                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3420                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3421                                 emit_jcc(cd, CC_Z, s2 + 5);
3422                         }
3423
3424                         /* interface instanceof code */
3425
3426                         if (!super || (super->flags & ACC_INTERFACE)) {
3427                                 if (super) {
3428                                         emit_test_reg_reg(cd, s1, s1);
3429                                         emit_jcc(cd, CC_Z, s2);
3430                                 }
3431
3432                                 emit_mov_membase_reg(cd, s1,
3433                                                                            OFFSET(java_objectheader, vftbl),
3434                                                                            REG_ITMP1);
3435                                 if (!super) {
3436                                         codegen_addpatchref(cd,
3437                                                                                 PATCHER_checkcast_instanceof_interface,
3438                                                                                 iptr->sx.s23.s3.c.ref, 0);
3439
3440                                         if (opt_showdisassemble) {
3441                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3442                                         }
3443                                 }
3444
3445                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3446                                                                                   OFFSET(vftbl_t, interfacetablelength),
3447                                                                                   REG_ITMP3);
3448                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3449                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3450
3451                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3452
3453                                 emit_jcc(cd, CC_LE, a);
3454                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3455                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3456                                                                                  superindex * sizeof(methodptr*),
3457                                                                                  REG_ITMP1);
3458                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3459                                 emit_setcc_reg(cd, CC_NE, d);
3460
3461                                 if (!super)
3462                                         emit_jmp_imm(cd, s3);
3463                         }
3464
3465                         /* class instanceof code */
3466
3467                         if (!super || !(super->flags & ACC_INTERFACE)) {
3468                                 if (super) {
3469                                         emit_test_reg_reg(cd, s1, s1);
3470                                         emit_jcc(cd, CC_E, s3);
3471                                 }
3472
3473                                 emit_mov_membase_reg(cd, s1,
3474                                                                            OFFSET(java_objectheader, vftbl),
3475                                                                            REG_ITMP1);
3476
3477                                 if (!super) {
3478                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3479                                                                                 iptr->sx.s23.s3.c.ref, 0);
3480
3481                                         if (opt_showdisassemble) {
3482                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3483                                         }
3484                                 }
3485
3486                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3487 #if defined(ENABLE_THREADS)
3488                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3489 #endif
3490                                 emit_movl_membase_reg(cd, REG_ITMP1,
3491                                                                                 OFFSET(vftbl_t, baseval),
3492                                                                                 REG_ITMP1);
3493                                 emit_movl_membase_reg(cd, REG_ITMP2,
3494                                                                                 OFFSET(vftbl_t, diffval),
3495                                                                                 REG_ITMP3);
3496                                 emit_movl_membase_reg(cd, REG_ITMP2,
3497                                                                                 OFFSET(vftbl_t, baseval),
3498                                                                                 REG_ITMP2);
3499 #if defined(ENABLE_THREADS)
3500                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3501 #endif
3502                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3503                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3504                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3505                                 emit_setcc_reg(cd, CC_BE, d);
3506                         }
3507                         emit_store_dst(jd, iptr, d);
3508                         }
3509                         break;
3510
3511                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3512
3513                         /* check for negative sizes and copy sizes to stack if necessary  */
3514
3515                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
3516
3517                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3518
3519                                 src = iptr->sx.s23.s2.args[s1];
3520
3521                                 /* copy SAVEDVAR sizes to stack */
3522
3523                                 if (src->varkind != ARGVAR) {
3524                                         s2 = emit_load(jd, iptr, src, REG_ITMP1);
3525                                         M_LST(s2, REG_SP, s1 * 8);
3526                                 }
3527                         }
3528
3529                         /* is a patcher function set? */
3530
3531                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3532                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3533                                                                         iptr->sx.s23.s3.c.ref, 0);
3534
3535                                 if (opt_showdisassemble) {
3536                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3537                                 }
3538                         }
3539
3540                         /* a0 = dimension count */
3541
3542                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
3543
3544                         /* a1 = classinfo */
3545
3546                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3547
3548                         /* a2 = pointer to dimensions = stack pointer */
3549
3550                         M_MOV(REG_SP, REG_A2);
3551
3552                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3553                         M_CALL(REG_ITMP1);
3554
3555                         /* check for exception before result assignment */
3556
3557                         M_TEST(REG_RESULT);
3558                         M_BEQ(0);
3559                         codegen_add_fillinstacktrace_ref(cd);
3560
3561                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3562                         M_INTMOVE(REG_RESULT, s1);
3563                         emit_store_dst(jd, iptr, s1);
3564                         break;
3565
3566                 default:
3567                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3568                         return false;
3569         } /* switch */
3570
3571         } /* for instruction */
3572
3573         /* copy values to interface registers */
3574
3575         len = bptr->outdepth;
3576         MCODECHECK(512);
3577 #if defined(ENABLE_LSRA)
3578         if (!opt_lsra)
3579 #endif
3580         while (len) {
3581                 len--;
3582                 src = bptr->outvars[len];
3583                 if ((src->varkind != STACKVAR)) {
3584                         s2 = src->type;
3585                         if (IS_FLT_DBL_TYPE(s2)) {
3586                                 s1 = emit_load(jd, iptr, src, REG_FTMP1);
3587                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3588                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3589
3590                                 } else {
3591                                         emit_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3592                                 }
3593
3594                         } else {
3595                                 s1 = emit_load(jd, iptr, src, REG_ITMP1);
3596                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3597                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3598
3599                                 } else {
3600                                         emit_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3601                                 }
3602                         }
3603                 }
3604         }
3605
3606         /* At the end of a basic block we may have to append some nops,
3607            because the patcher stub calling code might be longer than the
3608            actual instruction. So codepatching does not change the
3609            following block unintentionally. */
3610
3611         if (cd->mcodeptr < cd->lastmcodeptr) {
3612                 while (cd->mcodeptr < cd->lastmcodeptr) {
3613                         M_NOP;
3614                 }
3615         }
3616
3617         } /* if (bptr -> flags >= BBREACHED) */
3618         } /* for basic block */
3619
3620         dseg_createlinenumbertable(cd);
3621
3622         /* generate stubs */
3623
3624         emit_exception_stubs(jd);
3625         emit_patcher_stubs(jd);
3626 #if 0
3627         emit_replacement_stubs(jd);
3628 #endif
3629
3630         codegen_finish(jd);
3631
3632         /* everything's ok */
3633
3634         return true;
3635 }
3636
3637
3638 /* createcompilerstub **********************************************************
3639
3640    Creates a stub routine which calls the compiler.
3641
3642 *******************************************************************************/
3643
3644 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3645 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3646
3647 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3648
3649
3650 u1 *createcompilerstub(methodinfo *m)
3651 {
3652         u1          *s;                     /* memory to hold the stub            */
3653         ptrint      *d;
3654         codeinfo    *code;
3655         codegendata *cd;
3656         s4           dumpsize;
3657
3658         s = CNEW(u1, COMPILERSTUB_SIZE);
3659
3660         /* set data pointer and code pointer */
3661
3662         d = (ptrint *) s;
3663         s = s + COMPILERSTUB_DATASIZE;
3664
3665         /* mark start of dump memory area */
3666
3667         dumpsize = dump_size();
3668
3669         cd = DNEW(codegendata);
3670         cd->mcodeptr = s;
3671
3672         /* Store the codeinfo pointer in the same place as in the
3673            methodheader for compiled methods. */
3674
3675         code = code_codeinfo_new(m);
3676
3677         d[0] = (ptrint) asm_call_jit_compiler;
3678         d[1] = (ptrint) m;
3679         d[2] = (ptrint) code;
3680
3681         /* code for the stub */
3682
3683         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3684         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3685         M_JMP(REG_ITMP3);
3686
3687 #if defined(ENABLE_STATISTICS)
3688         if (opt_stat)
3689                 count_cstub_len += COMPILERSTUB_SIZE;
3690 #endif
3691
3692         /* release dump area */
3693
3694         dump_release(dumpsize);
3695
3696         return s;
3697 }
3698
3699
3700 /* createnativestub ************************************************************
3701
3702    Creates a stub routine which calls a native method.
3703
3704 *******************************************************************************/
3705
3706 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3707 {
3708         methodinfo   *m;
3709         codeinfo     *code;
3710         codegendata  *cd;
3711         registerdata *rd;
3712         methoddesc   *md;
3713         s4            nativeparams;
3714         s4            i, j;                 /* count variables                    */
3715         s4            t;
3716         s4            s1, s2;
3717
3718         /* get required compiler data */
3719
3720         m    = jd->m;
3721         code = jd->code;
3722         cd   = jd->cd;
3723         rd   = jd->rd;
3724
3725         /* initialize variables */
3726
3727         md = m->parseddesc;
3728         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3729
3730         /* calculate stack frame size */
3731
3732         cd->stackframesize =
3733                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3734                 sizeof(localref_table) / SIZEOF_VOID_P +
3735                 INT_ARG_CNT + FLT_ARG_CNT +
3736                 1 +                       /* functionptr, TODO: store in data segment */
3737                 nmd->memuse;
3738
3739         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3740
3741         /* create method header */
3742
3743         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
3744         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
3745         (void) dseg_adds4(cd, 0);                              /* IsSync          */
3746         (void) dseg_adds4(cd, 0);                              /* IsLeaf          */
3747         (void) dseg_adds4(cd, 0);                              /* IntSave         */
3748         (void) dseg_adds4(cd, 0);                              /* FltSave         */
3749         (void) dseg_addlinenumbertablesize(cd);
3750         (void) dseg_adds4(cd, 0);                              /* ExTableSize     */
3751
3752         /* generate native method profiling code */
3753
3754         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3755                 /* count frequency */
3756
3757                 M_MOV_IMM(code, REG_ITMP3);
3758                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3759         }
3760
3761         /* generate stub code */
3762
3763         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3764
3765 #if !defined(NDEBUG)
3766         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3767                 emit_verbosecall_enter(jd);
3768 #endif
3769
3770         /* get function address (this must happen before the stackframeinfo) */
3771
3772 #if !defined(WITH_STATIC_CLASSPATH)
3773         if (f == NULL) {
3774                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
3775
3776                 if (opt_showdisassemble) {
3777                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3778                 }
3779         }
3780 #endif
3781
3782         M_MOV_IMM(f, REG_ITMP3);
3783
3784
3785         /* save integer and float argument registers */
3786
3787         for (i = 0, j = 0; i < md->paramcount; i++) {
3788                 if (!md->params[i].inmemory) {
3789                         s1 = md->params[i].regoff;
3790
3791                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3792                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3793                         else
3794                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3795
3796                         j++;
3797                 }
3798         }
3799
3800         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3801
3802         /* create dynamic stack info */
3803
3804         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3805         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3806         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3807         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3808         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3809         M_CALL(REG_ITMP1);
3810
3811         /* restore integer and float argument registers */
3812
3813         for (i = 0, j = 0; i < md->paramcount; i++) {
3814                 if (!md->params[i].inmemory) {
3815                         s1 = md->params[i].regoff;
3816
3817                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3818                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3819                         else
3820                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3821
3822                         j++;
3823                 }
3824         }
3825
3826         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3827
3828
3829         /* copy or spill arguments to new locations */
3830
3831         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3832                 t = md->paramtypes[i].type;
3833
3834                 if (IS_INT_LNG_TYPE(t)) {
3835                         if (!md->params[i].inmemory) {
3836                                 s1 = rd->argintregs[md->params[i].regoff];
3837
3838                                 if (!nmd->params[j].inmemory) {
3839                                         s2 = rd->argintregs[nmd->params[j].regoff];
3840                                         M_INTMOVE(s1, s2);
3841
3842                                 } else {
3843                                         s2 = nmd->params[j].regoff;
3844                                         M_LST(s1, REG_SP, s2 * 8);
3845                                 }
3846
3847                         } else {
3848                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3849                                 s2 = nmd->params[j].regoff;
3850                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3851                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3852                         }
3853
3854                 } else {
3855                         /* We only copy spilled float arguments, as the float argument    */
3856                         /* registers keep unchanged.                                      */
3857
3858                         if (md->params[i].inmemory) {
3859                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3860                                 s2 = nmd->params[j].regoff;
3861
3862                                 if (IS_2_WORD_TYPE(t)) {
3863                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3864                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3865                                 } else {
3866                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3867                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3868                                 }
3869                         }
3870                 }
3871         }
3872
3873         /* put class into second argument register */
3874
3875         if (m->flags & ACC_STATIC)
3876                 M_MOV_IMM(m->class, REG_A1);
3877
3878         /* put env into first argument register */
3879
3880         M_MOV_IMM(_Jv_env, REG_A0);
3881
3882         /* do the native function call */
3883
3884         M_CALL(REG_ITMP3);
3885
3886         /* save return value */
3887
3888         if (md->returntype.type != TYPE_VOID) {
3889                 if (IS_INT_LNG_TYPE(md->returntype.type))
3890                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3891                 else
3892                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3893         }
3894
3895 #if !defined(NDEBUG)
3896         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3897                 emit_verbosecall_exit(jd);
3898 #endif
3899
3900         /* remove native stackframe info */
3901
3902         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3903         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3904         M_CALL(REG_ITMP1);
3905         M_MOV(REG_RESULT, REG_ITMP3);
3906
3907         /* restore return value */
3908
3909         if (md->returntype.type != TYPE_VOID) {
3910                 if (IS_INT_LNG_TYPE(md->returntype.type))
3911                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3912                 else
3913                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3914         }
3915
3916         /* remove stackframe */
3917
3918         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3919
3920         /* test for exception */
3921
3922         M_TEST(REG_ITMP3);
3923         M_BNE(1);
3924         M_RET;
3925
3926         /* handle exception */
3927
3928         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3929         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3930         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3931
3932         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3933         M_JMP(REG_ITMP3);
3934
3935         /* generate patcher stubs */
3936
3937         emit_patcher_stubs(jd);
3938
3939         codegen_finish(jd);
3940
3941         return code->entrypoint;
3942 }
3943
3944
3945 /*
3946  * These are local overrides for various environment variables in Emacs.
3947  * Please do not remove this and leave it at the end of the file, where
3948  * Emacs will automagically detect them.
3949  * ---------------------------------------------------------------------
3950  * Local variables:
3951  * mode: c
3952  * indent-tabs-mode: t
3953  * c-basic-offset: 4
3954  * tab-width: 4
3955  * End:
3956  * vim:noexpandtab:sw=4:ts=4:
3957  */