src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id: codegen.c 7281 2007-02-03 19:51:36Z twisti $
  26
  27 */
  28
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <stdio.h>
  34
  35 #include "vm/types.h"
  36
  37 #include "md-abi.h"
  38
  39 #include "vm/jit/x86_64/arch.h"
  40 #include "vm/jit/x86_64/codegen.h"
  41 #include "vm/jit/x86_64/emit.h"
  42
  43 #include "mm/memory.h"
  44 #include "native/jni.h"
  45 #include "native/native.h"
  46
  47 #if defined(ENABLE_THREADS)
  48 # include "threads/native/lock.h"
  49 #endif
  50
  51 #include "vm/builtin.h"
  52 #include "vm/exceptions.h"
  53 #include "vm/global.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/asmpart.h"
  58 #include "vm/jit/codegen-common.h"
  59 #include "vm/jit/dseg.h"
  60 #include "vm/jit/emit-common.h"
  61 #include "vm/jit/jit.h"
  62 #include "vm/jit/methodheader.h"
  63 #include "vm/jit/parse.h"
  64 #include "vm/jit/patcher.h"
  65 #include "vm/jit/reg.h"
  66 #include "vm/jit/replace.h"
  67 #include "vm/jit/stacktrace.h"
  68
  69 #if defined(ENABLE_LSRA)
  70 # include "vm/jit/allocator/lsra.h"
  71 #endif
  72
  73 #include "vmcore/loader.h"
  74 #include "vmcore/options.h"
  75 #include "vmcore/statistics.h"
  76
  77
  78 /* codegen *********************************************************************
  79
  80    Generates machine code.
  81
  82 *******************************************************************************/
  83
  84 bool codegen(jitdata *jd)
  85 {
  86         methodinfo         *m;
  87         codeinfo           *code;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         varinfo            *var, *dst;
  94         basicblock         *bptr;
  95         instruction        *iptr;
  96         exception_entry    *ex;
  97         constant_classref  *cr;
  98         unresolved_class   *uc;
  99         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 100         unresolved_method  *um;
 101         builtintable_entry *bte;
 102         methoddesc         *md;
 103         fieldinfo          *fi;
 104         unresolved_field   *uf;
 105         s4                  fieldtype;
 106         s4                 varindex;
 107
 108         /* get required compiler data */
 109
 110         m    = jd->m;
 111         code = jd->code;
 112         cd   = jd->cd;
 113         rd   = jd->rd;
 114
 115         /* prevent compiler warnings */
 116
 117         d   = 0;
 118         lm  = NULL;
 119         um  = NULL;
 120         bte = NULL;
 121
 122         {
 123         s4 i, p, t, l;
 124         s4 savedregs_num;
 125
 126         savedregs_num = 0;
 127
 128         /* space to save used callee saved registers */
 129
 130         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 131         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 132
 133         cd->stackframesize = rd->memuse + savedregs_num;
 134
 135 #if defined(ENABLE_THREADS)
 136         /* space to save argument of monitor_enter */
 137
 138         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 139                 cd->stackframesize++;
 140 #endif
 141
 142         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 143            native code e.g. libc or jni (alignment problems with
 144            movaps). */
 145
 146         if (!jd->isleafmethod || opt_verbosecall)
 147                 cd->stackframesize |= 0x1;
 148
 149         /* create method header */
 150
 151         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 152         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 153
 154 #if defined(ENABLE_THREADS)
 155         /* IsSync contains the offset relative to the stack pointer for the
 156            argument of monitor_exit used in the exception handler. Since the
 157            offset could be zero and give a wrong meaning of the flag it is
 158            offset by one.
 159         */
 160
 161         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 162                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 163         else
 164 #endif
 165                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 166
 167         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 168         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 169         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 170
 171         (void) dseg_addlinenumbertablesize(cd);
 172
 173         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 174
 175         /* create exception table */
 176
 177         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 178                 dseg_add_target(cd, ex->start);
 179                 dseg_add_target(cd, ex->end);
 180                 dseg_add_target(cd, ex->handler);
 181                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 182         }
 183
 184 #if defined(ENABLE_PROFILING)
 185         /* generate method profiling code */
 186
 187         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 188                 /* count frequency */
 189
 190                 M_MOV_IMM(code, REG_ITMP3);
 191                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 192
 193                 PROFILE_CYCLE_START;
 194         }
 195 #endif
 196
 197         /* create stack frame (if necessary) */
 198
 199         if (cd->stackframesize)
 200                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 201
 202         /* save used callee saved registers */
 203
 204         p = cd->stackframesize;
 205         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 206                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 207         }
 208         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 209                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 210         }
 211
 212         /* take arguments out of register or stack frame */
 213
 214         md = m->parseddesc;
 215
 216         for (p = 0, l = 0; p < md->paramcount; p++) {
 217                 t = md->paramtypes[p].type;
 218
 219                 varindex = jd->local_map[l * 5 + t];
 220
 221                 l++;
 222                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 223                         l++;
 224
 225                 if (varindex == UNUSED)
 226                         continue;
 227
 228                 var = VAR(varindex);
 229
 230                 s1 = md->params[p].regoff;
 231
 232                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 233                         s2 = rd->argintregs[s1];
 234                         if (!md->params[p].inmemory) {           /* register arguments    */
 235                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 236                                         M_INTMOVE(s2, var->vv.regoff);
 237
 238                                 } else {                             /* reg arg -> spilled    */
 239                                     M_LST(s2, REG_SP, var->vv.regoff * 8);
 240                                 }
 241
 242                         } else {                                 /* stack arguments       */
 243                                 if (!IS_INMEMORY(var->flags)) {      /* stack arg -> register */
 244                                         /* + 8 for return address */
 245                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 246
 247                                 } else {                             /* stack arg -> spilled  */
 248                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 249                                 }
 250                         }
 251
 252                 } else {                                     /* floating args         */
 253                         if (!md->params[p].inmemory) {           /* register arguments    */
 254                                 s2 = rd->argfltregs[s1];
 255                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 256                                         M_FLTMOVE(s2, var->vv.regoff);
 257
 258                                 } else {                                         /* reg arg -> spilled    */
 259                                         M_DST(s2, REG_SP, var->vv.regoff * 8);
 260                                 }
 261
 262                         } else {                                 /* stack arguments       */
 263                                 if (!IS_INMEMORY(var->flags)) {      /* stack-arg -> register */
 264                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 265
 266                                 } else {
 267                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 268                                 }
 269                         }
 270                 }
 271         }  /* end for */
 272
 273         /* save monitorenter argument */
 274
 275 #if defined(ENABLE_THREADS)
 276         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 277                 /* stack offset for monitor argument */
 278
 279                 s1 = rd->memuse;
 280
 281                 if (opt_verbosecall) {
 282                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 283
 284                         for (p = 0; p < INT_ARG_CNT; p++)
 285                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 286
 287                         for (p = 0; p < FLT_ARG_CNT; p++)
 288                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 289
 290                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 291                 }
 292
 293                 /* decide which monitor enter function to call */
 294
 295                 if (m->flags & ACC_STATIC) {
 296                         M_MOV_IMM(&m->class->object.header, REG_A0);
 297                 }
 298                 else {
 299                         M_TEST(REG_A0);
 300                         M_BEQ(0);
 301                         codegen_add_nullpointerexception_ref(cd);
 302                 }
 303
 304                 M_AST(REG_A0, REG_SP, s1 * 8);
 305                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 306                 M_CALL(REG_ITMP1);
 307
 308                 if (opt_verbosecall) {
 309                         for (p = 0; p < INT_ARG_CNT; p++)
 310                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 311
 312                         for (p = 0; p < FLT_ARG_CNT; p++)
 313                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 314
 315                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 316                 }
 317         }
 318 #endif
 319
 320 #if !defined(NDEBUG)
 321         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 322                 emit_verbosecall_enter(jd);
 323 #endif /* !defined(NDEBUG) */
 324
 325         }
 326
 327         /* end of header generation */
 328
 329         /* create replacement points */
 330
 331         REPLACEMENT_POINTS_INIT(cd, jd);
 332
 333         /* walk through all basic blocks */
 334
 335         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 336
 337                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 338
 339                 if (bptr->flags >= BBREACHED) {
 340
 341                 /* branch resolving */
 342
 343                 codegen_resolve_branchrefs(cd, bptr);
 344
 345                 /* handle replacement points */
 346
 347                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 348
 349                 /* copy interface registers to their destination */
 350
 351                 len = bptr->indepth;
 352                 MCODECHECK(512);
 353
 354 #if defined(ENABLE_PROFILING)
 355                 /* generate basicblock profiling code */
 356
 357                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 358                         /* count frequency */
 359
 360                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 361                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 362
 363                         /* if this is an exception handler, start profiling again */
 364
 365                         if (bptr->type == BBTYPE_EXH)
 366                                 PROFILE_CYCLE_START;
 367                 }
 368 #endif
 369
 370 #if defined(ENABLE_LSRA)
 371                 if (opt_lsra) {
 372                         while (len) {
 373                                 len--;
 374                                 src = bptr->invars[len];
 375                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 376                                         if (bptr->type == BBTYPE_EXH) {
 377 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 378                                                 if (!IS_INMEMORY(src->flags))
 379                                                         d= src->vv.regoff;
 380                                                 else
 381                                                         d=REG_ITMP1;
 382                                                 M_INTMOVE(REG_ITMP1, d);
 383                                                 emit_store(jd, NULL, src, d);
 384                                         }
 385                                 }
 386                         }
 387
 388                 } else {
 389 #endif
 390
 391                 while (len) {
 392                         len--;
 393                         var = VAR(bptr->invars[len]);
 394                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 395                                 if (bptr->type == BBTYPE_EXH) {
 396                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 397                                         M_INTMOVE(REG_ITMP1, d);
 398                                         emit_store(jd, NULL, var, d);
 399                                 }
 400                         }
 401                         else {
 402                                 assert((var->flags & INOUT));
 403                         }
 404                 }
 405 #if defined(ENABLE_LSRA)
 406                 }
 407 #endif
 408                 /* walk through all instructions */
 409
 410                 len = bptr->icount;
 411                 currentline = 0;
 412
 413                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 414                         if (iptr->line != currentline) {
 415                                 dseg_addlinenumber(cd, iptr->line);
 416                                 currentline = iptr->line;
 417                         }
 418
 419                         MCODECHECK(1024);                         /* 1KB should be enough */
 420
 421                 switch (iptr->opc) {
 422                 case ICMD_NOP:        /* ...  ==> ...                                 */
 423                 case ICMD_POP:        /* ..., value  ==> ...                          */
 424                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 425                         break;
 426
 427                 case ICMD_INLINE_START:
 428
 429                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 430                         break;
 431
 432                 case ICMD_INLINE_BODY:
 433
 434                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 435                         dseg_addlinenumber_inline_start(cd, iptr);
 436                         dseg_addlinenumber(cd, iptr->line);
 437                         break;
 438
 439                 case ICMD_INLINE_END:
 440
 441                         dseg_addlinenumber_inline_end(cd, iptr);
 442                         dseg_addlinenumber(cd, iptr->line);
 443                         break;
 444
 445                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 446
 447                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 448                         emit_nullpointer_check(cd, iptr, s1);
 449                         break;
 450
 451                 /* constant operations ************************************************/
 452
 453                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 454
 455                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 456                         ICONST(d, iptr->sx.val.i);
 457                         emit_store_dst(jd, iptr, d);
 458                         break;
 459
 460                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 461
 462                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 463                         LCONST(d, iptr->sx.val.l);
 464                         emit_store_dst(jd, iptr, d);
 465                         break;
 466
 467                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 468
 469                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 470                         disp = dseg_add_float(cd, iptr->sx.val.f);
 471                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 472                         emit_store_dst(jd, iptr, d);
 473                         break;
 474
 475                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 476
 477                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 478                         disp = dseg_add_double(cd, iptr->sx.val.d);
 479                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 480                         emit_store_dst(jd, iptr, d);
 481                         break;
 482
 483                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 484
 485                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 486
 487                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 488                                 cr = iptr->sx.val.c.ref;
 489
 490 /*                              PROFILE_CYCLE_STOP; */
 491
 492                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 493
 494 /*                              PROFILE_CYCLE_START; */
 495
 496                                 M_MOV_IMM(NULL, d);
 497
 498                         } else {
 499                                 if (iptr->sx.val.anyptr == 0)
 500                                         M_CLR(d);
 501                                 else
 502                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 503                         }
 504                         emit_store_dst(jd, iptr, d);
 505                         break;
 506
 507
 508                 /* load/store/copy/move operations ************************************/
 509
 510                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 511                 case ICMD_ALOAD:      /* s1 = local variable                          */
 512                 case ICMD_LLOAD:
 513                 case ICMD_FLOAD:
 514                 case ICMD_DLOAD:
 515                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 516                 case ICMD_LSTORE:
 517                 case ICMD_FSTORE:
 518                 case ICMD_DSTORE:
 519                 case ICMD_COPY:
 520                 case ICMD_MOVE:
 521
 522                         emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 523                         break;
 524
 525                 case ICMD_ASTORE:
 526                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 527                                 emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 528                         break;
 529
 530                 /* integer operations *************************************************/
 531
 532                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 533
 534                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 535                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 536                         M_INTMOVE(s1, d);
 537                         M_INEG(d);
 538                         emit_store_dst(jd, iptr, d);
 539                         break;
 540
 541                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 542
 543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 545                         M_INTMOVE(s1, d);
 546                         M_LNEG(d);
 547                         emit_store_dst(jd, iptr, d);
 548                         break;
 549
 550                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 551
 552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 553                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 554                         M_ISEXT(s1, d);
 555                         emit_store_dst(jd, iptr, d);
 556                         break;
 557
 558                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 559
 560                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 561                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 562                         M_IMOV(s1, d);
 563                         emit_store_dst(jd, iptr, d);
 564                         break;
 565
 566                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 567
 568                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 569                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 570                         M_BSEXT(s1, d);
 571                         emit_store_dst(jd, iptr, d);
 572                         break;
 573
 574                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 575
 576                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 577                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 578                         M_CZEXT(s1, d);
 579                         emit_store_dst(jd, iptr, d);
 580                         break;
 581
 582                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 583
 584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 585                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 586                         M_SSEXT(s1, d);
 587                         emit_store_dst(jd, iptr, d);
 588                         break;
 589
 590
 591                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 592
 593                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 594                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 595                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 596                         if (s2 == d)
 597                                 M_IADD(s1, d);
 598                         else {
 599                                 M_INTMOVE(s1, d);
 600                                 M_IADD(s2, d);
 601                         }
 602                         emit_store_dst(jd, iptr, d);
 603                         break;
 604
 605                 case ICMD_IINC:
 606                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 607                                       /* sx.val.i = constant                             */
 608
 609                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 610                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 611
 612                         /* Using inc and dec is not faster than add (tested with
 613                            sieve). */
 614
 615                         M_INTMOVE(s1, d);
 616                         M_IADD_IMM(iptr->sx.val.i, d);
 617                         emit_store_dst(jd, iptr, d);
 618                         break;
 619
 620                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 621
 622                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 623                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 624                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 625                         if (s2 == d)
 626                                 M_LADD(s1, d);
 627                         else {
 628                                 M_INTMOVE(s1, d);
 629                                 M_LADD(s2, d);
 630                         }
 631                         emit_store_dst(jd, iptr, d);
 632                         break;
 633
 634                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 635                                       /* sx.val.l = constant                             */
 636
 637                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 638                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 639                         M_INTMOVE(s1, d);
 640                         if (IS_IMM32(iptr->sx.val.l))
 641                                 M_LADD_IMM(iptr->sx.val.l, d);
 642                         else {
 643                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 644                                 M_LADD(REG_ITMP2, d);
 645                         }
 646                         emit_store_dst(jd, iptr, d);
 647                         break;
 648
 649                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 650
 651                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 652                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 653                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 654                         if (s2 == d) {
 655                                 M_INTMOVE(s1, REG_ITMP1);
 656                                 M_ISUB(s2, REG_ITMP1);
 657                                 M_INTMOVE(REG_ITMP1, d);
 658                         } else {
 659                                 M_INTMOVE(s1, d);
 660                                 M_ISUB(s2, d);
 661                         }
 662                         emit_store_dst(jd, iptr, d);
 663                         break;
 664
 665                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 666                                       /* sx.val.i = constant                             */
 667
 668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 669                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 670                         M_INTMOVE(s1, d);
 671                         M_ISUB_IMM(iptr->sx.val.i, d);
 672                         emit_store_dst(jd, iptr, d);
 673                         break;
 674
 675                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 676
 677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 678                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 679                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 680                         if (s2 == d) {
 681                                 M_INTMOVE(s1, REG_ITMP1);
 682                                 M_LSUB(s2, REG_ITMP1);
 683                                 M_INTMOVE(REG_ITMP1, d);
 684                         } else {
 685                                 M_INTMOVE(s1, d);
 686                                 M_LSUB(s2, d);
 687                         }
 688                         emit_store_dst(jd, iptr, d);
 689                         break;
 690
 691                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 692                                       /* sx.val.l = constant                             */
 693
 694                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 695                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 696                         M_INTMOVE(s1, d);
 697                         if (IS_IMM32(iptr->sx.val.l))
 698                                 M_LSUB_IMM(iptr->sx.val.l, d);
 699                         else {
 700                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 701                                 M_LSUB(REG_ITMP2, d);
 702                         }
 703                         emit_store_dst(jd, iptr, d);
 704                         break;
 705
 706                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 707
 708                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 709                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 710                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 711                         if (s2 == d)
 712                                 M_IMUL(s1, d);
 713                         else {
 714                                 M_INTMOVE(s1, d);
 715                                 M_IMUL(s2, d);
 716                         }
 717                         emit_store_dst(jd, iptr, d);
 718                         break;
 719
 720                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 721                                       /* sx.val.i = constant                             */
 722
 723                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 724                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 725                         if (iptr->sx.val.i == 2) {
 726                                 M_INTMOVE(s1, d);
 727                                 M_ISLL_IMM(1, d);
 728                         } else
 729                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 730                         emit_store_dst(jd, iptr, d);
 731                         break;
 732
 733                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 734
 735                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 736                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 737                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 738                         if (s2 == d)
 739                                 M_LMUL(s1, d);
 740                         else {
 741                                 M_INTMOVE(s1, d);
 742                                 M_LMUL(s2, d);
 743                         }
 744                         emit_store_dst(jd, iptr, d);
 745                         break;
 746
 747                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 748                                       /* sx.val.l = constant                             */
 749
 750                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 751                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 752                         if (IS_IMM32(iptr->sx.val.l))
 753                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 754                         else {
 755                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 756                                 M_INTMOVE(s1, d);
 757                                 M_LMUL(REG_ITMP2, d);
 758                         }
 759                         emit_store_dst(jd, iptr, d);
 760                         break;
 761
 762                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 763
 764                         s1 = emit_load_s1(jd, iptr, RAX);
 765                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 766                         d = codegen_reg_of_dst(jd, iptr, RAX);
 767
 768                         M_INTMOVE(s1, RAX);
 769                         M_INTMOVE(s2, REG_ITMP3);
 770                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 771
 772                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 773
 774                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 775                         M_BNE(4 + 6);
 776                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 777                         M_BEQ(1 + 3);                                          /* 6 bytes */
 778
 779                         emit_cltd(cd);                                         /* 1 byte  */
 780                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 781
 782                         M_INTMOVE(RAX, d);
 783                         emit_store_dst(jd, iptr, d);
 784                         dst = VAROP(iptr->dst);
 785                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 786                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 787                         break;
 788
 789                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 790
 791                         s1 = emit_load_s1(jd, iptr, RAX);
 792                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 793                         d = codegen_reg_of_dst(jd, iptr, RDX);
 794
 795                         M_INTMOVE(s1, RAX);
 796                         M_INTMOVE(s2, REG_ITMP3);
 797                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 798
 799                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 800
 801                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 802                         M_BNE(3 + 4 + 6);
 803                         M_CLR(RDX);                                            /* 3 bytes */
 804                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 805                         M_BEQ(1 + 3);                                          /* 6 bytes */
 806
 807                         emit_cltd(cd);                                         /* 1 byte  */
 808                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 809
 810                         M_INTMOVE(RDX, d);
 811                         emit_store_dst(jd, iptr, d);
 812                         dst = VAROP(iptr->dst);
 813                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 814                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 815                         break;
 816
 817                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 818                                       /* sx.val.i = constant                             */
 819
 820                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 821                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 822                         M_INTMOVE(s1, REG_ITMP1);
 823                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 824                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 825                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 826                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 827                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 828                         emit_store_dst(jd, iptr, d);
 829                         break;
 830
 831                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 832                                       /* sx.val.i = constant                             */
 833
 834                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 835                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 836                         M_INTMOVE(s1, REG_ITMP1);
 837                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 838                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 839                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 840                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 841                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 842                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 843                         emit_store_dst(jd, iptr, d);
 844                         break;
 845
 846
 847                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 848
 849                         s1 = emit_load_s1(jd, iptr, RAX);
 850                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 851                         d = codegen_reg_of_dst(jd, iptr, RAX);
 852
 853                         M_INTMOVE(s1, RAX);
 854                         M_INTMOVE(s2, REG_ITMP3);
 855                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 856
 857                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 858
 859                         /* check as described in jvm spec */
 860                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 861                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 862                         M_BNE(4 + 6);
 863                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 864                         M_BEQ(2 + 3);                                          /* 6 bytes */
 865
 866                         emit_cqto(cd);                                         /* 2 bytes */
 867                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 868
 869                         M_INTMOVE(RAX, d);
 870                         emit_store_dst(jd, iptr, d);
 871                         dst = VAROP(iptr->dst);
 872                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 873                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 874                         break;
 875
 876                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 877
 878                         s1 = emit_load_s1(jd, iptr, RAX);
 879                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 880                         d = codegen_reg_of_dst(jd, iptr, RDX);
 881
 882                         M_INTMOVE(s1, RAX);
 883                         M_INTMOVE(s2, REG_ITMP3);
 884                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 885
 886                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 887
 888                         /* check as described in jvm spec */
 889                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 890                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 891                         M_BNE(3 + 4 + 6);
 892                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 893                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 894                         M_BEQ(2 + 3);                                          /* 6 bytes */
 895
 896                         emit_cqto(cd);                                         /* 2 bytes */
 897                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 898
 899                         M_INTMOVE(RDX, d);
 900                         emit_store_dst(jd, iptr, d);
 901                         dst = VAROP(iptr->dst);
 902                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 903                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 904                         break;
 905
 906                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 907                                       /* sx.val.i = constant                             */
 908
 909                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 910                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 911                         M_INTMOVE(s1, REG_ITMP1);
 912                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 913                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 914                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 915                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 916                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 917                         emit_store_dst(jd, iptr, d);
 918                         break;
 919
 920                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 921                                       /* sx.val.l = constant                             */
 922
 923                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 924                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 925                         M_INTMOVE(s1, REG_ITMP1);
 926                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 927                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 928                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 929                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 930                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 931                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 932                         emit_store_dst(jd, iptr, d);
 933                         break;
 934
 935                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 936
 937                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 938                         emit_ishift(jd, SHIFT_SHL, iptr);
 939                         break;
 940
 941                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 942                                       /* sx.val.i = constant                             */
 943
 944                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 945                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 946                         M_INTMOVE(s1, d);
 947                         M_ISLL_IMM(iptr->sx.val.i, d);
 948                         emit_store_dst(jd, iptr, d);
 949                         break;
 950
 951                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 952
 953                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 954                         emit_ishift(jd, SHIFT_SAR, iptr);
 955                         break;
 956
 957                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 958                                       /* sx.val.i = constant                             */
 959
 960                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 961                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 962                         M_INTMOVE(s1, d);
 963                         M_ISRA_IMM(iptr->sx.val.i, d);
 964                         emit_store_dst(jd, iptr, d);
 965                         break;
 966
 967                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 968
 969                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 970                         emit_ishift(jd, SHIFT_SHR, iptr);
 971                         break;
 972
 973                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 974                                       /* sx.val.i = constant                             */
 975
 976                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 977                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 978                         M_INTMOVE(s1, d);
 979                         M_ISRL_IMM(iptr->sx.val.i, d);
 980                         emit_store_dst(jd, iptr, d);
 981                         break;
 982
 983                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 984
 985                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 986                         emit_lshift(jd, SHIFT_SHL, iptr);
 987                         break;
 988
 989         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 990                                           /* sx.val.i = constant                             */
 991
 992                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 993                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 994                         M_INTMOVE(s1, d);
 995                         M_LSLL_IMM(iptr->sx.val.i, d);
 996                         emit_store_dst(jd, iptr, d);
 997                         break;
 998
 999                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1000
1001                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1002                         emit_lshift(jd, SHIFT_SAR, iptr);
1003                         break;
1004
1005                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1006                                       /* sx.val.i = constant                             */
1007
1008                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1009                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1010                         M_INTMOVE(s1, d);
1011                         M_LSRA_IMM(iptr->sx.val.i, d);
1012                         emit_store_dst(jd, iptr, d);
1013                         break;
1014
1015                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1016
1017                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1018                         emit_lshift(jd, SHIFT_SHR, iptr);
1019                         break;
1020
1021                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1022                                       /* sx.val.l = constant                             */
1023
1024                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1025                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1026                         M_INTMOVE(s1, d);
1027                         M_LSRL_IMM(iptr->sx.val.i, d);
1028                         emit_store_dst(jd, iptr, d);
1029                         break;
1030
1031                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1032
1033                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1034                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1035                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1036                         if (s2 == d)
1037                                 M_IAND(s1, d);
1038                         else {
1039                                 M_INTMOVE(s1, d);
1040                                 M_IAND(s2, d);
1041                         }
1042                         emit_store_dst(jd, iptr, d);
1043                         break;
1044
1045                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1046                                       /* sx.val.i = constant                             */
1047
1048                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1049                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1050                         M_INTMOVE(s1, d);
1051                         M_IAND_IMM(iptr->sx.val.i, d);
1052                         emit_store_dst(jd, iptr, d);
1053                         break;
1054
1055                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1056
1057                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1058                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1059                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1060                         if (s2 == d)
1061                                 M_LAND(s1, d);
1062                         else {
1063                                 M_INTMOVE(s1, d);
1064                                 M_LAND(s2, d);
1065                         }
1066                         emit_store_dst(jd, iptr, d);
1067                         break;
1068
1069                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1070                                       /* sx.val.l = constant                             */
1071
1072                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1073                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1074                         M_INTMOVE(s1, d);
1075                         if (IS_IMM32(iptr->sx.val.l))
1076                                 M_LAND_IMM(iptr->sx.val.l, d);
1077                         else {
1078                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1079                                 M_LAND(REG_ITMP2, d);
1080                         }
1081                         emit_store_dst(jd, iptr, d);
1082                         break;
1083
1084                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1085
1086                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1087                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1088                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1089                         if (s2 == d)
1090                                 M_IOR(s1, d);
1091                         else {
1092                                 M_INTMOVE(s1, d);
1093                                 M_IOR(s2, d);
1094                         }
1095                         emit_store_dst(jd, iptr, d);
1096                         break;
1097
1098                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1099                                       /* sx.val.i = constant                             */
1100
1101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1102                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1103                         M_INTMOVE(s1, d);
1104                         M_IOR_IMM(iptr->sx.val.i, d);
1105                         emit_store_dst(jd, iptr, d);
1106                         break;
1107
1108                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1109
1110                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1111                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1112                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1113                         if (s2 == d)
1114                                 M_LOR(s1, d);
1115                         else {
1116                                 M_INTMOVE(s1, d);
1117                                 M_LOR(s2, d);
1118                         }
1119                         emit_store_dst(jd, iptr, d);
1120                         break;
1121
1122                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1123                                       /* sx.val.l = constant                             */
1124
1125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1126                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1127                         M_INTMOVE(s1, d);
1128                         if (IS_IMM32(iptr->sx.val.l))
1129                                 M_LOR_IMM(iptr->sx.val.l, d);
1130                         else {
1131                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1132                                 M_LOR(REG_ITMP2, d);
1133                         }
1134                         emit_store_dst(jd, iptr, d);
1135                         break;
1136
1137                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1138
1139                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1140                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1141                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1142                         if (s2 == d)
1143                                 M_IXOR(s1, d);
1144                         else {
1145                                 M_INTMOVE(s1, d);
1146                                 M_IXOR(s2, d);
1147                         }
1148                         emit_store_dst(jd, iptr, d);
1149                         break;
1150
1151                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1152                                       /* sx.val.i = constant                             */
1153
1154                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1155                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1156                         M_INTMOVE(s1, d);
1157                         M_IXOR_IMM(iptr->sx.val.i, d);
1158                         emit_store_dst(jd, iptr, d);
1159                         break;
1160
1161                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1162
1163                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1164                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1165                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1166                         if (s2 == d)
1167                                 M_LXOR(s1, d);
1168                         else {
1169                                 M_INTMOVE(s1, d);
1170                                 M_LXOR(s2, d);
1171                         }
1172                         emit_store_dst(jd, iptr, d);
1173                         break;
1174
1175                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1176                                       /* sx.val.l = constant                             */
1177
1178                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1179                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1180                         M_INTMOVE(s1, d);
1181                         if (IS_IMM32(iptr->sx.val.l))
1182                                 M_LXOR_IMM(iptr->sx.val.l, d);
1183                         else {
1184                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1185                                 M_LXOR(REG_ITMP2, d);
1186                         }
1187                         emit_store_dst(jd, iptr, d);
1188                         break;
1189
1190
1191                 /* floating operations ************************************************/
1192
1193                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1194
1195                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1196                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1197                         disp = dseg_add_s4(cd, 0x80000000);
1198                         M_FLTMOVE(s1, d);
1199                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1200                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1201                         emit_store_dst(jd, iptr, d);
1202                         break;
1203
1204                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1205
1206                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1207                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1208                         disp = dseg_add_s8(cd, 0x8000000000000000);
1209                         M_FLTMOVE(s1, d);
1210                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1211                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1212                         emit_store_dst(jd, iptr, d);
1213                         break;
1214
1215                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1216
1217                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1218                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1219                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1220                         if (s2 == d)
1221                                 M_FADD(s1, d);
1222                         else {
1223                                 M_FLTMOVE(s1, d);
1224                                 M_FADD(s2, d);
1225                         }
1226                         emit_store_dst(jd, iptr, d);
1227                         break;
1228
1229                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1230
1231                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1232                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1233                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1234                         if (s2 == d)
1235                                 M_DADD(s1, d);
1236                         else {
1237                                 M_FLTMOVE(s1, d);
1238                                 M_DADD(s2, d);
1239                         }
1240                         emit_store_dst(jd, iptr, d);
1241                         break;
1242
1243                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1244
1245                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1246                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1247                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1248                         if (s2 == d) {
1249                                 M_FLTMOVE(s2, REG_FTMP2);
1250                                 s2 = REG_FTMP2;
1251                         }
1252                         M_FLTMOVE(s1, d);
1253                         M_FSUB(s2, d);
1254                         emit_store_dst(jd, iptr, d);
1255                         break;
1256
1257                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1258
1259                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1260                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1261                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1262                         if (s2 == d) {
1263                                 M_FLTMOVE(s2, REG_FTMP2);
1264                                 s2 = REG_FTMP2;
1265                         }
1266                         M_FLTMOVE(s1, d);
1267                         M_DSUB(s2, d);
1268                         emit_store_dst(jd, iptr, d);
1269                         break;
1270
1271                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1272
1273                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1274                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1275                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1276                         if (s2 == d)
1277                                 M_FMUL(s1, d);
1278                         else {
1279                                 M_FLTMOVE(s1, d);
1280                                 M_FMUL(s2, d);
1281                         }
1282                         emit_store_dst(jd, iptr, d);
1283                         break;
1284
1285                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1286
1287                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1288                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1289                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1290                         if (s2 == d)
1291                                 M_DMUL(s1, d);
1292                         else {
1293                                 M_FLTMOVE(s1, d);
1294                                 M_DMUL(s2, d);
1295                         }
1296                         emit_store_dst(jd, iptr, d);
1297                         break;
1298
1299                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1300
1301                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1302                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1303                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1304                         if (s2 == d) {
1305                                 M_FLTMOVE(s2, REG_FTMP2);
1306                                 s2 = REG_FTMP2;
1307                         }
1308                         M_FLTMOVE(s1, d);
1309                         M_FDIV(s2, d);
1310                         emit_store_dst(jd, iptr, d);
1311                         break;
1312
1313                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1314
1315                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1316                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1317                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1318                         if (s2 == d) {
1319                                 M_FLTMOVE(s2, REG_FTMP2);
1320                                 s2 = REG_FTMP2;
1321                         }
1322                         M_FLTMOVE(s1, d);
1323                         M_DDIV(s2, d);
1324                         emit_store_dst(jd, iptr, d);
1325                         break;
1326
1327                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1328
1329                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1330                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1331                         M_CVTIF(s1, d);
1332                         emit_store_dst(jd, iptr, d);
1333                         break;
1334
1335                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1336
1337                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1338                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1339                         M_CVTID(s1, d);
1340                         emit_store_dst(jd, iptr, d);
1341                         break;
1342
1343                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1344
1345                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1346                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1347                         M_CVTLF(s1, d);
1348                         emit_store_dst(jd, iptr, d);
1349                         break;
1350
1351                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1352
1353                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1354                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1355                         M_CVTLD(s1, d);
1356                         emit_store_dst(jd, iptr, d);
1357                         break;
1358
1359                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1360
1361                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1362                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1363                         M_CVTFI(s1, d);
1364                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1365                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1366                                 ((REG_RESULT == d) ? 0 : 3);
1367                         M_BNE(disp);
1368                         M_FLTMOVE(s1, REG_FTMP1);
1369                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1370                         M_CALL(REG_ITMP2);
1371                         M_INTMOVE(REG_RESULT, d);
1372                         emit_store_dst(jd, iptr, d);
1373                         break;
1374
1375                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1376
1377                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1378                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1379                         M_CVTDI(s1, d);
1380                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1381                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1382                                 ((REG_RESULT == d) ? 0 : 3);
1383                         M_BNE(disp);
1384                         M_FLTMOVE(s1, REG_FTMP1);
1385                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1386                         M_CALL(REG_ITMP2);
1387                         M_INTMOVE(REG_RESULT, d);
1388                         emit_store_dst(jd, iptr, d);
1389                         break;
1390
1391                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1392
1393                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1394                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1395                         M_CVTFL(s1, d);
1396                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1397                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1398                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1399                                 ((REG_RESULT == d) ? 0 : 3);
1400                         M_BNE(disp);
1401                         M_FLTMOVE(s1, REG_FTMP1);
1402                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1403                         M_CALL(REG_ITMP2);
1404                         M_INTMOVE(REG_RESULT, d);
1405                         emit_store_dst(jd, iptr, d);
1406                         break;
1407
1408                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1409
1410                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1411                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1412                         M_CVTDL(s1, d);
1413                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1414                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1415                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1416                                 ((REG_RESULT == d) ? 0 : 3);
1417                         M_BNE(disp);
1418                         M_FLTMOVE(s1, REG_FTMP1);
1419                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1420                         M_CALL(REG_ITMP2);
1421                         M_INTMOVE(REG_RESULT, d);
1422                         emit_store_dst(jd, iptr, d);
1423                         break;
1424
1425                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1426
1427                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1428                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1429                         M_CVTFD(s1, d);
1430                         emit_store_dst(jd, iptr, d);
1431                         break;
1432
1433                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1434
1435                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1436                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1437                         M_CVTDF(s1, d);
1438                         emit_store_dst(jd, iptr, d);
1439                         break;
1440
1441                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1442                                           /* == => 0, < => 1, > => -1 */
1443
1444                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1445                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1446                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1447                         M_CLR(d);
1448                         M_MOV_IMM(1, REG_ITMP1);
1449                         M_MOV_IMM(-1, REG_ITMP2);
1450                         emit_ucomiss_reg_reg(cd, s1, s2);
1451                         M_CMOVB(REG_ITMP1, d);
1452                         M_CMOVA(REG_ITMP2, d);
1453                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1454                         emit_store_dst(jd, iptr, d);
1455                         break;
1456
1457                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1458                                           /* == => 0, < => 1, > => -1 */
1459
1460                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1461                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1462                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1463                         M_CLR(d);
1464                         M_MOV_IMM(1, REG_ITMP1);
1465                         M_MOV_IMM(-1, REG_ITMP2);
1466                         emit_ucomiss_reg_reg(cd, s1, s2);
1467                         M_CMOVB(REG_ITMP1, d);
1468                         M_CMOVA(REG_ITMP2, d);
1469                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1470                         emit_store_dst(jd, iptr, d);
1471                         break;
1472
1473                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1474                                           /* == => 0, < => 1, > => -1 */
1475
1476                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1477                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1478                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1479                         M_CLR(d);
1480                         M_MOV_IMM(1, REG_ITMP1);
1481                         M_MOV_IMM(-1, REG_ITMP2);
1482                         emit_ucomisd_reg_reg(cd, s1, s2);
1483                         M_CMOVB(REG_ITMP1, d);
1484                         M_CMOVA(REG_ITMP2, d);
1485                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1486                         emit_store_dst(jd, iptr, d);
1487                         break;
1488
1489                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1490                                           /* == => 0, < => 1, > => -1 */
1491
1492                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1493                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1494                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1495                         M_CLR(d);
1496                         M_MOV_IMM(1, REG_ITMP1);
1497                         M_MOV_IMM(-1, REG_ITMP2);
1498                         emit_ucomisd_reg_reg(cd, s1, s2);
1499                         M_CMOVB(REG_ITMP1, d);
1500                         M_CMOVA(REG_ITMP2, d);
1501                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1502                         emit_store_dst(jd, iptr, d);
1503                         break;
1504
1505
1506                 /* memory operations **************************************************/
1507
1508                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1509
1510                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1511                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1512                         emit_nullpointer_check(cd, iptr, s1);
1513                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1514                         emit_store_dst(jd, iptr, d);
1515                         break;
1516
1517                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1518
1519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1520                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1522                         emit_array_checks(cd, iptr, s1, s2);
1523                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1524                         emit_store_dst(jd, iptr, d);
1525                         break;
1526
1527                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1528
1529                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1530                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1531                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1532                         emit_array_checks(cd, iptr, s1, s2);
1533                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1534                         emit_store_dst(jd, iptr, d);
1535                         break;
1536
1537                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1538
1539                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1540                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1541                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1542                         emit_array_checks(cd, iptr, s1, s2);
1543                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1544                         emit_store_dst(jd, iptr, d);
1545                         break;
1546
1547                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1548
1549                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1550                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1551                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1552                         emit_array_checks(cd, iptr, s1, s2);
1553                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1554                         emit_store_dst(jd, iptr, d);
1555                         break;
1556
1557                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1558
1559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1560                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1561                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1562                         emit_array_checks(cd, iptr, s1, s2);
1563                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1564                         emit_store_dst(jd, iptr, d);
1565                         break;
1566
1567                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1568
1569                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1570                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1571                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1572                         emit_array_checks(cd, iptr, s1, s2);
1573                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1574                         emit_store_dst(jd, iptr, d);
1575                         break;
1576
1577                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1578
1579                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1580                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1581                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1582                         emit_array_checks(cd, iptr, s1, s2);
1583                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1584                         emit_store_dst(jd, iptr, d);
1585                         break;
1586
1587                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1588
1589                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1590                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1591                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1592                         emit_array_checks(cd, iptr, s1, s2);
1593                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1594                         emit_store_dst(jd, iptr, d);
1595                         break;
1596
1597
1598                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1599
1600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1601                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1602                         emit_array_checks(cd, iptr, s1, s2);
1603                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1604                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1605                         break;
1606
1607                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1608
1609                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1610                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1611                         emit_array_checks(cd, iptr, s1, s2);
1612                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1613                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1614                         break;
1615
1616                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1617
1618                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1619                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1620                         emit_array_checks(cd, iptr, s1, s2);
1621                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1622                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1623                         break;
1624
1625                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1626
1627                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1628                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1629                         emit_array_checks(cd, iptr, s1, s2);
1630                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1631                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1632                         break;
1633
1634                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1635
1636                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1637                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1638                         emit_array_checks(cd, iptr, s1, s2);
1639                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1640                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1641                         break;
1642
1643                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1644
1645                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1646                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1647                         emit_array_checks(cd, iptr, s1, s2);
1648                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1649                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1650                         break;
1651
1652                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1653
1654                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1655                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1656                         emit_array_checks(cd, iptr, s1, s2);
1657                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1658                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1659                         break;
1660
1661                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1662
1663                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1664                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1665                         emit_array_checks(cd, iptr, s1, s2);
1666                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1667
1668                         M_MOV(s1, REG_A0);
1669                         M_MOV(s3, REG_A1);
1670                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1671                         M_CALL(REG_ITMP1);
1672                         M_TEST(REG_RESULT);
1673                         M_BEQ(0);
1674                         codegen_add_arraystoreexception_ref(cd);
1675
1676                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1677                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1678                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1679                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1680                         break;
1681
1682
1683                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1684
1685                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1686                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1687                         emit_array_checks(cd, iptr, s1, s2);
1688                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1689                         break;
1690
1691                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1692
1693                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1694                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1695                         emit_array_checks(cd, iptr, s1, s2);
1696                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1697                         break;
1698
1699                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1700
1701                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1702                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1703                         emit_array_checks(cd, iptr, s1, s2);
1704                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1705                         break;
1706
1707                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1708
1709                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1710                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1711                         emit_array_checks(cd, iptr, s1, s2);
1712                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1713                         break;
1714
1715                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1716
1717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1718                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1719                         emit_array_checks(cd, iptr, s1, s2);
1720
1721                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1722                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1723                         } else {
1724                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1725                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1726                         }
1727                         break;
1728
1729                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1730
1731                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1732                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1733                         emit_array_checks(cd, iptr, s1, s2);
1734                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1735                         break;
1736
1737
1738                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1739
1740                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1741                                 uf        = iptr->sx.s23.s3.uf;
1742                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1743                                 disp      = dseg_add_unique_address(cd, NULL);
1744                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1745
1746                                 /* must be calculated before codegen_add_patch_ref */
1747
1748                                 if (opt_shownops)
1749                                         disp -= PATCHER_CALL_SIZE;
1750
1751 /*                              PROFILE_CYCLE_STOP; */
1752
1753                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1754
1755 /*                              PROFILE_CYCLE_START; */
1756                         }
1757                         else {
1758                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1759                                 fieldtype = fi->type;
1760                                 disp      = dseg_add_address(cd, &(fi->value));
1761                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1762
1763                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1764                                         PROFILE_CYCLE_STOP;
1765
1766                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1767
1768                                         if (opt_shownops)
1769                                                 disp -= PATCHER_CALL_SIZE;
1770
1771                                         PROFILE_CYCLE_START;
1772                                 }
1773                         }
1774
1775                         /* This approach is much faster than moving the field
1776                            address inline into a register. */
1777
1778                         M_ALD(REG_ITMP1, RIP, disp);
1779
1780                         switch (fieldtype) {
1781                         case TYPE_INT:
1782                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1783                                 M_ILD(d, REG_ITMP1, 0);
1784                                 break;
1785                         case TYPE_LNG:
1786                         case TYPE_ADR:
1787                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1788                                 M_LLD(d, REG_ITMP1, 0);
1789                                 break;
1790                         case TYPE_FLT:
1791                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1792                                 M_FLD(d, REG_ITMP1, 0);
1793                                 break;
1794                         case TYPE_DBL:
1795                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1796                                 M_DLD(d, REG_ITMP1, 0);
1797                                 break;
1798                         }
1799                         emit_store_dst(jd, iptr, d);
1800                         break;
1801
1802                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1803
1804                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1805                                 uf        = iptr->sx.s23.s3.uf;
1806                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1807                                 disp      = dseg_add_unique_address(cd, NULL);
1808                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1809
1810                                 /* must be calculated before codegen_add_patch_ref */
1811
1812                                 if (opt_shownops)
1813                                         disp -= PATCHER_CALL_SIZE;
1814
1815 /*                              PROFILE_CYCLE_STOP; */
1816
1817                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1818
1819 /*                              PROFILE_CYCLE_START; */
1820                         }
1821                         else {
1822                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1823                                 fieldtype = fi->type;
1824                                 disp      = dseg_add_address(cd, &(fi->value));
1825                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1826
1827                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1828                                         PROFILE_CYCLE_STOP;
1829
1830                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1831
1832                                         if (opt_shownops)
1833                                                 disp -= PATCHER_CALL_SIZE;
1834
1835                                         PROFILE_CYCLE_START;
1836                                 }
1837                         }
1838
1839                         /* This approach is much faster than moving the field
1840                            address inline into a register. */
1841
1842                         M_ALD(REG_ITMP1, RIP, disp);
1843
1844                         switch (fieldtype) {
1845                         case TYPE_INT:
1846                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1847                                 M_IST(s1, REG_ITMP1, 0);
1848                                 break;
1849                         case TYPE_LNG:
1850                         case TYPE_ADR:
1851                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1852                                 M_LST(s1, REG_ITMP1, 0);
1853                                 break;
1854                         case TYPE_FLT:
1855                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1856                                 M_FST(s1, REG_ITMP1, 0);
1857                                 break;
1858                         case TYPE_DBL:
1859                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1860                                 M_DST(s1, REG_ITMP1, 0);
1861                                 break;
1862                         }
1863                         break;
1864
1865                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1866                                           /* val = value (in current instruction)     */
1867                                           /* following NOP)                           */
1868
1869                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1870                                 uf        = iptr->sx.s23.s3.uf;
1871                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1872                                 disp      = dseg_add_unique_address(cd, NULL);
1873                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1874
1875                                 /* must be calculated before codegen_add_patch_ref */
1876
1877                                 if (opt_shownops)
1878                                         disp -= PATCHER_CALL_SIZE;
1879
1880
1881 /*                              PROFILE_CYCLE_STOP; */
1882
1883                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1884
1885 /*                              PROFILE_CYCLE_START; */
1886                         }
1887                         else {
1888                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1889                                 fieldtype = fi->type;
1890                                 disp      = dseg_add_address(cd, &(fi->value));
1891                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1892
1893                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1894                                         PROFILE_CYCLE_STOP;
1895
1896                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1897
1898                                         if (opt_shownops)
1899                                                 disp -= PATCHER_CALL_SIZE;
1900
1901                                         PROFILE_CYCLE_START;
1902                                 }
1903                         }
1904
1905                         /* This approach is much faster than moving the field
1906                            address inline into a register. */
1907
1908                         M_ALD(REG_ITMP1, RIP, disp);
1909
1910                         switch (fieldtype) {
1911                         case TYPE_INT:
1912                         case TYPE_FLT:
1913                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1914                                 break;
1915                         case TYPE_LNG:
1916                         case TYPE_ADR:
1917                         case TYPE_DBL:
1918                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1919                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1920                                 else {
1921                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1922                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1923                                 }
1924                                 break;
1925                         }
1926                         break;
1927
1928                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1929
1930                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1931                         emit_nullpointer_check(cd, iptr, s1);
1932
1933                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1934                                 uf        = iptr->sx.s23.s3.uf;
1935                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1936                                 disp      = 0;
1937
1938 /*                              PROFILE_CYCLE_STOP; */
1939
1940                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1941
1942 /*                              PROFILE_CYCLE_START; */
1943                         }
1944                         else {
1945                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1946                                 fieldtype = fi->type;
1947                                 disp      = fi->offset;
1948                         }
1949
1950                         switch (fieldtype) {
1951                         case TYPE_INT:
1952                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1953                                 M_ILD32(d, s1, disp);
1954                                 break;
1955                         case TYPE_LNG:
1956                         case TYPE_ADR:
1957                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1958                                 M_LLD32(d, s1, disp);
1959                                 break;
1960                         case TYPE_FLT:
1961                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1962                                 M_FLD32(d, s1, disp);
1963                                 break;
1964                         case TYPE_DBL:
1965                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1966                                 M_DLD32(d, s1, disp);
1967                                 break;
1968                         }
1969                         emit_store_dst(jd, iptr, d);
1970                         break;
1971
1972                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1973
1974                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1975                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1976                         emit_nullpointer_check(cd, iptr, s1);
1977
1978                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1979                                 uf        = iptr->sx.s23.s3.uf;
1980                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1981                                 disp      = 0;
1982
1983 /*                              PROFILE_CYCLE_STOP; */
1984
1985                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1986
1987 /*                              PROFILE_CYCLE_START; */
1988                         }
1989                         else {
1990                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1991                                 fieldtype = fi->type;
1992                                 disp      = fi->offset;
1993                         }
1994
1995                         switch (fieldtype) {
1996                         case TYPE_INT:
1997                                 M_IST32(s2, s1, disp);
1998                                 break;
1999                         case TYPE_LNG:
2000                         case TYPE_ADR:
2001                                 M_LST32(s2, s1, disp);
2002                                 break;
2003                         case TYPE_FLT:
2004                                 M_FST32(s2, s1, disp);
2005                                 break;
2006                         case TYPE_DBL:
2007                                 M_DST32(s2, s1, disp);
2008                                 break;
2009                         }
2010                         break;
2011
2012                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2013                                           /* val = value (in current instruction)     */
2014                                           /* following NOP)                           */
2015
2016                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2017                         emit_nullpointer_check(cd, iptr, s1);
2018
2019                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2020                                 uf        = iptr->sx.s23.s3.uf;
2021                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2022                                 disp      = 0;
2023
2024 /*                              PROFILE_CYCLE_STOP; */
2025
2026                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2027
2028 /*                              PROFILE_CYCLE_START; */
2029                         }
2030                         else {
2031                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2032                                 fieldtype = fi->type;
2033                                 disp      = fi->offset;
2034                         }
2035
2036                         switch (fieldtype) {
2037                         case TYPE_INT:
2038                         case TYPE_FLT:
2039                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2040                                 break;
2041                         case TYPE_LNG:
2042                         case TYPE_ADR:
2043                         case TYPE_DBL:
2044                                 /* XXX why no check for IS_IMM32? */
2045                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2046                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2047                                 break;
2048                         }
2049                         break;
2050
2051
2052                 /* branch operations **************************************************/
2053
2054                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2055
2056                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2057                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2058
2059                         PROFILE_CYCLE_STOP;
2060
2061 #ifdef ENABLE_VERIFIER
2062                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2063                                 uc = iptr->sx.s23.s2.uc;
2064
2065                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2066                         }
2067 #endif /* ENABLE_VERIFIER */
2068
2069                         M_CALL_IMM(0);                            /* passing exception pc */
2070                         M_POP(REG_ITMP2_XPC);
2071
2072                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2073                         M_JMP(REG_ITMP3);
2074                         break;
2075
2076                 case ICMD_GOTO:         /* ... ==> ...                                */
2077                 case ICMD_RET:          /* ... ==> ...                                */
2078
2079                         M_JMP_IMM(0);
2080                         codegen_add_branch_ref(cd, iptr->dst.block);
2081                         break;
2082
2083                 case ICMD_JSR:          /* ... ==> ...                                */
2084
2085                         M_JMP_IMM(0);
2086                         codegen_add_branch_ref(cd, iptr->sx.s23.s3.jsrtarget.block);
2087                         break;
2088
2089                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2090
2091                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2092                         M_TEST(s1);
2093                         M_BEQ(0);
2094                         codegen_add_branch_ref(cd, iptr->dst.block);
2095                         break;
2096
2097                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2098
2099                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2100                         M_TEST(s1);
2101                         M_BNE(0);
2102                         codegen_add_branch_ref(cd, iptr->dst.block);
2103                         break;
2104
2105                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2106
2107                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2108                         M_ICMP_IMM(iptr->sx.val.i, s1);
2109                         M_BEQ(0);
2110                         codegen_add_branch_ref(cd, iptr->dst.block);
2111                         break;
2112
2113                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2114
2115                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2116                         M_ICMP_IMM(iptr->sx.val.i, s1);
2117                         M_BLT(0);
2118                         codegen_add_branch_ref(cd, iptr->dst.block);
2119                         break;
2120
2121                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2122
2123                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2124                         M_ICMP_IMM(iptr->sx.val.i, s1);
2125                         M_BLE(0);
2126                         codegen_add_branch_ref(cd, iptr->dst.block);
2127                         break;
2128
2129                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2130
2131                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2132                         M_ICMP_IMM(iptr->sx.val.i, s1);
2133                         M_BNE(0);
2134                         codegen_add_branch_ref(cd, iptr->dst.block);
2135                         break;
2136
2137                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2138
2139                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2140                         M_ICMP_IMM(iptr->sx.val.i, s1);
2141                         M_BGT(0);
2142                         codegen_add_branch_ref(cd, iptr->dst.block);
2143                         break;
2144
2145                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2146
2147                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2148                         M_ICMP_IMM(iptr->sx.val.i, s1);
2149                         M_BGE(0);
2150                         codegen_add_branch_ref(cd, iptr->dst.block);
2151                         break;
2152
2153                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2154
2155                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2156                         if (IS_IMM32(iptr->sx.val.l))
2157                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2158                         else {
2159                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2160                                 M_LCMP(REG_ITMP2, s1);
2161                         }
2162                         M_BEQ(0);
2163                         codegen_add_branch_ref(cd, iptr->dst.block);
2164                         break;
2165
2166                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2167
2168                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2169                         if (IS_IMM32(iptr->sx.val.l))
2170                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2171                         else {
2172                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2173                                 M_LCMP(REG_ITMP2, s1);
2174                         }
2175                         M_BLT(0);
2176                         codegen_add_branch_ref(cd, iptr->dst.block);
2177                         break;
2178
2179                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2180
2181                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2182                         if (IS_IMM32(iptr->sx.val.l))
2183                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2184                         else {
2185                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2186                                 M_LCMP(REG_ITMP2, s1);
2187                         }
2188                         M_BLE(0);
2189                         codegen_add_branch_ref(cd, iptr->dst.block);
2190                         break;
2191
2192                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2193
2194                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2195                         if (IS_IMM32(iptr->sx.val.l))
2196                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2197                         else {
2198                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2199                                 M_LCMP(REG_ITMP2, s1);
2200                         }
2201                         M_BNE(0);
2202                         codegen_add_branch_ref(cd, iptr->dst.block);
2203                         break;
2204
2205                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2206
2207                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2208                         if (IS_IMM32(iptr->sx.val.l))
2209                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2210                         else {
2211                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2212                                 M_LCMP(REG_ITMP2, s1);
2213                         }
2214                         M_BGT(0);
2215                         codegen_add_branch_ref(cd, iptr->dst.block);
2216                         break;
2217
2218                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2219
2220                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2221                         if (IS_IMM32(iptr->sx.val.l))
2222                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2223                         else {
2224                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2225                                 M_LCMP(REG_ITMP2, s1);
2226                         }
2227                         M_BGE(0);
2228                         codegen_add_branch_ref(cd, iptr->dst.block);
2229                         break;
2230
2231                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2232
2233                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2234                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2235                         M_ICMP(s2, s1);
2236                         M_BEQ(0);
2237                         codegen_add_branch_ref(cd, iptr->dst.block);
2238                         break;
2239
2240                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2241                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2242
2243                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2244                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2245                         M_LCMP(s2, s1);
2246                         M_BEQ(0);
2247                         codegen_add_branch_ref(cd, iptr->dst.block);
2248                         break;
2249
2250                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2251
2252                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2253                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2254                         M_ICMP(s2, s1);
2255                         M_BNE(0);
2256                         codegen_add_branch_ref(cd, iptr->dst.block);
2257                         break;
2258
2259                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2260                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2261
2262                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2263                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2264                         M_LCMP(s2, s1);
2265                         M_BNE(0);
2266                         codegen_add_branch_ref(cd, iptr->dst.block);
2267                         break;
2268
2269                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2270
2271                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2272                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2273                         M_ICMP(s2, s1);
2274                         M_BLT(0);
2275                         codegen_add_branch_ref(cd, iptr->dst.block);
2276                         break;
2277
2278                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2279
2280                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2281                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2282                         M_LCMP(s2, s1);
2283                         M_BLT(0);
2284                         codegen_add_branch_ref(cd, iptr->dst.block);
2285                         break;
2286
2287                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2288
2289                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2290                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2291                         M_ICMP(s2, s1);
2292                         M_BGT(0);
2293                         codegen_add_branch_ref(cd, iptr->dst.block);
2294                         break;
2295
2296                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2297
2298                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2299                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2300                         M_LCMP(s2, s1);
2301                         M_BGT(0);
2302                         codegen_add_branch_ref(cd, iptr->dst.block);
2303                         break;
2304
2305                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2306
2307                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2308                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2309                         M_ICMP(s2, s1);
2310                         M_BLE(0);
2311                         codegen_add_branch_ref(cd, iptr->dst.block);
2312                         break;
2313
2314                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2315
2316                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2317                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2318                         M_LCMP(s2, s1);
2319                         M_BLE(0);
2320                         codegen_add_branch_ref(cd, iptr->dst.block);
2321                         break;
2322
2323                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2324
2325                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2326                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2327                         M_ICMP(s2, s1);
2328                         M_BGE(0);
2329                         codegen_add_branch_ref(cd, iptr->dst.block);
2330                         break;
2331
2332                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2333
2334                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2335                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2336                         M_LCMP(s2, s1);
2337                         M_BGE(0);
2338                         codegen_add_branch_ref(cd, iptr->dst.block);
2339                         break;
2340
2341                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2342                 case ICMD_LRETURN:
2343
2344                         REPLACEMENT_POINT_RETURN(cd, iptr);
2345                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2346                         M_INTMOVE(s1, REG_RESULT);
2347                         goto nowperformreturn;
2348
2349                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2350
2351                         REPLACEMENT_POINT_RETURN(cd, iptr);
2352                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2353                         M_INTMOVE(s1, REG_RESULT);
2354
2355 #ifdef ENABLE_VERIFIER
2356                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2357                                 uc = iptr->sx.s23.s2.uc;
2358
2359                                 PROFILE_CYCLE_STOP;
2360
2361                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2362
2363                                 PROFILE_CYCLE_START;
2364                         }
2365 #endif /* ENABLE_VERIFIER */
2366                         goto nowperformreturn;
2367
2368                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2369                 case ICMD_DRETURN:
2370
2371                         REPLACEMENT_POINT_RETURN(cd, iptr);
2372                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2373                         M_FLTMOVE(s1, REG_FRESULT);
2374                         goto nowperformreturn;
2375
2376                 case ICMD_RETURN:      /* ...  ==> ...                                */
2377
2378                         REPLACEMENT_POINT_RETURN(cd, iptr);
2379
2380 nowperformreturn:
2381                         {
2382                         s4 i, p;
2383
2384                         p = cd->stackframesize;
2385
2386 #if !defined(NDEBUG)
2387                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2388                                 emit_verbosecall_exit(jd);
2389 #endif /* !defined(NDEBUG) */
2390
2391 #if defined(ENABLE_THREADS)
2392                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2393                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2394
2395                                 /* we need to save the proper return value */
2396                                 switch (iptr->opc) {
2397                                 case ICMD_IRETURN:
2398                                 case ICMD_ARETURN:
2399                                 case ICMD_LRETURN:
2400                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2401                                         break;
2402                                 case ICMD_FRETURN:
2403                                 case ICMD_DRETURN:
2404                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2405                                         break;
2406                                 }
2407
2408                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2409                                 M_CALL(REG_ITMP1);
2410
2411                                 /* and now restore the proper return value */
2412                                 switch (iptr->opc) {
2413                                 case ICMD_IRETURN:
2414                                 case ICMD_ARETURN:
2415                                 case ICMD_LRETURN:
2416                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2417                                         break;
2418                                 case ICMD_FRETURN:
2419                                 case ICMD_DRETURN:
2420                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2421                                         break;
2422                                 }
2423                         }
2424 #endif
2425
2426                         /* restore saved registers */
2427
2428                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2429                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2430                         }
2431                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2432                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2433                         }
2434
2435                         /* deallocate stack */
2436
2437                         if (cd->stackframesize)
2438                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2439
2440                         /* generate method profiling code */
2441
2442                         PROFILE_CYCLE_STOP;
2443
2444                         M_RET;
2445                         }
2446                         break;
2447
2448
2449                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2450                         {
2451                                 s4 i, l;
2452                                 branch_target_t *table;
2453
2454                                 table = iptr->dst.table;
2455
2456                                 l = iptr->sx.s23.s2.tablelow;
2457                                 i = iptr->sx.s23.s3.tablehigh;
2458
2459                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2460                                 M_INTMOVE(s1, REG_ITMP1);
2461
2462                                 if (l != 0)
2463                                         M_ISUB_IMM(l, REG_ITMP1);
2464
2465                                 /* number of targets */
2466                                 i = i - l + 1;
2467
2468                 /* range check */
2469                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2470                                 M_BA(0);
2471
2472                                 codegen_add_branch_ref(cd, table[0].block); /* default target */
2473
2474                                 /* build jump table top down and use address of lowest entry */
2475
2476                                 table += i;
2477
2478                                 while (--i >= 0) {
2479                                         dseg_add_target(cd, table->block);
2480                                         --table;
2481                                 }
2482
2483                                 /* length of dataseg after last dseg_add_target is used
2484                                    by load */
2485
2486                                 M_MOV_IMM(0, REG_ITMP2);
2487                                 dseg_adddata(cd);
2488                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2489                                 M_JMP(REG_ITMP1);
2490                         }
2491                         break;
2492
2493
2494                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2495                         {
2496                                 s4 i;
2497                                 lookup_target_t *lookup;
2498
2499                                 lookup = iptr->dst.lookup;
2500
2501                                 i = iptr->sx.s23.s2.lookupcount;
2502
2503                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2504                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2505
2506                                 while (--i >= 0) {
2507                                         M_ICMP_IMM(lookup->value, s1);
2508                                         M_BEQ(0);
2509                                         codegen_add_branch_ref(cd, lookup->target.block);
2510                                         lookup++;
2511                                 }
2512
2513                                 M_JMP_IMM(0);
2514
2515                                 codegen_add_branch_ref(cd, iptr->sx.s23.s3.lookupdefault.block);
2516                         }
2517                         break;
2518
2519
2520                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2521
2522                         bte = iptr->sx.s23.s3.bte;
2523                         md  = bte->md;
2524                         goto gen_method;
2525
2526                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2527
2528                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2529                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2530                 case ICMD_INVOKEINTERFACE:
2531
2532                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2533
2534                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2535                                 lm = NULL;
2536                                 um = iptr->sx.s23.s3.um;
2537                                 md = um->methodref->parseddesc.md;
2538                         }
2539                         else {
2540                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2541                                 um = NULL;
2542                                 md = lm->parseddesc;
2543                         }
2544
2545 gen_method:
2546                         s3 = md->paramcount;
2547
2548                         MCODECHECK((20 * s3) + 128);
2549
2550                         /* copy arguments to registers or stack location */
2551
2552                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2553                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2554
2555                                 /* Already Preallocated (ARGVAR) ? */
2556                                 if (var->flags & PREALLOC)
2557                                         continue;
2558
2559                                 if (IS_INT_LNG_TYPE(var->type)) {
2560                                         if (!md->params[s3].inmemory) {
2561                                                 s1 = rd->argintregs[md->params[s3].regoff];
2562                                                 d = emit_load(jd, iptr, var, s1);
2563                                                 M_INTMOVE(d, s1);
2564                                         }
2565                                         else {
2566                                                 d = emit_load(jd, iptr, var, REG_ITMP1);
2567                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2568                                         }
2569                                 }
2570                                 else {
2571                                         if (!md->params[s3].inmemory) {
2572                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2573                                                 d = emit_load(jd, iptr, var, s1);
2574                                                 M_FLTMOVE(d, s1);
2575                                         }
2576                                         else {
2577                                                 d = emit_load(jd, iptr, var, REG_FTMP1);
2578
2579                                                 if (IS_2_WORD_TYPE(var->type))
2580                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2581                                                 else
2582                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2583                                         }
2584                                 }
2585                         }
2586
2587                         /* generate method profiling code */
2588
2589                         PROFILE_CYCLE_STOP;
2590
2591                         switch (iptr->opc) {
2592                         case ICMD_BUILTIN:
2593                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2594                                 M_CALL(REG_ITMP1);
2595
2596                                 if (INSTRUCTION_MUST_CHECK(iptr)) {
2597                                         M_TEST(REG_RESULT);
2598                                         M_BEQ(0);
2599                                         codegen_add_fillinstacktrace_ref(cd);
2600                                 }
2601                                 break;
2602
2603                         case ICMD_INVOKESPECIAL:
2604                                 M_TEST(REG_A0);
2605                                 M_BEQ(0);
2606                                 codegen_add_nullpointerexception_ref(cd);
2607
2608                                 /* fall through */
2609
2610                         case ICMD_INVOKESTATIC:
2611                                 if (lm == NULL) {
2612                                         disp = dseg_add_unique_address(cd, NULL);
2613                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2614
2615                                         /* must be calculated before codegen_add_patch_ref */
2616
2617                                         if (opt_shownops)
2618                                                 disp -= PATCHER_CALL_SIZE;
2619
2620                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2621                                                                                   um, disp);
2622
2623 /*                                      a = 0; */
2624                                 }
2625                                 else {
2626                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2627                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2628
2629 /*                                      a = (ptrint) lm->stubroutine; */
2630                                 }
2631
2632 /*                              M_MOV_IMM(a, REG_ITMP2); */
2633                                 M_ALD(REG_ITMP2, RIP, disp);
2634                                 M_CALL(REG_ITMP2);
2635                                 break;
2636
2637                         case ICMD_INVOKEVIRTUAL:
2638                                 emit_nullpointer_check(cd, iptr, REG_A0);
2639
2640                                 if (lm == NULL) {
2641                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2642
2643                                         s1 = 0;
2644                                 }
2645                                 else
2646                                         s1 = OFFSET(vftbl_t, table[0]) +
2647                                                 sizeof(methodptr) * lm->vftblindex;
2648
2649                                 M_ALD(REG_METHODPTR, REG_A0,
2650                                           OFFSET(java_objectheader, vftbl));
2651                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2652                                 M_CALL(REG_ITMP3);
2653                                 break;
2654
2655                         case ICMD_INVOKEINTERFACE:
2656                                 emit_nullpointer_check(cd, iptr, REG_A0);
2657
2658                                 if (lm == NULL) {
2659                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2660
2661                                         s1 = 0;
2662                                         s2 = 0;
2663                                 }
2664                                 else {
2665                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2666                                                 sizeof(methodptr) * lm->class->index;
2667
2668                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2669                                 }
2670
2671                                 M_ALD(REG_METHODPTR, REG_A0,
2672                                           OFFSET(java_objectheader, vftbl));
2673                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2674                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2675                                 M_CALL(REG_ITMP3);
2676                                 break;
2677                         }
2678
2679                         /* generate method profiling code */
2680
2681                         PROFILE_CYCLE_START;
2682
2683                         /* store size of call code in replacement point */
2684
2685                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2686
2687                         /* store return value */
2688
2689                         d = md->returntype.type;
2690
2691                         if (d != TYPE_VOID) {
2692                                 if (IS_INT_LNG_TYPE(d)) {
2693                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2694                                         M_INTMOVE(REG_RESULT, s1);
2695                                 }
2696                                 else {
2697                                         s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2698                                         M_FLTMOVE(REG_FRESULT, s1);
2699                                 }
2700                                 emit_store_dst(jd, iptr, s1);
2701                         }
2702                         break;
2703
2704
2705                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2706
2707                                       /* val.a: (classinfo *) superclass              */
2708
2709                         /*  superclass is an interface:
2710                          *
2711                          *  OK if ((sub == NULL) ||
2712                          *         (sub->vftbl->interfacetablelength > super->index) &&
2713                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
2714                          *
2715                          *  superclass is a class:
2716                          *
2717                          *  OK if ((sub == NULL) || (0
2718                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
2719                          *         super->vftbl->diffval));
2720                          */
2721
2722                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2723                                 /* object type cast-check */
2724
2725                                 classinfo *super;
2726                                 vftbl_t   *supervftbl;
2727                                 s4         superindex;
2728
2729                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2730                                         super = NULL;
2731                                         superindex = 0;
2732                                         supervftbl = NULL;
2733                                 }
2734                                 else {
2735                                         super = iptr->sx.s23.s3.c.cls;
2736                                         superindex = super->index;
2737                                         supervftbl = super->vftbl;
2738                                 }
2739
2740 #if defined(ENABLE_THREADS)
2741                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2742 #endif
2743                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2744
2745                                 /* calculate interface checkcast code size */
2746
2747                                 s2 = 3; /* mov_membase_reg */
2748                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
2749
2750                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
2751                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
2752                                         3 /* test */ + 6 /* jcc */;
2753
2754                                 if (super == NULL)
2755                                         s2 += (opt_shownops ? 5 : 0);
2756
2757                                 /* calculate class checkcast code size */
2758
2759                                 s3 = 3; /* mov_membase_reg */
2760                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
2761                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
2762
2763 #if 0
2764                                 if (s1 != REG_ITMP1) {
2765                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2766                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
2767                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2768                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
2769                                         a += 3;    /* sub */
2770
2771                                 } else
2772 #endif
2773                                         {
2774                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
2775                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
2776                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2777                                         }
2778
2779                                 s3 += 3 /* cmp */ + 6 /* jcc */;
2780
2781                                 if (super == NULL)
2782                                         s3 += (opt_shownops ? 5 : 0);
2783
2784                                 /* if class is not resolved, check which code to call */
2785
2786                                 if (super == NULL) {
2787                                         M_TEST(s1);
2788                                         M_BEQ(6 + (opt_shownops ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
2789
2790                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2791                                                                                   iptr->sx.s23.s3.c.ref, 0);
2792
2793                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2794                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2795                                         M_BEQ(s2 + 5);
2796                                 }
2797
2798                                 /* interface checkcast code */
2799
2800                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2801                                         if (super != NULL) {
2802                                                 M_TEST(s1);
2803                                                 M_BEQ(s2);
2804                                         }
2805
2806                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2807
2808                                         if (super == NULL) {
2809                                                 codegen_add_patch_ref(cd,
2810                                                                                           PATCHER_checkcast_instanceof_interface,
2811                                                                                           iptr->sx.s23.s3.c.ref,
2812                                                                                           0);
2813                                         }
2814
2815                                         emit_movl_membase32_reg(cd, REG_ITMP2,
2816                                                                                           OFFSET(vftbl_t, interfacetablelength),
2817                                                                                           REG_ITMP3);
2818                                         /* XXX TWISTI: should this be int arithmetic? */
2819                                         M_LSUB_IMM32(superindex, REG_ITMP3);
2820                                         M_TEST(REG_ITMP3);
2821                                         M_BLE(0);
2822                                         codegen_add_classcastexception_ref(cd, s1);
2823                                         emit_mov_membase32_reg(cd, REG_ITMP2,
2824                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
2825                                                                                          superindex * sizeof(methodptr*),
2826                                                                                          REG_ITMP3);
2827                                         M_TEST(REG_ITMP3);
2828                                         M_BEQ(0);
2829                                         codegen_add_classcastexception_ref(cd, s1);
2830
2831                                         if (super == NULL)
2832                                                 M_JMP_IMM(s3);
2833                                 }
2834
2835                                 /* class checkcast code */
2836
2837                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2838                                         if (super != NULL) {
2839                                                 M_TEST(s1);
2840                                                 M_BEQ(s3);
2841                                         }
2842
2843                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2844
2845                                         if (super == NULL) {
2846                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2847                                                                                           iptr->sx.s23.s3.c.ref,
2848                                                                                           0);
2849                                         }
2850
2851                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2852 #if defined(ENABLE_THREADS)
2853                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2854 #endif
2855                                         emit_movl_membase32_reg(cd, REG_ITMP2,
2856                                                                                           OFFSET(vftbl_t, baseval),
2857                                                                                           REG_ITMP2);
2858                                         /*                                      if (s1 != REG_ITMP1) { */
2859                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2860                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2861                                         /*                                                                                              REG_ITMP1); */
2862                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2863                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2864                                         /*                                                                                              REG_ITMP3); */
2865                                         /*  #if defined(ENABLE_THREADS) */
2866                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2867                                         /*  #endif */
2868                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2869
2870                                         /*                                      } else { */
2871                                         emit_movl_membase32_reg(cd, REG_ITMP3,
2872                                                                                           OFFSET(vftbl_t, baseval),
2873                                                                                           REG_ITMP3);
2874                                         M_LSUB(REG_ITMP3, REG_ITMP2);
2875                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2876                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2877                                         /*                                      } */
2878 #if defined(ENABLE_THREADS)
2879                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2880 #endif
2881                                         M_LCMP(REG_ITMP3, REG_ITMP2);
2882                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
2883                                         codegen_add_classcastexception_ref(cd, s1);
2884                                 }
2885
2886                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2887                         }
2888                         else {
2889                                 /* array type cast-check */
2890
2891                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2892                                 M_INTMOVE(s1, REG_A0);
2893
2894                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2895                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2896                                                                                   iptr->sx.s23.s3.c.ref, 0);
2897                                 }
2898
2899                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2900                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2901                                 M_CALL(REG_ITMP1);
2902
2903                                 /* s1 may have been destroyed over the function call */
2904                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2905                                 M_TEST(REG_RESULT);
2906                                 M_BEQ(0);
2907                                 codegen_add_classcastexception_ref(cd, s1);
2908
2909                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2910                         }
2911
2912                         M_INTMOVE(s1, d);
2913                         emit_store_dst(jd, iptr, d);
2914                         break;
2915
2916                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2917
2918                                       /* val.a: (classinfo *) superclass              */
2919
2920                         /*  superclass is an interface:
2921                          *
2922                          *  return (sub != NULL) &&
2923                          *         (sub->vftbl->interfacetablelength > super->index) &&
2924                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
2925                          *
2926                          *  superclass is a class:
2927                          *
2928                          *  return ((sub != NULL) && (0
2929                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
2930                          *          super->vftbl->diffvall));
2931                          */
2932
2933                         {
2934                         classinfo *super;
2935                         vftbl_t   *supervftbl;
2936                         s4         superindex;
2937
2938                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2939                                 super = NULL;
2940                                 superindex = 0;
2941                                 supervftbl = NULL;
2942
2943                         } else {
2944                                 super = iptr->sx.s23.s3.c.cls;
2945                                 superindex = super->index;
2946                                 supervftbl = super->vftbl;
2947                         }
2948
2949 #if defined(ENABLE_THREADS)
2950             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2951 #endif
2952
2953                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2954                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2955                         if (s1 == d) {
2956                                 M_INTMOVE(s1, REG_ITMP1);
2957                                 s1 = REG_ITMP1;
2958                         }
2959
2960                         /* calculate interface instanceof code size */
2961
2962                         s2 = 3; /* mov_membase_reg */
2963                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
2964                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
2965                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
2966                                 3 /* test */ + 4 /* setcc */;
2967
2968                         if (!super)
2969                                 s2 += (opt_shownops ? 5 : 0);
2970
2971                         /* calculate class instanceof code size */
2972
2973                         s3 = 3; /* mov_membase_reg */
2974                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
2975                         s3 += 10; /* mov_imm_reg */
2976                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
2977                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
2978                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
2979                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
2980                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
2981                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2982                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
2983
2984                         if (super == NULL)
2985                                 s3 += (opt_shownops ? 5 : 0);
2986
2987                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
2988
2989                         /* if class is not resolved, check which code to call */
2990
2991                         if (super == NULL) {
2992                                 emit_test_reg_reg(cd, s1, s1);
2993                                 emit_jcc(cd, CC_Z, (6 + (opt_shownops ? 5 : 0) +
2994                                                                                          7 + 6 + s2 + 5 + s3));
2995
2996                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2997                                                                           iptr->sx.s23.s3.c.ref, 0);
2998
2999                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3000                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3001                                 emit_jcc(cd, CC_Z, s2 + 5);
3002                         }
3003
3004                         /* interface instanceof code */
3005
3006                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3007                                 if (super != NULL) {
3008                                         emit_test_reg_reg(cd, s1, s1);
3009                                         emit_jcc(cd, CC_Z, s2);
3010                                 }
3011
3012                                 emit_mov_membase_reg(cd, s1,
3013                                                                          OFFSET(java_objectheader, vftbl),
3014                                                                          REG_ITMP1);
3015
3016                                 if (super == NULL) {
3017                                         codegen_add_patch_ref(cd,
3018                                                                                   PATCHER_checkcast_instanceof_interface,
3019                                                                                   iptr->sx.s23.s3.c.ref, 0);
3020                                 }
3021
3022                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3023                                                                                   OFFSET(vftbl_t, interfacetablelength),
3024                                                                                   REG_ITMP3);
3025                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3026                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3027
3028                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3029
3030                                 emit_jcc(cd, CC_LE, a);
3031                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3032                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3033                                                                                  superindex * sizeof(methodptr*),
3034                                                                                  REG_ITMP1);
3035                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3036                                 emit_setcc_reg(cd, CC_NE, d);
3037
3038                                 if (!super)
3039                                         emit_jmp_imm(cd, s3);
3040                         }
3041
3042                         /* class instanceof code */
3043
3044                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3045                                 if (super != NULL) {
3046                                         emit_test_reg_reg(cd, s1, s1);
3047                                         emit_jcc(cd, CC_E, s3);
3048                                 }
3049
3050                                 emit_mov_membase_reg(cd, s1,
3051                                                                            OFFSET(java_objectheader, vftbl),
3052                                                                            REG_ITMP1);
3053
3054                                 if (super == NULL) {
3055                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
3056                                                                                   iptr->sx.s23.s3.c.ref, 0);
3057                                 }
3058
3059                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3060 #if defined(ENABLE_THREADS)
3061                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3062 #endif
3063                                 emit_movl_membase_reg(cd, REG_ITMP1,
3064                                                                                 OFFSET(vftbl_t, baseval),
3065                                                                                 REG_ITMP1);
3066                                 emit_movl_membase_reg(cd, REG_ITMP2,
3067                                                                                 OFFSET(vftbl_t, diffval),
3068                                                                                 REG_ITMP3);
3069                                 emit_movl_membase_reg(cd, REG_ITMP2,
3070                                                                                 OFFSET(vftbl_t, baseval),
3071                                                                                 REG_ITMP2);
3072 #if defined(ENABLE_THREADS)
3073                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3074 #endif
3075                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3076                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3077                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3078                                 emit_setcc_reg(cd, CC_BE, d);
3079                         }
3080                         emit_store_dst(jd, iptr, d);
3081                         }
3082                         break;
3083
3084                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3085
3086                         /* check for negative sizes and copy sizes to stack if necessary  */
3087
3088                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
3089
3090                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3091
3092                                 /* copy SAVEDVAR sizes to stack */
3093                                 var = VAR(iptr->sx.s23.s2.args[s1]);
3094
3095                                 /* Already Preallocated? */
3096                                 if (!(var->flags & PREALLOC)) {
3097                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
3098                                         M_LST(s2, REG_SP, s1 * 8);
3099                                 }
3100                         }
3101
3102                         /* is a patcher function set? */
3103
3104                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3105                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
3106                                                                           iptr->sx.s23.s3.c.ref, 0);
3107                         }
3108
3109                         /* a0 = dimension count */
3110
3111                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
3112
3113                         /* a1 = classinfo */
3114
3115                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3116
3117                         /* a2 = pointer to dimensions = stack pointer */
3118
3119                         M_MOV(REG_SP, REG_A2);
3120
3121                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3122                         M_CALL(REG_ITMP1);
3123
3124                         /* check for exception before result assignment */
3125
3126                         M_TEST(REG_RESULT);
3127                         M_BEQ(0);
3128                         codegen_add_fillinstacktrace_ref(cd);
3129
3130                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3131                         M_INTMOVE(REG_RESULT, s1);
3132                         emit_store_dst(jd, iptr, s1);
3133                         break;
3134
3135                 default:
3136                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
3137                                                                                    iptr->opc);
3138                         return false;
3139         } /* switch */
3140
3141         } /* for instruction */
3142
3143         MCODECHECK(512); /* XXX require a lower number? */
3144
3145         /* At the end of a basic block we may have to append some nops,
3146            because the patcher stub calling code might be longer than the
3147            actual instruction. So codepatching does not change the
3148            following block unintentionally. */
3149
3150         if (cd->mcodeptr < cd->lastmcodeptr) {
3151                 while (cd->mcodeptr < cd->lastmcodeptr) {
3152                         M_NOP;
3153                 }
3154         }
3155
3156         } /* if (bptr -> flags >= BBREACHED) */
3157         } /* for basic block */
3158
3159         dseg_createlinenumbertable(cd);
3160
3161         /* generate stubs */
3162
3163         emit_exception_stubs(jd);
3164         emit_patcher_stubs(jd);
3165         REPLACEMENT_EMIT_STUBS(jd);
3166
3167         codegen_finish(jd);
3168
3169         /* everything's ok */
3170
3171         return true;
3172 }
3173
3174
3175 /* createcompilerstub **********************************************************
3176
3177    Creates a stub routine which calls the compiler.
3178
3179 *******************************************************************************/
3180
3181 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3182 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3183
3184 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3185
3186
3187 u1 *createcompilerstub(methodinfo *m)
3188 {
3189         u1          *s;                     /* memory to hold the stub            */
3190         ptrint      *d;
3191         codegendata *cd;
3192         s4           dumpsize;
3193
3194         s = CNEW(u1, COMPILERSTUB_SIZE);
3195
3196         /* set data pointer and code pointer */
3197
3198         d = (ptrint *) s;
3199         s = s + COMPILERSTUB_DATASIZE;
3200
3201         /* mark start of dump memory area */
3202
3203         dumpsize = dump_size();
3204
3205         cd = DNEW(codegendata);
3206         cd->mcodeptr = s;
3207
3208         /* The codeinfo pointer is actually a pointer to the
3209            methodinfo. This fakes a codeinfo structure. */
3210
3211         d[0] = (ptrint) asm_call_jit_compiler;
3212         d[1] = (ptrint) m;
3213         d[2] = (ptrint) &d[1];                                    /* fake code->m */
3214
3215         /* code for the stub */
3216
3217         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3218         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3219         M_JMP(REG_ITMP3);
3220
3221 #if defined(ENABLE_STATISTICS)
3222         if (opt_stat)
3223                 count_cstub_len += COMPILERSTUB_SIZE;
3224 #endif
3225
3226         /* release dump area */
3227
3228         dump_release(dumpsize);
3229
3230         return s;
3231 }
3232
3233
3234 /* createnativestub ************************************************************
3235
3236    Creates a stub routine which calls a native method.
3237
3238 *******************************************************************************/
3239
3240 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3241 {
3242         methodinfo   *m;
3243         codeinfo     *code;
3244         codegendata  *cd;
3245         registerdata *rd;
3246         methoddesc   *md;
3247         s4            nativeparams;
3248         s4            i, j;                 /* count variables                    */
3249         s4            t;
3250         s4            s1, s2;
3251
3252         /* get required compiler data */
3253
3254         m    = jd->m;
3255         code = jd->code;
3256         cd   = jd->cd;
3257         rd   = jd->rd;
3258
3259         /* initialize variables */
3260
3261         md = m->parseddesc;
3262         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3263
3264         /* calculate stack frame size */
3265
3266         cd->stackframesize =
3267                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3268                 sizeof(localref_table) / SIZEOF_VOID_P +
3269                 INT_ARG_CNT + FLT_ARG_CNT +
3270                 1 +                       /* functionptr, TODO: store in data segment */
3271                 nmd->memuse;
3272
3273         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3274
3275         /* create method header */
3276
3277         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3278         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3279         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3280         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3281         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3282         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3283         (void) dseg_addlinenumbertablesize(cd);
3284         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3285
3286 #if defined(ENABLE_PROFILING)
3287         /* generate native method profiling code */
3288
3289         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3290                 /* count frequency */
3291
3292                 M_MOV_IMM(code, REG_ITMP3);
3293                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3294         }
3295 #endif
3296
3297         /* generate stub code */
3298
3299         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3300
3301 #if !defined(NDEBUG)
3302         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3303                 emit_verbosecall_enter(jd);
3304 #endif
3305
3306         /* get function address (this must happen before the stackframeinfo) */
3307
3308 #if !defined(WITH_STATIC_CLASSPATH)
3309         if (f == NULL)
3310                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3311 #endif
3312
3313         M_MOV_IMM(f, REG_ITMP3);
3314
3315
3316         /* save integer and float argument registers */
3317
3318         for (i = 0, j = 0; i < md->paramcount; i++) {
3319                 if (!md->params[i].inmemory) {
3320                         s1 = md->params[i].regoff;
3321
3322                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3323                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3324                         else
3325                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3326
3327                         j++;
3328                 }
3329         }
3330
3331         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3332
3333         /* create dynamic stack info */
3334
3335         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3336         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3337         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3338         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3339         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3340         M_CALL(REG_ITMP1);
3341
3342         /* restore integer and float argument registers */
3343
3344         for (i = 0, j = 0; i < md->paramcount; i++) {
3345                 if (!md->params[i].inmemory) {
3346                         s1 = md->params[i].regoff;
3347
3348                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3349                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3350                         else
3351                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3352
3353                         j++;
3354                 }
3355         }
3356
3357         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3358
3359
3360         /* copy or spill arguments to new locations */
3361
3362         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3363                 t = md->paramtypes[i].type;
3364
3365                 if (IS_INT_LNG_TYPE(t)) {
3366                         if (!md->params[i].inmemory) {
3367                                 s1 = rd->argintregs[md->params[i].regoff];
3368
3369                                 if (!nmd->params[j].inmemory) {
3370                                         s2 = rd->argintregs[nmd->params[j].regoff];
3371                                         M_INTMOVE(s1, s2);
3372
3373                                 } else {
3374                                         s2 = nmd->params[j].regoff;
3375                                         M_LST(s1, REG_SP, s2 * 8);
3376                                 }
3377
3378                         } else {
3379                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3380                                 s2 = nmd->params[j].regoff;
3381                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3382                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3383                         }
3384
3385                 } else {
3386                         /* We only copy spilled float arguments, as the float argument    */
3387                         /* registers keep unchanged.                                      */
3388
3389                         if (md->params[i].inmemory) {
3390                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3391                                 s2 = nmd->params[j].regoff;
3392
3393                                 if (IS_2_WORD_TYPE(t)) {
3394                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3395                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3396                                 } else {
3397                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3398                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3399                                 }
3400                         }
3401                 }
3402         }
3403
3404         /* put class into second argument register */
3405
3406         if (m->flags & ACC_STATIC)
3407                 M_MOV_IMM(m->class, REG_A1);
3408
3409         /* put env into first argument register */
3410
3411         M_MOV_IMM(_Jv_env, REG_A0);
3412
3413         /* do the native function call */
3414
3415         M_CALL(REG_ITMP3);
3416
3417         /* save return value */
3418
3419         if (md->returntype.type != TYPE_VOID) {
3420                 if (IS_INT_LNG_TYPE(md->returntype.type))
3421                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3422                 else
3423                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3424         }
3425
3426 #if !defined(NDEBUG)
3427         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3428                 emit_verbosecall_exit(jd);
3429 #endif
3430
3431         /* remove native stackframe info */
3432
3433         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3434         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3435         M_CALL(REG_ITMP1);
3436         M_MOV(REG_RESULT, REG_ITMP3);
3437
3438         /* restore return value */
3439
3440         if (md->returntype.type != TYPE_VOID) {
3441                 if (IS_INT_LNG_TYPE(md->returntype.type))
3442                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3443                 else
3444                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3445         }
3446
3447         /* remove stackframe */
3448
3449         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3450
3451         /* test for exception */
3452
3453         M_TEST(REG_ITMP3);
3454         M_BNE(1);
3455         M_RET;
3456
3457         /* handle exception */
3458
3459         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3460         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3461         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3462
3463         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3464         M_JMP(REG_ITMP3);
3465
3466         /* generate patcher stubs */
3467
3468         emit_patcher_stubs(jd);
3469
3470         codegen_finish(jd);
3471
3472         return code->entrypoint;
3473 }
3474
3475
3476 /*
3477  * These are local overrides for various environment variables in Emacs.
3478  * Please do not remove this and leave it at the end of the file, where
3479  * Emacs will automagically detect them.
3480  * ---------------------------------------------------------------------
3481  * Local variables:
3482  * mode: c
3483  * indent-tabs-mode: t
3484  * c-basic-offset: 4
3485  * tab-width: 4
3486  * End:
3487  * vim:noexpandtab:sw=4:ts=4:
3488  */