src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29             Christian Ullrich
  30             Edwin Steiner
  31
  32    $Id: codegen.c 6148 2006-12-07 23:57:45Z edwin $
  33
  34 */
  35
  36
  37 #include "config.h"
  38
  39 #include <assert.h>
  40 #include <stdio.h>
  41
  42 #include "vm/types.h"
  43
  44 #include "md-abi.h"
  45
  46 #include "vm/jit/x86_64/arch.h"
  47 #include "vm/jit/x86_64/codegen.h"
  48 #include "vm/jit/x86_64/emit.h"
  49
  50 #include "mm/memory.h"
  51 #include "native/jni.h"
  52 #include "native/native.h"
  53
  54 #if defined(ENABLE_THREADS)
  55 # include "threads/native/lock.h"
  56 #endif
  57
  58 #include "vm/builtin.h"
  59 #include "vm/exceptions.h"
  60 #include "vm/global.h"
  61 #include "vm/loader.h"
  62 #include "vm/options.h"
  63 #include "vm/statistics.h"
  64 #include "vm/stringlocal.h"
  65 #include "vm/vm.h"
  66 #include "vm/jit/asmpart.h"
  67 #include "vm/jit/codegen-common.h"
  68 #include "vm/jit/dseg.h"
  69 #include "vm/jit/emit-common.h"
  70 #include "vm/jit/jit.h"
  71 #include "vm/jit/methodheader.h"
  72 #include "vm/jit/parse.h"
  73 #include "vm/jit/patcher.h"
  74 #include "vm/jit/reg.h"
  75 #include "vm/jit/replace.h"
  76
  77 #if defined(ENABLE_LSRA)
  78 # include "vm/jit/allocator/lsra.h"
  79 #endif
  80
  81
  82 /* codegen *********************************************************************
  83
  84    Generates machine code.
  85
  86 *******************************************************************************/
  87
  88 bool codegen(jitdata *jd)
  89 {
  90         methodinfo         *m;
  91         codeinfo           *code;
  92         codegendata        *cd;
  93         registerdata       *rd;
  94         s4                  len, s1, s2, s3, d, disp;
  95         u2                  currentline;
  96         ptrint              a;
  97         varinfo            *var, *dst;
  98         basicblock         *bptr;
  99         instruction        *iptr;
 100         exception_entry    *ex;
 101         constant_classref  *cr;
 102         unresolved_class   *uc;
 103         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 104         unresolved_method  *um;
 105         builtintable_entry *bte;
 106         methoddesc         *md;
 107         fieldinfo          *fi;
 108         unresolved_field   *uf;
 109         s4                  fieldtype;
 110         s4                 varindex;
 111
 112         /* get required compiler data */
 113
 114         m    = jd->m;
 115         code = jd->code;
 116         cd   = jd->cd;
 117         rd   = jd->rd;
 118
 119         /* prevent compiler warnings */
 120
 121         d   = 0;
 122         lm  = NULL;
 123         um  = NULL;
 124         bte = NULL;
 125
 126         {
 127         s4 i, p, t, l;
 128         s4 savedregs_num;
 129
 130         savedregs_num = 0;
 131
 132         /* space to save used callee saved registers */
 133
 134         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 135         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 136
 137         cd->stackframesize = rd->memuse + savedregs_num;
 138
 139 #if defined(ENABLE_THREADS)
 140         /* space to save argument of monitor_enter */
 141
 142         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 143                 cd->stackframesize++;
 144 #endif
 145
 146         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 147            native code e.g. libc or jni (alignment problems with
 148            movaps). */
 149
 150         if (!jd->isleafmethod || opt_verbosecall)
 151                 cd->stackframesize |= 0x1;
 152
 153         /* create method header */
 154
 155         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 156         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 157
 158 #if defined(ENABLE_THREADS)
 159         /* IsSync contains the offset relative to the stack pointer for the
 160            argument of monitor_exit used in the exception handler. Since the
 161            offset could be zero and give a wrong meaning of the flag it is
 162            offset by one.
 163         */
 164
 165         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 166                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 167         else
 168 #endif
 169                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 170
 171         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 172         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 173         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 174
 175         (void) dseg_addlinenumbertablesize(cd);
 176
 177         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 178
 179         /* create exception table */
 180
 181         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 182                 dseg_add_target(cd, ex->start);
 183                 dseg_add_target(cd, ex->end);
 184                 dseg_add_target(cd, ex->handler);
 185                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 186         }
 187
 188         /* generate method profiling code */
 189
 190         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 191                 /* count frequency */
 192
 193                 M_MOV_IMM(code, REG_ITMP3);
 194                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 195
 196                 PROFILE_CYCLE_START;
 197         }
 198
 199         /* create stack frame (if necessary) */
 200
 201         if (cd->stackframesize)
 202                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 203
 204         /* save used callee saved registers */
 205
 206         p = cd->stackframesize;
 207         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 208                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 209         }
 210         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 211                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 212         }
 213
 214         /* take arguments out of register or stack frame */
 215
 216         md = m->parseddesc;
 217
 218         for (p = 0, l = 0; p < md->paramcount; p++) {
 219                 t = md->paramtypes[p].type;
 220
 221                 varindex = jd->local_map[l * 5 + t];
 222
 223                 l++;
 224                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 225                         l++;
 226
 227                 if (varindex == UNUSED)
 228                         continue;
 229
 230                 var = VAR(varindex);
 231
 232                 s1 = md->params[p].regoff;
 233
 234                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 235                         s2 = rd->argintregs[s1];
 236                         if (!md->params[p].inmemory) {           /* register arguments    */
 237                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 238                                         M_INTMOVE(s2, var->vv.regoff);
 239
 240                                 } else {                             /* reg arg -> spilled    */
 241                                     M_LST(s2, REG_SP, var->vv.regoff * 8);
 242                                 }
 243
 244                         } else {                                 /* stack arguments       */
 245                                 if (!IS_INMEMORY(var->flags)) {      /* stack arg -> register */
 246                                         /* + 8 for return address */
 247                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 248
 249                                 } else {                             /* stack arg -> spilled  */
 250                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 251                                 }
 252                         }
 253
 254                 } else {                                     /* floating args         */
 255                         if (!md->params[p].inmemory) {           /* register arguments    */
 256                                 s2 = rd->argfltregs[s1];
 257                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 258                                         M_FLTMOVE(s2, var->vv.regoff);
 259
 260                                 } else {                                         /* reg arg -> spilled    */
 261                                         M_DST(s2, REG_SP, var->vv.regoff * 8);
 262                                 }
 263
 264                         } else {                                 /* stack arguments       */
 265                                 if (!IS_INMEMORY(var->flags)) {      /* stack-arg -> register */
 266                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 267
 268                                 } else {
 269                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 270                                 }
 271                         }
 272                 }
 273         }  /* end for */
 274
 275         /* save monitorenter argument */
 276
 277 #if defined(ENABLE_THREADS)
 278         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 279                 /* stack offset for monitor argument */
 280
 281                 s1 = rd->memuse;
 282
 283                 if (opt_verbosecall) {
 284                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 285
 286                         for (p = 0; p < INT_ARG_CNT; p++)
 287                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 288
 289                         for (p = 0; p < FLT_ARG_CNT; p++)
 290                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 291
 292                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 293                 }
 294
 295                 /* decide which monitor enter function to call */
 296
 297                 if (m->flags & ACC_STATIC) {
 298                         M_MOV_IMM(&m->class->object.header, REG_A0);
 299                 }
 300                 else {
 301                         M_TEST(REG_A0);
 302                         M_BEQ(0);
 303                         codegen_add_nullpointerexception_ref(cd);
 304                 }
 305
 306                 M_AST(REG_A0, REG_SP, s1 * 8);
 307                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 308                 M_CALL(REG_ITMP1);
 309
 310                 if (opt_verbosecall) {
 311                         for (p = 0; p < INT_ARG_CNT; p++)
 312                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 313
 314                         for (p = 0; p < FLT_ARG_CNT; p++)
 315                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 316
 317                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 318                 }
 319         }
 320 #endif
 321
 322 #if !defined(NDEBUG)
 323         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 324                 emit_verbosecall_enter(jd);
 325 #endif /* !defined(NDEBUG) */
 326
 327         }
 328
 329         /* end of header generation */
 330
 331         /* create replacement points */
 332
 333         REPLACEMENT_POINTS_INIT(cd, jd);
 334
 335         /* walk through all basic blocks */
 336
 337         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 338
 339                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 340
 341                 if (bptr->flags >= BBREACHED) {
 342
 343                 /* branch resolving */
 344
 345                 codegen_resolve_branchrefs(cd, bptr);
 346
 347                 /* handle replacement points */
 348
 349                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 350
 351                 /* copy interface registers to their destination */
 352
 353                 len = bptr->indepth;
 354                 MCODECHECK(512);
 355
 356                 /* generate basicblock profiling code */
 357
 358                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 359                         /* count frequency */
 360
 361                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 362                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 363
 364                         /* if this is an exception handler, start profiling again */
 365
 366                         if (bptr->type == BBTYPE_EXH)
 367                                 PROFILE_CYCLE_START;
 368                 }
 369
 370 #if defined(ENABLE_LSRA)
 371                 if (opt_lsra) {
 372                         while (len) {
 373                                 len--;
 374                                 src = bptr->invars[len];
 375                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 376                                         if (bptr->type == BBTYPE_EXH) {
 377 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 378                                                 if (!IS_INMEMORY(src->flags))
 379                                                         d= src->vv.regoff;
 380                                                 else
 381                                                         d=REG_ITMP1;
 382                                                 M_INTMOVE(REG_ITMP1, d);
 383                                                 emit_store(jd, NULL, src, d);
 384                                         }
 385                                 }
 386                         }
 387
 388                 } else {
 389 #endif
 390
 391                 while (len) {
 392                         len--;
 393                         var = VAR(bptr->invars[len]);
 394                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 395                                 if (bptr->type == BBTYPE_EXH) {
 396                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 397                                         M_INTMOVE(REG_ITMP1, d);
 398                                         emit_store(jd, NULL, var, d);
 399                                 }
 400                         }
 401                         else {
 402                                 assert((var->flags & INOUT));
 403                         }
 404                 }
 405 #if defined(ENABLE_LSRA)
 406                 }
 407 #endif
 408                 /* walk through all instructions */
 409
 410                 len = bptr->icount;
 411                 currentline = 0;
 412
 413                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 414                         if (iptr->line != currentline) {
 415                                 dseg_addlinenumber(cd, iptr->line);
 416                                 currentline = iptr->line;
 417                         }
 418
 419                         MCODECHECK(1024);                         /* 1KB should be enough */
 420
 421                 switch (iptr->opc) {
 422                 case ICMD_NOP:        /* ...  ==> ...                                 */
 423                 case ICMD_POP:        /* ..., value  ==> ...                          */
 424                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 425                         break;
 426
 427                 case ICMD_INLINE_START:
 428
 429                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 430                         break;
 431
 432                 case ICMD_INLINE_BODY:
 433
 434                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 435                         dseg_addlinenumber_inline_start(cd, iptr);
 436                         dseg_addlinenumber(cd, iptr->line);
 437                         break;
 438
 439                 case ICMD_INLINE_END:
 440
 441                         dseg_addlinenumber_inline_end(cd, iptr);
 442                         dseg_addlinenumber(cd, iptr->line);
 443                         break;
 444
 445                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 446
 447                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 448                         M_TEST(s1);
 449                         M_BEQ(0);
 450                         codegen_add_nullpointerexception_ref(cd);
 451                         break;
 452
 453                 /* constant operations ************************************************/
 454
 455                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 456
 457                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 458                         ICONST(d, iptr->sx.val.i);
 459                         emit_store_dst(jd, iptr, d);
 460                         break;
 461
 462                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 463
 464                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 465                         LCONST(d, iptr->sx.val.l);
 466                         emit_store_dst(jd, iptr, d);
 467                         break;
 468
 469                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 470
 471                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 472                         disp = dseg_add_float(cd, iptr->sx.val.f);
 473                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 474                         emit_store_dst(jd, iptr, d);
 475                         break;
 476
 477                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 478
 479                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 480                         disp = dseg_add_double(cd, iptr->sx.val.d);
 481                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 482                         emit_store_dst(jd, iptr, d);
 483                         break;
 484
 485                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 486
 487                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 488
 489                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 490                                 cr = iptr->sx.val.c.ref;
 491
 492 /*                              PROFILE_CYCLE_STOP; */
 493
 494                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 495
 496 /*                              PROFILE_CYCLE_START; */
 497
 498                                 M_MOV_IMM(NULL, d);
 499
 500                         } else {
 501                                 if (iptr->sx.val.anyptr == 0)
 502                                         M_CLR(d);
 503                                 else
 504                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 505                         }
 506                         emit_store_dst(jd, iptr, d);
 507                         break;
 508
 509
 510                 /* load/store/copy/move operations ************************************/
 511
 512                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 513                 case ICMD_ALOAD:      /* s1 = local variable                          */
 514                 case ICMD_LLOAD:
 515                 case ICMD_FLOAD:
 516                 case ICMD_DLOAD:
 517                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 518                 case ICMD_LSTORE:
 519                 case ICMD_FSTORE:
 520                 case ICMD_DSTORE:
 521                 case ICMD_COPY:
 522                 case ICMD_MOVE:
 523
 524                         emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 525                         break;
 526
 527                 case ICMD_ASTORE:
 528                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 529                                 emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 530                         break;
 531
 532                 /* integer operations *************************************************/
 533
 534                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 535
 536                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 537                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 538                         M_INTMOVE(s1, d);
 539                         M_INEG(d);
 540                         emit_store_dst(jd, iptr, d);
 541                         break;
 542
 543                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 544
 545                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 546                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 547                         M_INTMOVE(s1, d);
 548                         M_LNEG(d);
 549                         emit_store_dst(jd, iptr, d);
 550                         break;
 551
 552                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 553
 554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 556                         M_ISEXT(s1, d);
 557                         emit_store_dst(jd, iptr, d);
 558                         break;
 559
 560                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 561
 562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 563                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 564                         M_IMOV(s1, d);
 565                         emit_store_dst(jd, iptr, d);
 566                         break;
 567
 568                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 569
 570                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 571                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 572                         M_BSEXT(s1, d);
 573                         emit_store_dst(jd, iptr, d);
 574                         break;
 575
 576                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 577
 578                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 579                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 580                         M_CZEXT(s1, d);
 581                         emit_store_dst(jd, iptr, d);
 582                         break;
 583
 584                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 585
 586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 587                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 588                         M_SSEXT(s1, d);
 589                         emit_store_dst(jd, iptr, d);
 590                         break;
 591
 592
 593                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 594
 595                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 596                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 597                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 598                         if (s2 == d)
 599                                 M_IADD(s1, d);
 600                         else {
 601                                 M_INTMOVE(s1, d);
 602                                 M_IADD(s2, d);
 603                         }
 604                         emit_store_dst(jd, iptr, d);
 605                         break;
 606
 607                 case ICMD_IINC:
 608                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 609                                       /* sx.val.i = constant                             */
 610
 611                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 612                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 613
 614                         /* Using inc and dec is not faster than add (tested with
 615                            sieve). */
 616
 617                         M_INTMOVE(s1, d);
 618                         M_IADD_IMM(iptr->sx.val.i, d);
 619                         emit_store_dst(jd, iptr, d);
 620                         break;
 621
 622                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 623
 624                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 625                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 626                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 627                         if (s2 == d)
 628                                 M_LADD(s1, d);
 629                         else {
 630                                 M_INTMOVE(s1, d);
 631                                 M_LADD(s2, d);
 632                         }
 633                         emit_store_dst(jd, iptr, d);
 634                         break;
 635
 636                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 637                                       /* sx.val.l = constant                             */
 638
 639                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 640                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 641                         M_INTMOVE(s1, d);
 642                         if (IS_IMM32(iptr->sx.val.l))
 643                                 M_LADD_IMM(iptr->sx.val.l, d);
 644                         else {
 645                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 646                                 M_LADD(REG_ITMP2, d);
 647                         }
 648                         emit_store_dst(jd, iptr, d);
 649                         break;
 650
 651                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 652
 653                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 654                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 655                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 656                         if (s2 == d) {
 657                                 M_INTMOVE(s1, REG_ITMP1);
 658                                 M_ISUB(s2, REG_ITMP1);
 659                                 M_INTMOVE(REG_ITMP1, d);
 660                         } else {
 661                                 M_INTMOVE(s1, d);
 662                                 M_ISUB(s2, d);
 663                         }
 664                         emit_store_dst(jd, iptr, d);
 665                         break;
 666
 667                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 668                                       /* sx.val.i = constant                             */
 669
 670                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 671                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 672                         M_INTMOVE(s1, d);
 673                         M_ISUB_IMM(iptr->sx.val.i, d);
 674                         emit_store_dst(jd, iptr, d);
 675                         break;
 676
 677                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 678
 679                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 680                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 681                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 682                         if (s2 == d) {
 683                                 M_INTMOVE(s1, REG_ITMP1);
 684                                 M_LSUB(s2, REG_ITMP1);
 685                                 M_INTMOVE(REG_ITMP1, d);
 686                         } else {
 687                                 M_INTMOVE(s1, d);
 688                                 M_LSUB(s2, d);
 689                         }
 690                         emit_store_dst(jd, iptr, d);
 691                         break;
 692
 693                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 694                                       /* sx.val.l = constant                             */
 695
 696                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 697                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 698                         M_INTMOVE(s1, d);
 699                         if (IS_IMM32(iptr->sx.val.l))
 700                                 M_LSUB_IMM(iptr->sx.val.l, d);
 701                         else {
 702                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 703                                 M_LSUB(REG_ITMP2, d);
 704                         }
 705                         emit_store_dst(jd, iptr, d);
 706                         break;
 707
 708                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 709
 710                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 711                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 712                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 713                         if (s2 == d)
 714                                 M_IMUL(s1, d);
 715                         else {
 716                                 M_INTMOVE(s1, d);
 717                                 M_IMUL(s2, d);
 718                         }
 719                         emit_store_dst(jd, iptr, d);
 720                         break;
 721
 722                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 723                                       /* sx.val.i = constant                             */
 724
 725                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 726                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 727                         if (iptr->sx.val.i == 2) {
 728                                 M_INTMOVE(s1, d);
 729                                 M_ISLL_IMM(1, d);
 730                         } else
 731                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 732                         emit_store_dst(jd, iptr, d);
 733                         break;
 734
 735                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 736
 737                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 738                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 739                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 740                         if (s2 == d)
 741                                 M_LMUL(s1, d);
 742                         else {
 743                                 M_INTMOVE(s1, d);
 744                                 M_LMUL(s2, d);
 745                         }
 746                         emit_store_dst(jd, iptr, d);
 747                         break;
 748
 749                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 750                                       /* sx.val.l = constant                             */
 751
 752                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 753                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 754                         if (IS_IMM32(iptr->sx.val.l))
 755                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 756                         else {
 757                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 758                                 M_INTMOVE(s1, d);
 759                                 M_LMUL(REG_ITMP2, d);
 760                         }
 761                         emit_store_dst(jd, iptr, d);
 762                         break;
 763
 764                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 765
 766                         s1 = emit_load_s1(jd, iptr, RAX);
 767                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 768                         d = codegen_reg_of_dst(jd, iptr, RAX);
 769
 770                         M_INTMOVE(s1, RAX);
 771                         M_INTMOVE(s2, REG_ITMP3);
 772                         emit_arithmetic_check(cd, REG_ITMP3);
 773
 774                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 775
 776                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 777                         M_BNE(4 + 6);
 778                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 779                         M_BEQ(1 + 3);                                          /* 6 bytes */
 780
 781                         emit_cltd(cd);                                         /* 1 byte  */
 782                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 783
 784                         M_INTMOVE(RAX, d);
 785                         emit_store_dst(jd, iptr, d);
 786                         dst = VAROP(iptr->dst);
 787                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 788                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 789                         break;
 790
 791                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 792
 793                         s1 = emit_load_s1(jd, iptr, RAX);
 794                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 795                         d = codegen_reg_of_dst(jd, iptr, RDX);
 796
 797                         M_INTMOVE(s1, RAX);
 798                         M_INTMOVE(s2, REG_ITMP3);
 799                         emit_arithmetic_check(cd, REG_ITMP3);
 800
 801                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 802
 803                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 804                         M_BNE(3 + 4 + 6);
 805                         M_CLR(RDX);                                            /* 3 bytes */
 806                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 807                         M_BEQ(1 + 3);                                          /* 6 bytes */
 808
 809                         emit_cltd(cd);                                         /* 1 byte  */
 810                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 811
 812                         M_INTMOVE(RDX, d);
 813                         emit_store_dst(jd, iptr, d);
 814                         dst = VAROP(iptr->dst);
 815                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 816                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 817                         break;
 818
 819                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 820                                       /* sx.val.i = constant                             */
 821
 822                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 823                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 824                         M_INTMOVE(s1, REG_ITMP1);
 825                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 826                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 827                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 828                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 829                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 830                         emit_store_dst(jd, iptr, d);
 831                         break;
 832
 833                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 834                                       /* sx.val.i = constant                             */
 835
 836                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 837                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 838                         M_INTMOVE(s1, REG_ITMP1);
 839                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 840                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 841                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 842                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 843                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 844                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 845                         emit_store_dst(jd, iptr, d);
 846                         break;
 847
 848
 849                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 850
 851                         s1 = emit_load_s1(jd, iptr, RAX);
 852                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 853                         d = codegen_reg_of_dst(jd, iptr, RAX);
 854
 855                         M_INTMOVE(s1, RAX);
 856                         M_INTMOVE(s2, REG_ITMP3);
 857                         emit_arithmetic_check(cd, REG_ITMP3);
 858
 859                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 860
 861                         /* check as described in jvm spec */
 862                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 863                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 864                         M_BNE(4 + 6);
 865                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 866                         M_BEQ(2 + 3);                                          /* 6 bytes */
 867
 868                         emit_cqto(cd);                                         /* 2 bytes */
 869                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 870
 871                         M_INTMOVE(RAX, d);
 872                         emit_store_dst(jd, iptr, d);
 873                         dst = VAROP(iptr->dst);
 874                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 875                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 876                         break;
 877
 878                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 879
 880                         s1 = emit_load_s1(jd, iptr, RAX);
 881                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 882                         d = codegen_reg_of_dst(jd, iptr, RDX);
 883
 884                         M_INTMOVE(s1, RAX);
 885                         M_INTMOVE(s2, REG_ITMP3);
 886                         emit_arithmetic_check(cd, REG_ITMP3);
 887
 888                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 889
 890                         /* check as described in jvm spec */
 891                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 892                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 893                         M_BNE(3 + 4 + 6);
 894                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 895                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 896                         M_BEQ(2 + 3);                                          /* 6 bytes */
 897
 898                         emit_cqto(cd);                                         /* 2 bytes */
 899                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 900
 901                         M_INTMOVE(RDX, d);
 902                         emit_store_dst(jd, iptr, d);
 903                         dst = VAROP(iptr->dst);
 904                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 905                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 906                         break;
 907
 908                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 909                                       /* sx.val.i = constant                             */
 910
 911                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 912                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 913                         M_INTMOVE(s1, REG_ITMP1);
 914                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 915                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 916                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 917                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 918                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 919                         emit_store_dst(jd, iptr, d);
 920                         break;
 921
 922                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 923                                       /* sx.val.l = constant                             */
 924
 925                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 926                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 927                         M_INTMOVE(s1, REG_ITMP1);
 928                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 929                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 930                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 931                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 932                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 933                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 934                         emit_store_dst(jd, iptr, d);
 935                         break;
 936
 937                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 938
 939                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 940                         emit_ishift(jd, SHIFT_SHL, iptr);
 941                         break;
 942
 943                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 944                                       /* sx.val.i = constant                             */
 945
 946                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 947                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 948                         M_INTMOVE(s1, d);
 949                         M_ISLL_IMM(iptr->sx.val.i, d);
 950                         emit_store_dst(jd, iptr, d);
 951                         break;
 952
 953                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 954
 955                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 956                         emit_ishift(jd, SHIFT_SAR, iptr);
 957                         break;
 958
 959                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 960                                       /* sx.val.i = constant                             */
 961
 962                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 963                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 964                         M_INTMOVE(s1, d);
 965                         M_ISRA_IMM(iptr->sx.val.i, d);
 966                         emit_store_dst(jd, iptr, d);
 967                         break;
 968
 969                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 970
 971                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 972                         emit_ishift(jd, SHIFT_SHR, iptr);
 973                         break;
 974
 975                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 976                                       /* sx.val.i = constant                             */
 977
 978                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 979                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 980                         M_INTMOVE(s1, d);
 981                         M_ISRL_IMM(iptr->sx.val.i, d);
 982                         emit_store_dst(jd, iptr, d);
 983                         break;
 984
 985                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 986
 987                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 988                         emit_lshift(jd, SHIFT_SHL, iptr);
 989                         break;
 990
 991         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 992                                           /* sx.val.i = constant                             */
 993
 994                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 995                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 996                         M_INTMOVE(s1, d);
 997                         M_LSLL_IMM(iptr->sx.val.i, d);
 998                         emit_store_dst(jd, iptr, d);
 999                         break;
1000
1001                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1002
1003                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1004                         emit_lshift(jd, SHIFT_SAR, iptr);
1005                         break;
1006
1007                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1008                                       /* sx.val.i = constant                             */
1009
1010                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1011                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1012                         M_INTMOVE(s1, d);
1013                         M_LSRA_IMM(iptr->sx.val.i, d);
1014                         emit_store_dst(jd, iptr, d);
1015                         break;
1016
1017                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1018
1019                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1020                         emit_lshift(jd, SHIFT_SHR, iptr);
1021                         break;
1022
1023                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1024                                       /* sx.val.l = constant                             */
1025
1026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1027                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1028                         M_INTMOVE(s1, d);
1029                         M_LSRL_IMM(iptr->sx.val.i, d);
1030                         emit_store_dst(jd, iptr, d);
1031                         break;
1032
1033                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1034
1035                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1036                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1037                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1038                         if (s2 == d)
1039                                 M_IAND(s1, d);
1040                         else {
1041                                 M_INTMOVE(s1, d);
1042                                 M_IAND(s2, d);
1043                         }
1044                         emit_store_dst(jd, iptr, d);
1045                         break;
1046
1047                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1048                                       /* sx.val.i = constant                             */
1049
1050                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1051                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1052                         M_INTMOVE(s1, d);
1053                         M_IAND_IMM(iptr->sx.val.i, d);
1054                         emit_store_dst(jd, iptr, d);
1055                         break;
1056
1057                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1058
1059                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1060                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1061                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1062                         if (s2 == d)
1063                                 M_LAND(s1, d);
1064                         else {
1065                                 M_INTMOVE(s1, d);
1066                                 M_LAND(s2, d);
1067                         }
1068                         emit_store_dst(jd, iptr, d);
1069                         break;
1070
1071                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1072                                       /* sx.val.l = constant                             */
1073
1074                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1075                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1076                         M_INTMOVE(s1, d);
1077                         if (IS_IMM32(iptr->sx.val.l))
1078                                 M_LAND_IMM(iptr->sx.val.l, d);
1079                         else {
1080                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1081                                 M_LAND(REG_ITMP2, d);
1082                         }
1083                         emit_store_dst(jd, iptr, d);
1084                         break;
1085
1086                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1087
1088                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1089                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1090                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1091                         if (s2 == d)
1092                                 M_IOR(s1, d);
1093                         else {
1094                                 M_INTMOVE(s1, d);
1095                                 M_IOR(s2, d);
1096                         }
1097                         emit_store_dst(jd, iptr, d);
1098                         break;
1099
1100                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1101                                       /* sx.val.i = constant                             */
1102
1103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1104                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1105                         M_INTMOVE(s1, d);
1106                         M_IOR_IMM(iptr->sx.val.i, d);
1107                         emit_store_dst(jd, iptr, d);
1108                         break;
1109
1110                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1111
1112                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1113                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1114                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1115                         if (s2 == d)
1116                                 M_LOR(s1, d);
1117                         else {
1118                                 M_INTMOVE(s1, d);
1119                                 M_LOR(s2, d);
1120                         }
1121                         emit_store_dst(jd, iptr, d);
1122                         break;
1123
1124                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1125                                       /* sx.val.l = constant                             */
1126
1127                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1128                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1129                         M_INTMOVE(s1, d);
1130                         if (IS_IMM32(iptr->sx.val.l))
1131                                 M_LOR_IMM(iptr->sx.val.l, d);
1132                         else {
1133                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1134                                 M_LOR(REG_ITMP2, d);
1135                         }
1136                         emit_store_dst(jd, iptr, d);
1137                         break;
1138
1139                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1140
1141                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1142                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1143                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1144                         if (s2 == d)
1145                                 M_IXOR(s1, d);
1146                         else {
1147                                 M_INTMOVE(s1, d);
1148                                 M_IXOR(s2, d);
1149                         }
1150                         emit_store_dst(jd, iptr, d);
1151                         break;
1152
1153                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1154                                       /* sx.val.i = constant                             */
1155
1156                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1157                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1158                         M_INTMOVE(s1, d);
1159                         M_IXOR_IMM(iptr->sx.val.i, d);
1160                         emit_store_dst(jd, iptr, d);
1161                         break;
1162
1163                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1164
1165                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1166                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1167                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1168                         if (s2 == d)
1169                                 M_LXOR(s1, d);
1170                         else {
1171                                 M_INTMOVE(s1, d);
1172                                 M_LXOR(s2, d);
1173                         }
1174                         emit_store_dst(jd, iptr, d);
1175                         break;
1176
1177                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1178                                       /* sx.val.l = constant                             */
1179
1180                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1181                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1182                         M_INTMOVE(s1, d);
1183                         if (IS_IMM32(iptr->sx.val.l))
1184                                 M_LXOR_IMM(iptr->sx.val.l, d);
1185                         else {
1186                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1187                                 M_LXOR(REG_ITMP2, d);
1188                         }
1189                         emit_store_dst(jd, iptr, d);
1190                         break;
1191
1192
1193                 /* floating operations ************************************************/
1194
1195                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1196
1197                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1198                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1199                         disp = dseg_add_s4(cd, 0x80000000);
1200                         M_FLTMOVE(s1, d);
1201                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1202                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1203                         emit_store_dst(jd, iptr, d);
1204                         break;
1205
1206                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1207
1208                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1209                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1210                         disp = dseg_add_s8(cd, 0x8000000000000000);
1211                         M_FLTMOVE(s1, d);
1212                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1213                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1214                         emit_store_dst(jd, iptr, d);
1215                         break;
1216
1217                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1218
1219                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1220                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1221                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1222                         if (s2 == d)
1223                                 M_FADD(s1, d);
1224                         else {
1225                                 M_FLTMOVE(s1, d);
1226                                 M_FADD(s2, d);
1227                         }
1228                         emit_store_dst(jd, iptr, d);
1229                         break;
1230
1231                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1232
1233                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1234                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1235                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1236                         if (s2 == d)
1237                                 M_DADD(s1, d);
1238                         else {
1239                                 M_FLTMOVE(s1, d);
1240                                 M_DADD(s2, d);
1241                         }
1242                         emit_store_dst(jd, iptr, d);
1243                         break;
1244
1245                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1246
1247                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1248                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1249                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1250                         if (s2 == d) {
1251                                 M_FLTMOVE(s2, REG_FTMP2);
1252                                 s2 = REG_FTMP2;
1253                         }
1254                         M_FLTMOVE(s1, d);
1255                         M_FSUB(s2, d);
1256                         emit_store_dst(jd, iptr, d);
1257                         break;
1258
1259                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1260
1261                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1262                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1263                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1264                         if (s2 == d) {
1265                                 M_FLTMOVE(s2, REG_FTMP2);
1266                                 s2 = REG_FTMP2;
1267                         }
1268                         M_FLTMOVE(s1, d);
1269                         M_DSUB(s2, d);
1270                         emit_store_dst(jd, iptr, d);
1271                         break;
1272
1273                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1274
1275                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1276                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1277                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1278                         if (s2 == d)
1279                                 M_FMUL(s1, d);
1280                         else {
1281                                 M_FLTMOVE(s1, d);
1282                                 M_FMUL(s2, d);
1283                         }
1284                         emit_store_dst(jd, iptr, d);
1285                         break;
1286
1287                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1288
1289                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1290                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1291                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1292                         if (s2 == d)
1293                                 M_DMUL(s1, d);
1294                         else {
1295                                 M_FLTMOVE(s1, d);
1296                                 M_DMUL(s2, d);
1297                         }
1298                         emit_store_dst(jd, iptr, d);
1299                         break;
1300
1301                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1302
1303                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1304                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1305                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1306                         if (s2 == d) {
1307                                 M_FLTMOVE(s2, REG_FTMP2);
1308                                 s2 = REG_FTMP2;
1309                         }
1310                         M_FLTMOVE(s1, d);
1311                         M_FDIV(s2, d);
1312                         emit_store_dst(jd, iptr, d);
1313                         break;
1314
1315                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1316
1317                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1318                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1319                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1320                         if (s2 == d) {
1321                                 M_FLTMOVE(s2, REG_FTMP2);
1322                                 s2 = REG_FTMP2;
1323                         }
1324                         M_FLTMOVE(s1, d);
1325                         M_DDIV(s2, d);
1326                         emit_store_dst(jd, iptr, d);
1327                         break;
1328
1329                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1330
1331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1332                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1333                         M_CVTIF(s1, d);
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1340                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1341                         M_CVTID(s1, d);
1342                         emit_store_dst(jd, iptr, d);
1343                         break;
1344
1345                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1346
1347                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1348                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1349                         M_CVTLF(s1, d);
1350                         emit_store_dst(jd, iptr, d);
1351                         break;
1352
1353                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1354
1355                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1357                         M_CVTLD(s1, d);
1358                         emit_store_dst(jd, iptr, d);
1359                         break;
1360
1361                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1362
1363                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1364                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1365                         M_CVTFI(s1, d);
1366                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1367                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1368                                 ((REG_RESULT == d) ? 0 : 3);
1369                         M_BNE(disp);
1370                         M_FLTMOVE(s1, REG_FTMP1);
1371                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1372                         M_CALL(REG_ITMP2);
1373                         M_INTMOVE(REG_RESULT, d);
1374                         emit_store_dst(jd, iptr, d);
1375                         break;
1376
1377                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1378
1379                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1380                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1381                         M_CVTDI(s1, d);
1382                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1383                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1384                                 ((REG_RESULT == d) ? 0 : 3);
1385                         M_BNE(disp);
1386                         M_FLTMOVE(s1, REG_FTMP1);
1387                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1388                         M_CALL(REG_ITMP2);
1389                         M_INTMOVE(REG_RESULT, d);
1390                         emit_store_dst(jd, iptr, d);
1391                         break;
1392
1393                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1394
1395                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1396                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1397                         M_CVTFL(s1, d);
1398                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1399                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1400                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1401                                 ((REG_RESULT == d) ? 0 : 3);
1402                         M_BNE(disp);
1403                         M_FLTMOVE(s1, REG_FTMP1);
1404                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1405                         M_CALL(REG_ITMP2);
1406                         M_INTMOVE(REG_RESULT, d);
1407                         emit_store_dst(jd, iptr, d);
1408                         break;
1409
1410                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1411
1412                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1413                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1414                         M_CVTDL(s1, d);
1415                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1416                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1417                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1418                                 ((REG_RESULT == d) ? 0 : 3);
1419                         M_BNE(disp);
1420                         M_FLTMOVE(s1, REG_FTMP1);
1421                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1422                         M_CALL(REG_ITMP2);
1423                         M_INTMOVE(REG_RESULT, d);
1424                         emit_store_dst(jd, iptr, d);
1425                         break;
1426
1427                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1428
1429                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1430                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1431                         M_CVTFD(s1, d);
1432                         emit_store_dst(jd, iptr, d);
1433                         break;
1434
1435                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1436
1437                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1438                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1439                         M_CVTDF(s1, d);
1440                         emit_store_dst(jd, iptr, d);
1441                         break;
1442
1443                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1444                                           /* == => 0, < => 1, > => -1 */
1445
1446                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1447                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1448                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1449                         M_CLR(d);
1450                         M_MOV_IMM(1, REG_ITMP1);
1451                         M_MOV_IMM(-1, REG_ITMP2);
1452                         emit_ucomiss_reg_reg(cd, s1, s2);
1453                         M_CMOVB(REG_ITMP1, d);
1454                         M_CMOVA(REG_ITMP2, d);
1455                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1456                         emit_store_dst(jd, iptr, d);
1457                         break;
1458
1459                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1460                                           /* == => 0, < => 1, > => -1 */
1461
1462                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1463                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1464                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1465                         M_CLR(d);
1466                         M_MOV_IMM(1, REG_ITMP1);
1467                         M_MOV_IMM(-1, REG_ITMP2);
1468                         emit_ucomiss_reg_reg(cd, s1, s2);
1469                         M_CMOVB(REG_ITMP1, d);
1470                         M_CMOVA(REG_ITMP2, d);
1471                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1472                         emit_store_dst(jd, iptr, d);
1473                         break;
1474
1475                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1476                                           /* == => 0, < => 1, > => -1 */
1477
1478                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1479                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1480                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1481                         M_CLR(d);
1482                         M_MOV_IMM(1, REG_ITMP1);
1483                         M_MOV_IMM(-1, REG_ITMP2);
1484                         emit_ucomisd_reg_reg(cd, s1, s2);
1485                         M_CMOVB(REG_ITMP1, d);
1486                         M_CMOVA(REG_ITMP2, d);
1487                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1488                         emit_store_dst(jd, iptr, d);
1489                         break;
1490
1491                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1492                                           /* == => 0, < => 1, > => -1 */
1493
1494                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1495                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1496                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1497                         M_CLR(d);
1498                         M_MOV_IMM(1, REG_ITMP1);
1499                         M_MOV_IMM(-1, REG_ITMP2);
1500                         emit_ucomisd_reg_reg(cd, s1, s2);
1501                         M_CMOVB(REG_ITMP1, d);
1502                         M_CMOVA(REG_ITMP2, d);
1503                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1504                         emit_store_dst(jd, iptr, d);
1505                         break;
1506
1507
1508                 /* memory operations **************************************************/
1509
1510                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1511
1512                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1513                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1514                         emit_nullpointer_check(cd, s1);
1515                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1516                         emit_store_dst(jd, iptr, d);
1517                         break;
1518
1519                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1520
1521                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1522                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1523                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1524                         emit_array_checks(cd, iptr, s1, s2);
1525                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1526                         emit_store_dst(jd, iptr, d);
1527                         break;
1528
1529                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1530
1531                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1532                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1533                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1534                         emit_array_checks(cd, iptr, s1, s2);
1535                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1536                         emit_store_dst(jd, iptr, d);
1537                         break;
1538
1539                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1540
1541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1542                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1544                         emit_array_checks(cd, iptr, s1, s2);
1545                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1546                         emit_store_dst(jd, iptr, d);
1547                         break;
1548
1549                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1550
1551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1552                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1553                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1554                         emit_array_checks(cd, iptr, s1, s2);
1555                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1556                         emit_store_dst(jd, iptr, d);
1557                         break;
1558
1559                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1560
1561                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1562                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1563                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1564                         emit_array_checks(cd, iptr, s1, s2);
1565                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1566                         emit_store_dst(jd, iptr, d);
1567                         break;
1568
1569                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1570
1571                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1572                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1573                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1574                         emit_array_checks(cd, iptr, s1, s2);
1575                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1576                         emit_store_dst(jd, iptr, d);
1577                         break;
1578
1579                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1580
1581                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1582                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1583                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1584                         emit_array_checks(cd, iptr, s1, s2);
1585                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1586                         emit_store_dst(jd, iptr, d);
1587                         break;
1588
1589                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1590
1591                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1592                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1593                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1594                         emit_array_checks(cd, iptr, s1, s2);
1595                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1596                         emit_store_dst(jd, iptr, d);
1597                         break;
1598
1599
1600                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1601
1602                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1603                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1604                         emit_array_checks(cd, iptr, s1, s2);
1605                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1606                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1607                         break;
1608
1609                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1610
1611                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1612                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1613                         emit_array_checks(cd, iptr, s1, s2);
1614                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1615                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1616                         break;
1617
1618                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1619
1620                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1621                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1622                         emit_array_checks(cd, iptr, s1, s2);
1623                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1624                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1625                         break;
1626
1627                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1628
1629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1631                         emit_array_checks(cd, iptr, s1, s2);
1632                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1633                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1634                         break;
1635
1636                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1637
1638                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1639                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1640                         emit_array_checks(cd, iptr, s1, s2);
1641                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1642                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1643                         break;
1644
1645                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1646
1647                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1648                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1649                         emit_array_checks(cd, iptr, s1, s2);
1650                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1651                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1652                         break;
1653
1654                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1655
1656                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1657                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1658                         emit_array_checks(cd, iptr, s1, s2);
1659                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1660                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1661                         break;
1662
1663                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1664
1665                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1666                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1667                         emit_array_checks(cd, iptr, s1, s2);
1668                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1669
1670                         M_MOV(s1, REG_A0);
1671                         M_MOV(s3, REG_A1);
1672                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1673                         M_CALL(REG_ITMP1);
1674                         M_TEST(REG_RESULT);
1675                         M_BEQ(0);
1676                         codegen_add_arraystoreexception_ref(cd);
1677
1678                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1679                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1680                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1681                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1682                         break;
1683
1684
1685                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1686
1687                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1688                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1689                         emit_array_checks(cd, iptr, s1, s2);
1690                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1691                         break;
1692
1693                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1694
1695                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1696                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1697                         emit_array_checks(cd, iptr, s1, s2);
1698                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1699                         break;
1700
1701                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1702
1703                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1704                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1705                         emit_array_checks(cd, iptr, s1, s2);
1706                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1707                         break;
1708
1709                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1710
1711                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1712                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1713                         emit_array_checks(cd, iptr, s1, s2);
1714                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1715                         break;
1716
1717                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1718
1719                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1720                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1721                         emit_array_checks(cd, iptr, s1, s2);
1722
1723                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1724                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1725                         } else {
1726                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1727                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1728                         }
1729                         break;
1730
1731                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1732
1733                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1734                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1735                         emit_array_checks(cd, iptr, s1, s2);
1736                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1737                         break;
1738
1739
1740                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1741
1742                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1743                                 uf        = iptr->sx.s23.s3.uf;
1744                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1745                                 disp      = dseg_add_unique_address(cd, NULL);
1746                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1747
1748                                 /* must be calculated before codegen_add_patch_ref */
1749
1750                                 if (opt_shownops)
1751                                         disp -= PATCHER_CALL_SIZE;
1752
1753 /*                              PROFILE_CYCLE_STOP; */
1754
1755                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1756
1757 /*                              PROFILE_CYCLE_START; */
1758                         }
1759                         else {
1760                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1761                                 fieldtype = fi->type;
1762                                 disp      = dseg_add_address(cd, &(fi->value));
1763                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1764
1765                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1766                                         PROFILE_CYCLE_STOP;
1767
1768                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1769
1770                                         if (opt_shownops)
1771                                                 disp -= PATCHER_CALL_SIZE;
1772
1773                                         PROFILE_CYCLE_START;
1774                                 }
1775                         }
1776
1777                         /* This approach is much faster than moving the field
1778                            address inline into a register. */
1779
1780                         M_ALD(REG_ITMP1, RIP, disp);
1781
1782                         switch (fieldtype) {
1783                         case TYPE_INT:
1784                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1785                                 M_ILD(d, REG_ITMP1, 0);
1786                                 break;
1787                         case TYPE_LNG:
1788                         case TYPE_ADR:
1789                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1790                                 M_LLD(d, REG_ITMP1, 0);
1791                                 break;
1792                         case TYPE_FLT:
1793                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1794                                 M_FLD(d, REG_ITMP1, 0);
1795                                 break;
1796                         case TYPE_DBL:
1797                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1798                                 M_DLD(d, REG_ITMP1, 0);
1799                                 break;
1800                         }
1801                         emit_store_dst(jd, iptr, d);
1802                         break;
1803
1804                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1805
1806                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1807                                 uf        = iptr->sx.s23.s3.uf;
1808                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1809                                 disp      = dseg_add_unique_address(cd, NULL);
1810                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1811
1812                                 /* must be calculated before codegen_add_patch_ref */
1813
1814                                 if (opt_shownops)
1815                                         disp -= PATCHER_CALL_SIZE;
1816
1817 /*                              PROFILE_CYCLE_STOP; */
1818
1819                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1820
1821 /*                              PROFILE_CYCLE_START; */
1822                         }
1823                         else {
1824                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1825                                 fieldtype = fi->type;
1826                                 disp      = dseg_add_address(cd, &(fi->value));
1827                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1828
1829                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1830                                         PROFILE_CYCLE_STOP;
1831
1832                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1833
1834                                         if (opt_shownops)
1835                                                 disp -= PATCHER_CALL_SIZE;
1836
1837                                         PROFILE_CYCLE_START;
1838                                 }
1839                         }
1840
1841                         /* This approach is much faster than moving the field
1842                            address inline into a register. */
1843
1844                         M_ALD(REG_ITMP1, RIP, disp);
1845
1846                         switch (fieldtype) {
1847                         case TYPE_INT:
1848                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1849                                 M_IST(s1, REG_ITMP1, 0);
1850                                 break;
1851                         case TYPE_LNG:
1852                         case TYPE_ADR:
1853                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1854                                 M_LST(s1, REG_ITMP1, 0);
1855                                 break;
1856                         case TYPE_FLT:
1857                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1858                                 M_FST(s1, REG_ITMP1, 0);
1859                                 break;
1860                         case TYPE_DBL:
1861                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1862                                 M_DST(s1, REG_ITMP1, 0);
1863                                 break;
1864                         }
1865                         break;
1866
1867                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1868                                           /* val = value (in current instruction)     */
1869                                           /* following NOP)                           */
1870
1871                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1872                                 uf        = iptr->sx.s23.s3.uf;
1873                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1874                                 disp      = dseg_add_unique_address(cd, NULL);
1875                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1876
1877                                 /* must be calculated before codegen_add_patch_ref */
1878
1879                                 if (opt_shownops)
1880                                         disp -= PATCHER_CALL_SIZE;
1881
1882
1883 /*                              PROFILE_CYCLE_STOP; */
1884
1885                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1886
1887 /*                              PROFILE_CYCLE_START; */
1888                         }
1889                         else {
1890                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1891                                 fieldtype = fi->type;
1892                                 disp      = dseg_add_address(cd, &(fi->value));
1893                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1894
1895                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1896                                         PROFILE_CYCLE_STOP;
1897
1898                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1899
1900                                         if (opt_shownops)
1901                                                 disp -= PATCHER_CALL_SIZE;
1902
1903                                         PROFILE_CYCLE_START;
1904                                 }
1905                         }
1906
1907                         /* This approach is much faster than moving the field
1908                            address inline into a register. */
1909
1910                         M_ALD(REG_ITMP1, RIP, disp);
1911
1912                         switch (fieldtype) {
1913                         case TYPE_INT:
1914                         case TYPE_FLT:
1915                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1916                                 break;
1917                         case TYPE_LNG:
1918                         case TYPE_ADR:
1919                         case TYPE_DBL:
1920                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1921                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1922                                 else {
1923                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1924                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1925                                 }
1926                                 break;
1927                         }
1928                         break;
1929
1930                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1931
1932                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1933                         emit_nullpointer_check(cd, s1);
1934
1935                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1936                                 uf        = iptr->sx.s23.s3.uf;
1937                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1938                                 disp      = 0;
1939
1940 /*                              PROFILE_CYCLE_STOP; */
1941
1942                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1943
1944 /*                              PROFILE_CYCLE_START; */
1945                         }
1946                         else {
1947                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1948                                 fieldtype = fi->type;
1949                                 disp      = fi->offset;
1950                         }
1951
1952                         switch (fieldtype) {
1953                         case TYPE_INT:
1954                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1955                                 M_ILD32(d, s1, disp);
1956                                 break;
1957                         case TYPE_LNG:
1958                         case TYPE_ADR:
1959                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1960                                 M_LLD32(d, s1, disp);
1961                                 break;
1962                         case TYPE_FLT:
1963                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1964                                 M_FLD32(d, s1, disp);
1965                                 break;
1966                         case TYPE_DBL:
1967                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1968                                 M_DLD32(d, s1, disp);
1969                                 break;
1970                         }
1971                         emit_store_dst(jd, iptr, d);
1972                         break;
1973
1974                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1975
1976                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1977                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1978                         emit_nullpointer_check(cd, s1);
1979
1980                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1981                                 uf        = iptr->sx.s23.s3.uf;
1982                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1983                                 disp      = 0;
1984
1985 /*                              PROFILE_CYCLE_STOP; */
1986
1987                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1988
1989 /*                              PROFILE_CYCLE_START; */
1990                         }
1991                         else {
1992                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1993                                 fieldtype = fi->type;
1994                                 disp      = fi->offset;
1995                         }
1996
1997                         switch (fieldtype) {
1998                         case TYPE_INT:
1999                                 M_IST32(s2, s1, disp);
2000                                 break;
2001                         case TYPE_LNG:
2002                         case TYPE_ADR:
2003                                 M_LST32(s2, s1, disp);
2004                                 break;
2005                         case TYPE_FLT:
2006                                 M_FST32(s2, s1, disp);
2007                                 break;
2008                         case TYPE_DBL:
2009                                 M_DST32(s2, s1, disp);
2010                                 break;
2011                         }
2012                         break;
2013
2014                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2015                                           /* val = value (in current instruction)     */
2016                                           /* following NOP)                           */
2017
2018                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2019                         emit_nullpointer_check(cd, s1);
2020
2021                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2022                                 uf        = iptr->sx.s23.s3.uf;
2023                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2024                                 disp      = 0;
2025
2026 /*                              PROFILE_CYCLE_STOP; */
2027
2028                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2029
2030 /*                              PROFILE_CYCLE_START; */
2031                         }
2032                         else {
2033                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2034                                 fieldtype = fi->type;
2035                                 disp      = fi->offset;
2036                         }
2037
2038                         switch (fieldtype) {
2039                         case TYPE_INT:
2040                         case TYPE_FLT:
2041                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2042                                 break;
2043                         case TYPE_LNG:
2044                         case TYPE_ADR:
2045                         case TYPE_DBL:
2046                                 /* XXX why no check for IS_IMM32? */
2047                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2048                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2049                                 break;
2050                         }
2051                         break;
2052
2053
2054                 /* branch operations **************************************************/
2055
2056                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2057
2058                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2059                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2060
2061                         PROFILE_CYCLE_STOP;
2062
2063 #ifdef ENABLE_VERIFIER
2064                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2065                                 uc = iptr->sx.s23.s2.uc;
2066
2067                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2068                         }
2069 #endif /* ENABLE_VERIFIER */
2070
2071                         M_CALL_IMM(0);                            /* passing exception pc */
2072                         M_POP(REG_ITMP2_XPC);
2073
2074                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2075                         M_JMP(REG_ITMP3);
2076                         break;
2077
2078                 case ICMD_GOTO:         /* ... ==> ...                                */
2079                 case ICMD_RET:          /* ... ==> ...                                */
2080
2081                         M_JMP_IMM(0);
2082                         codegen_add_branch_ref(cd, iptr->dst.block);
2083                         break;
2084
2085                 case ICMD_JSR:          /* ... ==> ...                                */
2086
2087                         M_JMP_IMM(0);
2088                         codegen_add_branch_ref(cd, iptr->sx.s23.s3.jsrtarget.block);
2089                         break;
2090
2091                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2092
2093                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2094                         M_TEST(s1);
2095                         M_BEQ(0);
2096                         codegen_add_branch_ref(cd, iptr->dst.block);
2097                         break;
2098
2099                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2100
2101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2102                         M_TEST(s1);
2103                         M_BNE(0);
2104                         codegen_add_branch_ref(cd, iptr->dst.block);
2105                         break;
2106
2107                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2108
2109                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2110                         M_ICMP_IMM(iptr->sx.val.i, s1);
2111                         M_BEQ(0);
2112                         codegen_add_branch_ref(cd, iptr->dst.block);
2113                         break;
2114
2115                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2116
2117                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2118                         M_ICMP_IMM(iptr->sx.val.i, s1);
2119                         M_BLT(0);
2120                         codegen_add_branch_ref(cd, iptr->dst.block);
2121                         break;
2122
2123                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2124
2125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2126                         M_ICMP_IMM(iptr->sx.val.i, s1);
2127                         M_BLE(0);
2128                         codegen_add_branch_ref(cd, iptr->dst.block);
2129                         break;
2130
2131                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2132
2133                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2134                         M_ICMP_IMM(iptr->sx.val.i, s1);
2135                         M_BNE(0);
2136                         codegen_add_branch_ref(cd, iptr->dst.block);
2137                         break;
2138
2139                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2140
2141                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2142                         M_ICMP_IMM(iptr->sx.val.i, s1);
2143                         M_BGT(0);
2144                         codegen_add_branch_ref(cd, iptr->dst.block);
2145                         break;
2146
2147                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2148
2149                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2150                         M_ICMP_IMM(iptr->sx.val.i, s1);
2151                         M_BGE(0);
2152                         codegen_add_branch_ref(cd, iptr->dst.block);
2153                         break;
2154
2155                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2156
2157                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2158                         if (IS_IMM32(iptr->sx.val.l))
2159                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2160                         else {
2161                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2162                                 M_LCMP(REG_ITMP2, s1);
2163                         }
2164                         M_BEQ(0);
2165                         codegen_add_branch_ref(cd, iptr->dst.block);
2166                         break;
2167
2168                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2169
2170                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2171                         if (IS_IMM32(iptr->sx.val.l))
2172                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2173                         else {
2174                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2175                                 M_LCMP(REG_ITMP2, s1);
2176                         }
2177                         M_BLT(0);
2178                         codegen_add_branch_ref(cd, iptr->dst.block);
2179                         break;
2180
2181                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2182
2183                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2184                         if (IS_IMM32(iptr->sx.val.l))
2185                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2186                         else {
2187                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2188                                 M_LCMP(REG_ITMP2, s1);
2189                         }
2190                         M_BLE(0);
2191                         codegen_add_branch_ref(cd, iptr->dst.block);
2192                         break;
2193
2194                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2195
2196                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2197                         if (IS_IMM32(iptr->sx.val.l))
2198                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2199                         else {
2200                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2201                                 M_LCMP(REG_ITMP2, s1);
2202                         }
2203                         M_BNE(0);
2204                         codegen_add_branch_ref(cd, iptr->dst.block);
2205                         break;
2206
2207                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2208
2209                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2210                         if (IS_IMM32(iptr->sx.val.l))
2211                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2212                         else {
2213                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2214                                 M_LCMP(REG_ITMP2, s1);
2215                         }
2216                         M_BGT(0);
2217                         codegen_add_branch_ref(cd, iptr->dst.block);
2218                         break;
2219
2220                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2221
2222                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2223                         if (IS_IMM32(iptr->sx.val.l))
2224                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2225                         else {
2226                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2227                                 M_LCMP(REG_ITMP2, s1);
2228                         }
2229                         M_BGE(0);
2230                         codegen_add_branch_ref(cd, iptr->dst.block);
2231                         break;
2232
2233                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2234
2235                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2236                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2237                         M_ICMP(s2, s1);
2238                         M_BEQ(0);
2239                         codegen_add_branch_ref(cd, iptr->dst.block);
2240                         break;
2241
2242                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2243                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2244
2245                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2246                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2247                         M_LCMP(s2, s1);
2248                         M_BEQ(0);
2249                         codegen_add_branch_ref(cd, iptr->dst.block);
2250                         break;
2251
2252                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2253
2254                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2255                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2256                         M_ICMP(s2, s1);
2257                         M_BNE(0);
2258                         codegen_add_branch_ref(cd, iptr->dst.block);
2259                         break;
2260
2261                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2262                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2263
2264                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2265                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2266                         M_LCMP(s2, s1);
2267                         M_BNE(0);
2268                         codegen_add_branch_ref(cd, iptr->dst.block);
2269                         break;
2270
2271                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2272
2273                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2274                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2275                         M_ICMP(s2, s1);
2276                         M_BLT(0);
2277                         codegen_add_branch_ref(cd, iptr->dst.block);
2278                         break;
2279
2280                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2281
2282                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2283                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2284                         M_LCMP(s2, s1);
2285                         M_BLT(0);
2286                         codegen_add_branch_ref(cd, iptr->dst.block);
2287                         break;
2288
2289                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2290
2291                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2292                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2293                         M_ICMP(s2, s1);
2294                         M_BGT(0);
2295                         codegen_add_branch_ref(cd, iptr->dst.block);
2296                         break;
2297
2298                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2299
2300                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2301                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2302                         M_LCMP(s2, s1);
2303                         M_BGT(0);
2304                         codegen_add_branch_ref(cd, iptr->dst.block);
2305                         break;
2306
2307                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2308
2309                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2310                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2311                         M_ICMP(s2, s1);
2312                         M_BLE(0);
2313                         codegen_add_branch_ref(cd, iptr->dst.block);
2314                         break;
2315
2316                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2317
2318                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2319                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2320                         M_LCMP(s2, s1);
2321                         M_BLE(0);
2322                         codegen_add_branch_ref(cd, iptr->dst.block);
2323                         break;
2324
2325                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2326
2327                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2328                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2329                         M_ICMP(s2, s1);
2330                         M_BGE(0);
2331                         codegen_add_branch_ref(cd, iptr->dst.block);
2332                         break;
2333
2334                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2335
2336                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2337                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2338                         M_LCMP(s2, s1);
2339                         M_BGE(0);
2340                         codegen_add_branch_ref(cd, iptr->dst.block);
2341                         break;
2342
2343                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2344                 case ICMD_LRETURN:
2345
2346                         REPLACEMENT_POINT_RETURN(cd, iptr);
2347                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2348                         M_INTMOVE(s1, REG_RESULT);
2349                         goto nowperformreturn;
2350
2351                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2352
2353                         REPLACEMENT_POINT_RETURN(cd, iptr);
2354                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2355                         M_INTMOVE(s1, REG_RESULT);
2356
2357 #ifdef ENABLE_VERIFIER
2358                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2359                                 uc = iptr->sx.s23.s2.uc;
2360
2361                                 PROFILE_CYCLE_STOP;
2362
2363                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2364
2365                                 PROFILE_CYCLE_START;
2366                         }
2367 #endif /* ENABLE_VERIFIER */
2368                         goto nowperformreturn;
2369
2370                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2371                 case ICMD_DRETURN:
2372
2373                         REPLACEMENT_POINT_RETURN(cd, iptr);
2374                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2375                         M_FLTMOVE(s1, REG_FRESULT);
2376                         goto nowperformreturn;
2377
2378                 case ICMD_RETURN:      /* ...  ==> ...                                */
2379
2380                         REPLACEMENT_POINT_RETURN(cd, iptr);
2381
2382 nowperformreturn:
2383                         {
2384                         s4 i, p;
2385
2386                         p = cd->stackframesize;
2387
2388 #if !defined(NDEBUG)
2389                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2390                                 emit_verbosecall_exit(jd);
2391 #endif /* !defined(NDEBUG) */
2392
2393 #if defined(ENABLE_THREADS)
2394                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2395                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2396
2397                                 /* we need to save the proper return value */
2398                                 switch (iptr->opc) {
2399                                 case ICMD_IRETURN:
2400                                 case ICMD_ARETURN:
2401                                 case ICMD_LRETURN:
2402                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2403                                         break;
2404                                 case ICMD_FRETURN:
2405                                 case ICMD_DRETURN:
2406                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2407                                         break;
2408                                 }
2409
2410                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2411                                 M_CALL(REG_ITMP1);
2412
2413                                 /* and now restore the proper return value */
2414                                 switch (iptr->opc) {
2415                                 case ICMD_IRETURN:
2416                                 case ICMD_ARETURN:
2417                                 case ICMD_LRETURN:
2418                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2419                                         break;
2420                                 case ICMD_FRETURN:
2421                                 case ICMD_DRETURN:
2422                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2423                                         break;
2424                                 }
2425                         }
2426 #endif
2427
2428                         /* restore saved registers */
2429
2430                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2431                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2432                         }
2433                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2434                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2435                         }
2436
2437                         /* deallocate stack */
2438
2439                         if (cd->stackframesize)
2440                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2441
2442                         /* generate method profiling code */
2443
2444                         PROFILE_CYCLE_STOP;
2445
2446                         M_RET;
2447                         }
2448                         break;
2449
2450
2451                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2452                         {
2453                                 s4 i, l;
2454                                 branch_target_t *table;
2455
2456                                 table = iptr->dst.table;
2457
2458                                 l = iptr->sx.s23.s2.tablelow;
2459                                 i = iptr->sx.s23.s3.tablehigh;
2460
2461                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2462                                 M_INTMOVE(s1, REG_ITMP1);
2463
2464                                 if (l != 0)
2465                                         M_ISUB_IMM(l, REG_ITMP1);
2466
2467                                 /* number of targets */
2468                                 i = i - l + 1;
2469
2470                 /* range check */
2471                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2472                                 M_BA(0);
2473
2474                                 codegen_add_branch_ref(cd, table[0].block); /* default target */
2475
2476                                 /* build jump table top down and use address of lowest entry */
2477
2478                                 table += i;
2479
2480                                 while (--i >= 0) {
2481                                         dseg_add_target(cd, table->block);
2482                                         --table;
2483                                 }
2484
2485                                 /* length of dataseg after last dseg_add_target is used
2486                                    by load */
2487
2488                                 M_MOV_IMM(0, REG_ITMP2);
2489                                 dseg_adddata(cd);
2490                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2491                                 M_JMP(REG_ITMP1);
2492                         }
2493                         break;
2494
2495
2496                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2497                         {
2498                                 s4 i;
2499                                 lookup_target_t *lookup;
2500
2501                                 lookup = iptr->dst.lookup;
2502
2503                                 i = iptr->sx.s23.s2.lookupcount;
2504
2505                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2506                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2507
2508                                 while (--i >= 0) {
2509                                         M_ICMP_IMM(lookup->value, s1);
2510                                         M_BEQ(0);
2511                                         codegen_add_branch_ref(cd, lookup->target.block);
2512                                         lookup++;
2513                                 }
2514
2515                                 M_JMP_IMM(0);
2516
2517                                 codegen_add_branch_ref(cd, iptr->sx.s23.s3.lookupdefault.block);
2518                         }
2519                         break;
2520
2521
2522                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2523
2524                         bte = iptr->sx.s23.s3.bte;
2525                         md  = bte->md;
2526                         goto gen_method;
2527
2528                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2529
2530                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2531                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2532                 case ICMD_INVOKEINTERFACE:
2533
2534                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2535
2536                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2537                                 lm = NULL;
2538                                 um = iptr->sx.s23.s3.um;
2539                                 md = um->methodref->parseddesc.md;
2540                         }
2541                         else {
2542                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2543                                 um = NULL;
2544                                 md = lm->parseddesc;
2545                         }
2546
2547 gen_method:
2548                         s3 = md->paramcount;
2549
2550                         MCODECHECK((20 * s3) + 128);
2551
2552                         /* copy arguments to registers or stack location */
2553
2554                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2555                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2556
2557                                 /* Already Preallocated (ARGVAR) ? */
2558                                 if (var->flags & PREALLOC)
2559                                         continue;
2560
2561                                 if (IS_INT_LNG_TYPE(var->type)) {
2562                                         if (!md->params[s3].inmemory) {
2563                                                 s1 = rd->argintregs[md->params[s3].regoff];
2564                                                 d = emit_load(jd, iptr, var, s1);
2565                                                 M_INTMOVE(d, s1);
2566                                         }
2567                                         else {
2568                                                 d = emit_load(jd, iptr, var, REG_ITMP1);
2569                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2570                                         }
2571                                 }
2572                                 else {
2573                                         if (!md->params[s3].inmemory) {
2574                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2575                                                 d = emit_load(jd, iptr, var, s1);
2576                                                 M_FLTMOVE(d, s1);
2577                                         }
2578                                         else {
2579                                                 d = emit_load(jd, iptr, var, REG_FTMP1);
2580
2581                                                 if (IS_2_WORD_TYPE(var->type))
2582                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2583                                                 else
2584                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2585                                         }
2586                                 }
2587                         }
2588
2589                         /* generate method profiling code */
2590
2591                         PROFILE_CYCLE_STOP;
2592
2593                         switch (iptr->opc) {
2594                         case ICMD_BUILTIN:
2595                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2596                                 M_CALL(REG_ITMP1);
2597
2598                                 if (INSTRUCTION_MUST_CHECK(iptr)) {
2599                                         M_TEST(REG_RESULT);
2600                                         M_BEQ(0);
2601                                         codegen_add_fillinstacktrace_ref(cd);
2602                                 }
2603                                 break;
2604
2605                         case ICMD_INVOKESPECIAL:
2606                                 M_TEST(REG_A0);
2607                                 M_BEQ(0);
2608                                 codegen_add_nullpointerexception_ref(cd);
2609
2610                                 /* fall through */
2611
2612                         case ICMD_INVOKESTATIC:
2613                                 if (lm == NULL) {
2614                                         disp = dseg_add_unique_address(cd, NULL);
2615                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2616
2617                                         /* must be calculated before codegen_add_patch_ref */
2618
2619                                         if (opt_shownops)
2620                                                 disp -= PATCHER_CALL_SIZE;
2621
2622                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2623                                                                                   um, disp);
2624
2625 /*                                      a = 0; */
2626                                 }
2627                                 else {
2628                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2629                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2630
2631 /*                                      a = (ptrint) lm->stubroutine; */
2632                                 }
2633
2634 /*                              M_MOV_IMM(a, REG_ITMP2); */
2635                                 M_ALD(REG_ITMP2, RIP, disp);
2636                                 M_CALL(REG_ITMP2);
2637                                 break;
2638
2639                         case ICMD_INVOKEVIRTUAL:
2640                                 emit_nullpointer_check(cd, REG_A0);
2641
2642                                 if (lm == NULL) {
2643                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2644
2645                                         s1 = 0;
2646                                 }
2647                                 else
2648                                         s1 = OFFSET(vftbl_t, table[0]) +
2649                                                 sizeof(methodptr) * lm->vftblindex;
2650
2651                                 M_ALD(REG_METHODPTR, REG_A0,
2652                                           OFFSET(java_objectheader, vftbl));
2653                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2654                                 M_CALL(REG_ITMP3);
2655                                 break;
2656
2657                         case ICMD_INVOKEINTERFACE:
2658                                 emit_nullpointer_check(cd, REG_A0);
2659
2660                                 if (lm == NULL) {
2661                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2662
2663                                         s1 = 0;
2664                                         s2 = 0;
2665                                 }
2666                                 else {
2667                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2668                                                 sizeof(methodptr) * lm->class->index;
2669
2670                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2671                                 }
2672
2673                                 M_ALD(REG_METHODPTR, REG_A0,
2674                                           OFFSET(java_objectheader, vftbl));
2675                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2676                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2677                                 M_CALL(REG_ITMP3);
2678                                 break;
2679                         }
2680
2681                         /* generate method profiling code */
2682
2683                         PROFILE_CYCLE_START;
2684
2685                         /* store size of call code in replacement point */
2686
2687                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2688
2689                         /* store return value */
2690
2691                         d = md->returntype.type;
2692
2693                         if (d != TYPE_VOID) {
2694                                 if (IS_INT_LNG_TYPE(d)) {
2695                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2696                                         M_INTMOVE(REG_RESULT, s1);
2697                                 }
2698                                 else {
2699                                         s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2700                                         M_FLTMOVE(REG_FRESULT, s1);
2701                                 }
2702                                 emit_store_dst(jd, iptr, s1);
2703                         }
2704                         break;
2705
2706
2707                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2708
2709                                       /* val.a: (classinfo *) superclass              */
2710
2711                         /*  superclass is an interface:
2712                          *
2713                          *  OK if ((sub == NULL) ||
2714                          *         (sub->vftbl->interfacetablelength > super->index) &&
2715                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
2716                          *
2717                          *  superclass is a class:
2718                          *
2719                          *  OK if ((sub == NULL) || (0
2720                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
2721                          *         super->vftbl->diffval));
2722                          */
2723
2724                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2725                                 /* object type cast-check */
2726
2727                                 classinfo *super;
2728                                 vftbl_t   *supervftbl;
2729                                 s4         superindex;
2730
2731                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2732                                         super = NULL;
2733                                         superindex = 0;
2734                                         supervftbl = NULL;
2735                                 }
2736                                 else {
2737                                         super = iptr->sx.s23.s3.c.cls;
2738                                         superindex = super->index;
2739                                         supervftbl = super->vftbl;
2740                                 }
2741
2742 #if defined(ENABLE_THREADS)
2743                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2744 #endif
2745                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2746
2747                                 /* calculate interface checkcast code size */
2748
2749                                 s2 = 3; /* mov_membase_reg */
2750                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
2751
2752                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
2753                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
2754                                         3 /* test */ + 6 /* jcc */;
2755
2756                                 if (super == NULL)
2757                                         s2 += (opt_shownops ? 5 : 0);
2758
2759                                 /* calculate class checkcast code size */
2760
2761                                 s3 = 3; /* mov_membase_reg */
2762                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
2763                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
2764
2765 #if 0
2766                                 if (s1 != REG_ITMP1) {
2767                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2768                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
2769                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2770                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
2771                                         a += 3;    /* sub */
2772
2773                                 } else
2774 #endif
2775                                         {
2776                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
2777                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
2778                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2779                                         }
2780
2781                                 s3 += 3 /* cmp */ + 6 /* jcc */;
2782
2783                                 if (super == NULL)
2784                                         s3 += (opt_shownops ? 5 : 0);
2785
2786                                 /* if class is not resolved, check which code to call */
2787
2788                                 if (super == NULL) {
2789                                         M_TEST(s1);
2790                                         M_BEQ(6 + (opt_shownops ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
2791
2792                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2793                                                                                   iptr->sx.s23.s3.c.ref, 0);
2794
2795                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2796                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2797                                         M_BEQ(s2 + 5);
2798                                 }
2799
2800                                 /* interface checkcast code */
2801
2802                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2803                                         if (super != NULL) {
2804                                                 M_TEST(s1);
2805                                                 M_BEQ(s2);
2806                                         }
2807
2808                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2809
2810                                         if (super == NULL) {
2811                                                 codegen_add_patch_ref(cd,
2812                                                                                           PATCHER_checkcast_instanceof_interface,
2813                                                                                           iptr->sx.s23.s3.c.ref,
2814                                                                                           0);
2815                                         }
2816
2817                                         emit_movl_membase32_reg(cd, REG_ITMP2,
2818                                                                                           OFFSET(vftbl_t, interfacetablelength),
2819                                                                                           REG_ITMP3);
2820                                         /* XXX TWISTI: should this be int arithmetic? */
2821                                         M_LSUB_IMM32(superindex, REG_ITMP3);
2822                                         M_TEST(REG_ITMP3);
2823                                         M_BLE(0);
2824                                         codegen_add_classcastexception_ref(cd, s1);
2825                                         emit_mov_membase32_reg(cd, REG_ITMP2,
2826                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
2827                                                                                          superindex * sizeof(methodptr*),
2828                                                                                          REG_ITMP3);
2829                                         M_TEST(REG_ITMP3);
2830                                         M_BEQ(0);
2831                                         codegen_add_classcastexception_ref(cd, s1);
2832
2833                                         if (super == NULL)
2834                                                 M_JMP_IMM(s3);
2835                                 }
2836
2837                                 /* class checkcast code */
2838
2839                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2840                                         if (super != NULL) {
2841                                                 M_TEST(s1);
2842                                                 M_BEQ(s3);
2843                                         }
2844
2845                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2846
2847                                         if (super == NULL) {
2848                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2849                                                                                           iptr->sx.s23.s3.c.ref,
2850                                                                                           0);
2851                                         }
2852
2853                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2854 #if defined(ENABLE_THREADS)
2855                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2856 #endif
2857                                         emit_movl_membase32_reg(cd, REG_ITMP2,
2858                                                                                           OFFSET(vftbl_t, baseval),
2859                                                                                           REG_ITMP2);
2860                                         /*                                      if (s1 != REG_ITMP1) { */
2861                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2862                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2863                                         /*                                                                                              REG_ITMP1); */
2864                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2865                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2866                                         /*                                                                                              REG_ITMP3); */
2867                                         /*  #if defined(ENABLE_THREADS) */
2868                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2869                                         /*  #endif */
2870                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2871
2872                                         /*                                      } else { */
2873                                         emit_movl_membase32_reg(cd, REG_ITMP3,
2874                                                                                           OFFSET(vftbl_t, baseval),
2875                                                                                           REG_ITMP3);
2876                                         M_LSUB(REG_ITMP3, REG_ITMP2);
2877                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2878                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2879                                         /*                                      } */
2880 #if defined(ENABLE_THREADS)
2881                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2882 #endif
2883                                         M_LCMP(REG_ITMP3, REG_ITMP2);
2884                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
2885                                         codegen_add_classcastexception_ref(cd, s1);
2886                                 }
2887
2888                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2889                         }
2890                         else {
2891                                 /* array type cast-check */
2892
2893                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2894                                 M_INTMOVE(s1, REG_A0);
2895
2896                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2897                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2898                                                                                   iptr->sx.s23.s3.c.ref, 0);
2899                                 }
2900
2901                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2902                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2903                                 M_CALL(REG_ITMP1);
2904
2905                                 /* s1 may have been destroyed over the function call */
2906                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2907                                 M_TEST(REG_RESULT);
2908                                 M_BEQ(0);
2909                                 codegen_add_classcastexception_ref(cd, s1);
2910
2911                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2912                         }
2913
2914                         M_INTMOVE(s1, d);
2915                         emit_store_dst(jd, iptr, d);
2916                         break;
2917
2918                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2919
2920                                       /* val.a: (classinfo *) superclass              */
2921
2922                         /*  superclass is an interface:
2923                          *
2924                          *  return (sub != NULL) &&
2925                          *         (sub->vftbl->interfacetablelength > super->index) &&
2926                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
2927                          *
2928                          *  superclass is a class:
2929                          *
2930                          *  return ((sub != NULL) && (0
2931                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
2932                          *          super->vftbl->diffvall));
2933                          */
2934
2935                         {
2936                         classinfo *super;
2937                         vftbl_t   *supervftbl;
2938                         s4         superindex;
2939
2940                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2941                                 super = NULL;
2942                                 superindex = 0;
2943                                 supervftbl = NULL;
2944
2945                         } else {
2946                                 super = iptr->sx.s23.s3.c.cls;
2947                                 superindex = super->index;
2948                                 supervftbl = super->vftbl;
2949                         }
2950
2951 #if defined(ENABLE_THREADS)
2952             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2953 #endif
2954
2955                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2956                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2957                         if (s1 == d) {
2958                                 M_INTMOVE(s1, REG_ITMP1);
2959                                 s1 = REG_ITMP1;
2960                         }
2961
2962                         /* calculate interface instanceof code size */
2963
2964                         s2 = 3; /* mov_membase_reg */
2965                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
2966                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
2967                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
2968                                 3 /* test */ + 4 /* setcc */;
2969
2970                         if (!super)
2971                                 s2 += (opt_shownops ? 5 : 0);
2972
2973                         /* calculate class instanceof code size */
2974
2975                         s3 = 3; /* mov_membase_reg */
2976                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
2977                         s3 += 10; /* mov_imm_reg */
2978                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
2979                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
2980                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
2981                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
2982                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
2983                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2984                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
2985
2986                         if (super == NULL)
2987                                 s3 += (opt_shownops ? 5 : 0);
2988
2989                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
2990
2991                         /* if class is not resolved, check which code to call */
2992
2993                         if (super == NULL) {
2994                                 emit_test_reg_reg(cd, s1, s1);
2995                                 emit_jcc(cd, CC_Z, (6 + (opt_shownops ? 5 : 0) +
2996                                                                                          7 + 6 + s2 + 5 + s3));
2997
2998                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2999                                                                           iptr->sx.s23.s3.c.ref, 0);
3000
3001                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3002                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3003                                 emit_jcc(cd, CC_Z, s2 + 5);
3004                         }
3005
3006                         /* interface instanceof code */
3007
3008                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3009                                 if (super != NULL) {
3010                                         emit_test_reg_reg(cd, s1, s1);
3011                                         emit_jcc(cd, CC_Z, s2);
3012                                 }
3013
3014                                 emit_mov_membase_reg(cd, s1,
3015                                                                          OFFSET(java_objectheader, vftbl),
3016                                                                          REG_ITMP1);
3017
3018                                 if (super == NULL) {
3019                                         codegen_add_patch_ref(cd,
3020                                                                                   PATCHER_checkcast_instanceof_interface,
3021                                                                                   iptr->sx.s23.s3.c.ref, 0);
3022                                 }
3023
3024                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3025                                                                                   OFFSET(vftbl_t, interfacetablelength),
3026                                                                                   REG_ITMP3);
3027                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3028                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3029
3030                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3031
3032                                 emit_jcc(cd, CC_LE, a);
3033                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3034                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3035                                                                                  superindex * sizeof(methodptr*),
3036                                                                                  REG_ITMP1);
3037                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3038                                 emit_setcc_reg(cd, CC_NE, d);
3039
3040                                 if (!super)
3041                                         emit_jmp_imm(cd, s3);
3042                         }
3043
3044                         /* class instanceof code */
3045
3046                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3047                                 if (super != NULL) {
3048                                         emit_test_reg_reg(cd, s1, s1);
3049                                         emit_jcc(cd, CC_E, s3);
3050                                 }
3051
3052                                 emit_mov_membase_reg(cd, s1,
3053                                                                            OFFSET(java_objectheader, vftbl),
3054                                                                            REG_ITMP1);
3055
3056                                 if (super == NULL) {
3057                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
3058                                                                                   iptr->sx.s23.s3.c.ref, 0);
3059                                 }
3060
3061                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3062 #if defined(ENABLE_THREADS)
3063                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3064 #endif
3065                                 emit_movl_membase_reg(cd, REG_ITMP1,
3066                                                                                 OFFSET(vftbl_t, baseval),
3067                                                                                 REG_ITMP1);
3068                                 emit_movl_membase_reg(cd, REG_ITMP2,
3069                                                                                 OFFSET(vftbl_t, diffval),
3070                                                                                 REG_ITMP3);
3071                                 emit_movl_membase_reg(cd, REG_ITMP2,
3072                                                                                 OFFSET(vftbl_t, baseval),
3073                                                                                 REG_ITMP2);
3074 #if defined(ENABLE_THREADS)
3075                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3076 #endif
3077                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3078                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3079                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3080                                 emit_setcc_reg(cd, CC_BE, d);
3081                         }
3082                         emit_store_dst(jd, iptr, d);
3083                         }
3084                         break;
3085
3086                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3087
3088                         /* check for negative sizes and copy sizes to stack if necessary  */
3089
3090                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
3091
3092                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3093
3094                                 /* copy SAVEDVAR sizes to stack */
3095                                 var = VAR(iptr->sx.s23.s2.args[s1]);
3096
3097                                 /* Already Preallocated? */
3098                                 if (!(var->flags & PREALLOC)) {
3099                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
3100                                         M_LST(s2, REG_SP, s1 * 8);
3101                                 }
3102                         }
3103
3104                         /* is a patcher function set? */
3105
3106                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3107                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
3108                                                                           iptr->sx.s23.s3.c.ref, 0);
3109                         }
3110
3111                         /* a0 = dimension count */
3112
3113                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
3114
3115                         /* a1 = classinfo */
3116
3117                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3118
3119                         /* a2 = pointer to dimensions = stack pointer */
3120
3121                         M_MOV(REG_SP, REG_A2);
3122
3123                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3124                         M_CALL(REG_ITMP1);
3125
3126                         /* check for exception before result assignment */
3127
3128                         M_TEST(REG_RESULT);
3129                         M_BEQ(0);
3130                         codegen_add_fillinstacktrace_ref(cd);
3131
3132                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3133                         M_INTMOVE(REG_RESULT, s1);
3134                         emit_store_dst(jd, iptr, s1);
3135                         break;
3136
3137                 default:
3138                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3139                         return false;
3140         } /* switch */
3141
3142         } /* for instruction */
3143
3144         MCODECHECK(512); /* XXX require a lower number? */
3145
3146         /* At the end of a basic block we may have to append some nops,
3147            because the patcher stub calling code might be longer than the
3148            actual instruction. So codepatching does not change the
3149            following block unintentionally. */
3150
3151         if (cd->mcodeptr < cd->lastmcodeptr) {
3152                 while (cd->mcodeptr < cd->lastmcodeptr) {
3153                         M_NOP;
3154                 }
3155         }
3156
3157         } /* if (bptr -> flags >= BBREACHED) */
3158         } /* for basic block */
3159
3160         dseg_createlinenumbertable(cd);
3161
3162         /* generate stubs */
3163
3164         emit_exception_stubs(jd);
3165         emit_patcher_stubs(jd);
3166         REPLACEMENT_EMIT_STUBS(jd);
3167
3168         codegen_finish(jd);
3169
3170         /* everything's ok */
3171
3172         return true;
3173 }
3174
3175
3176 /* createcompilerstub **********************************************************
3177
3178    Creates a stub routine which calls the compiler.
3179
3180 *******************************************************************************/
3181
3182 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3183 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3184
3185 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3186
3187
3188 u1 *createcompilerstub(methodinfo *m)
3189 {
3190         u1          *s;                     /* memory to hold the stub            */
3191         ptrint      *d;
3192         codegendata *cd;
3193         s4           dumpsize;
3194
3195         s = CNEW(u1, COMPILERSTUB_SIZE);
3196
3197         /* set data pointer and code pointer */
3198
3199         d = (ptrint *) s;
3200         s = s + COMPILERSTUB_DATASIZE;
3201
3202         /* mark start of dump memory area */
3203
3204         dumpsize = dump_size();
3205
3206         cd = DNEW(codegendata);
3207         cd->mcodeptr = s;
3208
3209         /* The codeinfo pointer is actually a pointer to the
3210            methodinfo. This fakes a codeinfo structure. */
3211
3212         d[0] = (ptrint) asm_call_jit_compiler;
3213         d[1] = (ptrint) m;
3214         d[2] = (ptrint) &d[1];                                    /* fake code->m */
3215
3216         /* code for the stub */
3217
3218         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3219         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3220         M_JMP(REG_ITMP3);
3221
3222 #if defined(ENABLE_STATISTICS)
3223         if (opt_stat)
3224                 count_cstub_len += COMPILERSTUB_SIZE;
3225 #endif
3226
3227         /* release dump area */
3228
3229         dump_release(dumpsize);
3230
3231         return s;
3232 }
3233
3234
3235 /* createnativestub ************************************************************
3236
3237    Creates a stub routine which calls a native method.
3238
3239 *******************************************************************************/
3240
3241 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3242 {
3243         methodinfo   *m;
3244         codeinfo     *code;
3245         codegendata  *cd;
3246         registerdata *rd;
3247         methoddesc   *md;
3248         s4            nativeparams;
3249         s4            i, j;                 /* count variables                    */
3250         s4            t;
3251         s4            s1, s2;
3252
3253         /* get required compiler data */
3254
3255         m    = jd->m;
3256         code = jd->code;
3257         cd   = jd->cd;
3258         rd   = jd->rd;
3259
3260         /* initialize variables */
3261
3262         md = m->parseddesc;
3263         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3264
3265         /* calculate stack frame size */
3266
3267         cd->stackframesize =
3268                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3269                 sizeof(localref_table) / SIZEOF_VOID_P +
3270                 INT_ARG_CNT + FLT_ARG_CNT +
3271                 1 +                       /* functionptr, TODO: store in data segment */
3272                 nmd->memuse;
3273
3274         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3275
3276         /* create method header */
3277
3278         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3279         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3280         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3281         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3282         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3283         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3284         (void) dseg_addlinenumbertablesize(cd);
3285         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3286
3287         /* generate native method profiling code */
3288
3289         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3290                 /* count frequency */
3291
3292                 M_MOV_IMM(code, REG_ITMP3);
3293                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3294         }
3295
3296         /* generate stub code */
3297
3298         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3299
3300 #if !defined(NDEBUG)
3301         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3302                 emit_verbosecall_enter(jd);
3303 #endif
3304
3305         /* get function address (this must happen before the stackframeinfo) */
3306
3307 #if !defined(WITH_STATIC_CLASSPATH)
3308         if (f == NULL)
3309                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3310 #endif
3311
3312         M_MOV_IMM(f, REG_ITMP3);
3313
3314
3315         /* save integer and float argument registers */
3316
3317         for (i = 0, j = 0; i < md->paramcount; i++) {
3318                 if (!md->params[i].inmemory) {
3319                         s1 = md->params[i].regoff;
3320
3321                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3322                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3323                         else
3324                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3325
3326                         j++;
3327                 }
3328         }
3329
3330         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3331
3332         /* create dynamic stack info */
3333
3334         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3335         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3336         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3337         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3338         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3339         M_CALL(REG_ITMP1);
3340
3341         /* restore integer and float argument registers */
3342
3343         for (i = 0, j = 0; i < md->paramcount; i++) {
3344                 if (!md->params[i].inmemory) {
3345                         s1 = md->params[i].regoff;
3346
3347                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3348                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3349                         else
3350                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3351
3352                         j++;
3353                 }
3354         }
3355
3356         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3357
3358
3359         /* copy or spill arguments to new locations */
3360
3361         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3362                 t = md->paramtypes[i].type;
3363
3364                 if (IS_INT_LNG_TYPE(t)) {
3365                         if (!md->params[i].inmemory) {
3366                                 s1 = rd->argintregs[md->params[i].regoff];
3367
3368                                 if (!nmd->params[j].inmemory) {
3369                                         s2 = rd->argintregs[nmd->params[j].regoff];
3370                                         M_INTMOVE(s1, s2);
3371
3372                                 } else {
3373                                         s2 = nmd->params[j].regoff;
3374                                         M_LST(s1, REG_SP, s2 * 8);
3375                                 }
3376
3377                         } else {
3378                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3379                                 s2 = nmd->params[j].regoff;
3380                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3381                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3382                         }
3383
3384                 } else {
3385                         /* We only copy spilled float arguments, as the float argument    */
3386                         /* registers keep unchanged.                                      */
3387
3388                         if (md->params[i].inmemory) {
3389                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3390                                 s2 = nmd->params[j].regoff;
3391
3392                                 if (IS_2_WORD_TYPE(t)) {
3393                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3394                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3395                                 } else {
3396                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3397                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3398                                 }
3399                         }
3400                 }
3401         }
3402
3403         /* put class into second argument register */
3404
3405         if (m->flags & ACC_STATIC)
3406                 M_MOV_IMM(m->class, REG_A1);
3407
3408         /* put env into first argument register */
3409
3410         M_MOV_IMM(_Jv_env, REG_A0);
3411
3412         /* do the native function call */
3413
3414         M_CALL(REG_ITMP3);
3415
3416         /* save return value */
3417
3418         if (md->returntype.type != TYPE_VOID) {
3419                 if (IS_INT_LNG_TYPE(md->returntype.type))
3420                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3421                 else
3422                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3423         }
3424
3425 #if !defined(NDEBUG)
3426         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3427                 emit_verbosecall_exit(jd);
3428 #endif
3429
3430         /* remove native stackframe info */
3431
3432         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3433         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3434         M_CALL(REG_ITMP1);
3435         M_MOV(REG_RESULT, REG_ITMP3);
3436
3437         /* restore return value */
3438
3439         if (md->returntype.type != TYPE_VOID) {
3440                 if (IS_INT_LNG_TYPE(md->returntype.type))
3441                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3442                 else
3443                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3444         }
3445
3446         /* remove stackframe */
3447
3448         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3449
3450         /* test for exception */
3451
3452         M_TEST(REG_ITMP3);
3453         M_BNE(1);
3454         M_RET;
3455
3456         /* handle exception */
3457
3458         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3459         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3460         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3461
3462         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3463         M_JMP(REG_ITMP3);
3464
3465         /* generate patcher stubs */
3466
3467         emit_patcher_stubs(jd);
3468
3469         codegen_finish(jd);
3470
3471         return code->entrypoint;
3472 }
3473
3474
3475 /*
3476  * These are local overrides for various environment variables in Emacs.
3477  * Please do not remove this and leave it at the end of the file, where
3478  * Emacs will automagically detect them.
3479  * ---------------------------------------------------------------------
3480  * Local variables:
3481  * mode: c
3482  * indent-tabs-mode: t
3483  * c-basic-offset: 4
3484  * tab-width: 4
3485  * End:
3486  * vim:noexpandtab:sw=4:ts=4:
3487  */