src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007, 2008, 2009, 2010
   4    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
   5
   6    This file is part of CACAO.
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2, or (at
  11    your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful, but
  14    WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16    General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, write to the Free Software
  20    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  21    02110-1301, USA.
  22
  23 */
  24
  25
  26 #include "config.h"
  27
  28 #include <assert.h>
  29 #include <stdio.h>
  30 #include <stdint.h>
  31
  32 #include "vm/types.h"
  33 #include "vm/os.hpp"
  34
  35 #include "md-abi.h"
  36
  37 #include "vm/jit/x86_64/arch.h"
  38 #include "vm/jit/x86_64/codegen.h"
  39 #include "vm/jit/x86_64/emit.h"
  40
  41 #include "mm/memory.hpp"
  42
  43 #include "native/localref.hpp"
  44 #include "native/native.hpp"
  45
  46 #include "threads/lock.hpp"
  47
  48 #include "vm/jit/builtin.hpp"
  49 #include "vm/exceptions.hpp"
  50 #include "vm/global.h"
  51 #include "vm/loader.hpp"
  52 #include "vm/options.h"
  53 #include "vm/primitive.hpp"
  54 #include "vm/statistics.h"
  55 #include "vm/string.hpp"
  56 #include "vm/vm.hpp"
  57
  58 #include "vm/jit/abi.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/code.hpp"
  61 #include "vm/jit/codegen-common.hpp"
  62 #include "vm/jit/dseg.h"
  63 #include "vm/jit/emit-common.hpp"
  64 #include "vm/jit/jit.hpp"
  65 #include "vm/jit/linenumbertable.hpp"
  66 #include "vm/jit/methodheader.h"
  67 #include "vm/jit/parse.hpp"
  68 #include "vm/jit/patcher-common.hpp"
  69 #include "vm/jit/reg.h"
  70 #include "vm/jit/stacktrace.hpp"
  71 #include "vm/jit/trap.hpp"
  72
  73
  74 /**
  75  * Generates machine code for the method prolog.
  76  */
  77 void codegen_emit_prolog(jitdata* jd)
  78 {
  79         varinfo*    var;
  80         methoddesc* md;
  81         int32_t     s1;
  82         int32_t     p, t, l;
  83         int32_t     varindex;
  84         int         i;
  85
  86         // Get required compiler data.
  87         methodinfo*   m  = jd->m;
  88         codegendata*  cd = jd->cd;
  89         registerdata* rd = jd->rd;
  90
  91         /* create stack frame (if necessary) */
  92
  93         if (cd->stackframesize)
  94                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
  95
  96         /* save used callee saved registers */
  97
  98         p = cd->stackframesize;
  99         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 100                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 101         }
 102         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 103                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 104         }
 105
 106         /* take arguments out of register or stack frame */
 107
 108         md = m->parseddesc;
 109
 110         for (p = 0, l = 0; p < md->paramcount; p++) {
 111                 t = md->paramtypes[p].type;
 112
 113                 varindex = jd->local_map[l * 5 + t];
 114
 115                 l++;
 116                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 117                         l++;
 118
 119                 if (varindex == UNUSED)
 120                         continue;
 121
 122                 var = VAR(varindex);
 123
 124                 s1 = md->params[p].regoff;
 125
 126                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 127                         if (!md->params[p].inmemory) {           /* register arguments    */
 128                                 if (!IS_INMEMORY(var->flags))
 129                                         M_INTMOVE(s1, var->vv.regoff);
 130                                 else
 131                                     M_LST(s1, REG_SP, var->vv.regoff);
 132                         }
 133                         else {                                 /* stack arguments       */
 134                                 if (!IS_INMEMORY(var->flags))
 135                                         /* + 8 for return address */
 136                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 137                                 else
 138                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 139                         }
 140                 }
 141                 else {                                     /* floating args         */
 142                         if (!md->params[p].inmemory) {           /* register arguments    */
 143                                 if (!IS_INMEMORY(var->flags))
 144                                         emit_fmove(cd, s1, var->vv.regoff);
 145                                 else
 146                                         M_DST(s1, REG_SP, var->vv.regoff);
 147                         }
 148                         else {                                 /* stack arguments       */
 149                                 if (!IS_INMEMORY(var->flags))
 150                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 151                                 else
 152                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 153                         }
 154                 }
 155         }
 156 }
 157
 158
 159 /**
 160  * Generates machine code for the method epilog.
 161  */
 162 void codegen_emit_epilog(jitdata* jd)
 163 {
 164         int32_t p;
 165         int i;
 166
 167         // Get required compiler data.
 168         codegendata*  cd = jd->cd;
 169         registerdata* rd = jd->rd;
 170
 171         p = cd->stackframesize;
 172
 173         /* restore saved registers */
 174
 175         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 176                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
 177         }
 178         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 179                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
 180         }
 181
 182         /* deallocate stack */
 183
 184         if (cd->stackframesize)
 185                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
 186
 187         M_RET;
 188 }
 189
 190 /**
 191  * Generates a memory barrier to be used after volatile writes. It can be
 192  * patched out later if the field turns out not to be volatile.
 193  */
 194 void codegen_emit_patchable_barrier(instruction *iptr, codegendata *cd, patchref_t *pr, fieldinfo *fi)
 195 {
 196         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 197                 /* Align on word boundary */
 198                 if ((((intptr_t) cd->mcodeptr) & 3) >= 2)
 199                         emit_nop(cd, 4 - (((intptr_t) cd->mcodeptr) & 3));
 200                 /* Displacement for patching out MFENCE */
 201                 pr->disp_mb = (cd->mcodeptr - cd->mcodebase - pr->mpc);
 202         }
 203         if (INSTRUCTION_IS_UNRESOLVED(iptr) || fi->flags & ACC_VOLATILE)
 204                 M_MFENCE;
 205 }
 206
 207 /**
 208  * Generates machine code for one ICMD.
 209  */
 210 void codegen_emit_instruction(jitdata* jd, instruction* iptr)
 211 {
 212         varinfo*            var;
 213         varinfo*            dst;
 214         builtintable_entry* bte;
 215         methodinfo*         lm;             // Local methodinfo for ICMD_INVOKE*.
 216         unresolved_method*  um;
 217         fieldinfo*          fi;
 218         unresolved_field*   uf;
 219         patchref_t*         pr;
 220         int32_t             fieldtype;
 221         int32_t             s1, s2, s3, d;
 222         int32_t             disp;
 223
 224         // Get required compiler data.
 225         codegendata*  cd = jd->cd;
 226
 227         switch (iptr->opc) {
 228
 229                 /* constant operations ************************************************/
 230
 231                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 232
 233                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 234                         disp = dseg_add_float(cd, iptr->sx.val.f);
 235                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 236                         emit_store_dst(jd, iptr, d);
 237                         break;
 238
 239                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 240
 241                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 242                         disp = dseg_add_double(cd, iptr->sx.val.d);
 243                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 244                         emit_store_dst(jd, iptr, d);
 245                         break;
 246
 247                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 248
 249                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 250
 251                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 252                                 constant_classref *cr = iptr->sx.val.c.ref;
 253                                 disp = dseg_add_unique_address(cd, cr);
 254
 255 /*                              PROFILE_CYCLE_STOP; */
 256
 257                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 258                                                                           cr, disp);
 259
 260 /*                              PROFILE_CYCLE_START; */
 261
 262                                 M_ALD(d, RIP, disp);
 263                         }
 264                         else {
 265                                 if (iptr->sx.val.anyptr == 0) {
 266                                         M_CLR(d);
 267                                 }
 268                                 else {
 269                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 270                                         M_ALD(d, RIP, disp);
 271                                 }
 272                         }
 273                         emit_store_dst(jd, iptr, d);
 274                         break;
 275
 276
 277                 /* integer operations *************************************************/
 278
 279                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 280
 281                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 282                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 283                         M_INTMOVE(s1, d);
 284                         M_INEG(d);
 285                         emit_store_dst(jd, iptr, d);
 286                         break;
 287
 288                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 289
 290                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 291                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 292                         M_INTMOVE(s1, d);
 293                         M_LNEG(d);
 294                         emit_store_dst(jd, iptr, d);
 295                         break;
 296
 297                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 298
 299                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 300                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 301                         M_ISEXT(s1, d);
 302                         emit_store_dst(jd, iptr, d);
 303                         break;
 304
 305                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 306
 307                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 308                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 309                         M_IMOV(s1, d);
 310                         emit_store_dst(jd, iptr, d);
 311                         break;
 312
 313                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 314
 315                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 316                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 317                         M_BSEXT(s1, d);
 318                         emit_store_dst(jd, iptr, d);
 319                         break;
 320
 321                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 322
 323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 324                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 325                         M_CZEXT(s1, d);
 326                         emit_store_dst(jd, iptr, d);
 327                         break;
 328
 329                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 330
 331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 332                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 333                         M_SSEXT(s1, d);
 334                         emit_store_dst(jd, iptr, d);
 335                         break;
 336
 337
 338                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 339
 340                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 341                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 342                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 343                         if (s2 == d)
 344                                 M_IADD(s1, d);
 345                         else {
 346                                 M_INTMOVE(s1, d);
 347                                 M_IADD(s2, d);
 348                         }
 349                         emit_store_dst(jd, iptr, d);
 350                         break;
 351
 352                 case ICMD_IINC:
 353                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 354                                       /* sx.val.i = constant                             */
 355
 356                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 357                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 358
 359                         /* Using inc and dec is not faster than add (tested with
 360                            sieve). */
 361
 362                         M_INTMOVE(s1, d);
 363                         M_IADD_IMM(iptr->sx.val.i, d);
 364                         emit_store_dst(jd, iptr, d);
 365                         break;
 366
 367                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 368
 369                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 370                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 371                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 372                         if (s2 == d)
 373                                 M_LADD(s1, d);
 374                         else {
 375                                 M_INTMOVE(s1, d);
 376                                 M_LADD(s2, d);
 377                         }
 378                         emit_store_dst(jd, iptr, d);
 379                         break;
 380
 381                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 382                                       /* sx.val.l = constant                             */
 383
 384                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 385                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 386                         M_INTMOVE(s1, d);
 387                         if (IS_IMM32(iptr->sx.val.l))
 388                                 M_LADD_IMM(iptr->sx.val.l, d);
 389                         else {
 390                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 391                                 M_LADD(REG_ITMP2, d);
 392                         }
 393                         emit_store_dst(jd, iptr, d);
 394                         break;
 395
 396                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 397
 398                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 399                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 400                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 401                         if (s2 == d) {
 402                                 M_INTMOVE(s1, REG_ITMP1);
 403                                 M_ISUB(s2, REG_ITMP1);
 404                                 M_INTMOVE(REG_ITMP1, d);
 405                         } else {
 406                                 M_INTMOVE(s1, d);
 407                                 M_ISUB(s2, d);
 408                         }
 409                         emit_store_dst(jd, iptr, d);
 410                         break;
 411
 412                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 413                                       /* sx.val.i = constant                             */
 414
 415                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 416                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 417                         M_INTMOVE(s1, d);
 418                         M_ISUB_IMM(iptr->sx.val.i, d);
 419                         emit_store_dst(jd, iptr, d);
 420                         break;
 421
 422                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 423
 424                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 425                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 426                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 427                         if (s2 == d) {
 428                                 M_INTMOVE(s1, REG_ITMP1);
 429                                 M_LSUB(s2, REG_ITMP1);
 430                                 M_INTMOVE(REG_ITMP1, d);
 431                         } else {
 432                                 M_INTMOVE(s1, d);
 433                                 M_LSUB(s2, d);
 434                         }
 435                         emit_store_dst(jd, iptr, d);
 436                         break;
 437
 438                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 439                                       /* sx.val.l = constant                             */
 440
 441                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 442                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 443                         M_INTMOVE(s1, d);
 444                         if (IS_IMM32(iptr->sx.val.l))
 445                                 M_LSUB_IMM(iptr->sx.val.l, d);
 446                         else {
 447                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 448                                 M_LSUB(REG_ITMP2, d);
 449                         }
 450                         emit_store_dst(jd, iptr, d);
 451                         break;
 452
 453                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 454
 455                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 456                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 457                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 458                         if (s2 == d)
 459                                 M_IMUL(s1, d);
 460                         else {
 461                                 M_INTMOVE(s1, d);
 462                                 M_IMUL(s2, d);
 463                         }
 464                         emit_store_dst(jd, iptr, d);
 465                         break;
 466
 467                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 468                                       /* sx.val.i = constant                             */
 469
 470                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 471                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 472                         if (iptr->sx.val.i == 2) {
 473                                 M_INTMOVE(s1, d);
 474                                 M_ISLL_IMM(1, d);
 475                         } else
 476                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 477                         emit_store_dst(jd, iptr, d);
 478                         break;
 479
 480                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 481
 482                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 483                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 484                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 485                         if (s2 == d)
 486                                 M_LMUL(s1, d);
 487                         else {
 488                                 M_INTMOVE(s1, d);
 489                                 M_LMUL(s2, d);
 490                         }
 491                         emit_store_dst(jd, iptr, d);
 492                         break;
 493
 494                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 495                                       /* sx.val.l = constant                             */
 496
 497                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 498                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 499                         if (IS_IMM32(iptr->sx.val.l))
 500                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 501                         else {
 502                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 503                                 M_INTMOVE(s1, d);
 504                                 M_LMUL(REG_ITMP2, d);
 505                         }
 506                         emit_store_dst(jd, iptr, d);
 507                         break;
 508
 509                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 510
 511                         s1 = emit_load_s1(jd, iptr, RAX);
 512                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 513                         d = codegen_reg_of_dst(jd, iptr, RAX);
 514
 515                         M_INTMOVE(s1, RAX);
 516                         M_INTMOVE(s2, REG_ITMP3);
 517                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 518
 519                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 520
 521                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 522                         M_BNE(4 + 6);
 523                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 524                         M_BEQ(1 + 3);                                          /* 6 bytes */
 525
 526                         emit_cltd(cd);                                         /* 1 byte  */
 527                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 528
 529                         M_INTMOVE(RAX, d);
 530                         emit_store_dst(jd, iptr, d);
 531                         dst = VAROP(iptr->dst);
 532                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 533                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 534                         break;
 535
 536                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 537
 538                         s1 = emit_load_s1(jd, iptr, RAX);
 539                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 540                         d = codegen_reg_of_dst(jd, iptr, RDX);
 541
 542                         M_INTMOVE(s1, RAX);
 543                         M_INTMOVE(s2, REG_ITMP3);
 544                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 545
 546                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 547
 548                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 549                         M_BNE(3 + 4 + 6);
 550                         M_CLR(RDX);                                            /* 3 bytes */
 551                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 552                         M_BEQ(1 + 3);                                          /* 6 bytes */
 553
 554                         emit_cltd(cd);                                         /* 1 byte  */
 555                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 556
 557                         M_INTMOVE(RDX, d);
 558                         emit_store_dst(jd, iptr, d);
 559                         dst = VAROP(iptr->dst);
 560                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 561                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 562                         break;
 563
 564                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 565                                       /* sx.val.i = constant                             */
 566
 567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 568                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 569                         M_INTMOVE(s1, REG_ITMP1);
 570                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 571                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 572                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 573                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 574                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 575                         emit_store_dst(jd, iptr, d);
 576                         break;
 577
 578                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 579                                       /* sx.val.i = constant                             */
 580
 581                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 582                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 583                         M_INTMOVE(s1, REG_ITMP1);
 584                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 585                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 586                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 587                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 588                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 589                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 590                         emit_store_dst(jd, iptr, d);
 591                         break;
 592
 593
 594                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 595
 596                         s1 = emit_load_s1(jd, iptr, RAX);
 597                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 598                         d = codegen_reg_of_dst(jd, iptr, RAX);
 599
 600                         M_INTMOVE(s1, RAX);
 601                         M_INTMOVE(s2, REG_ITMP3);
 602                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 603
 604                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 605
 606                         /* check as described in jvm spec */
 607                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 608                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 609                         M_BNE(4 + 6);
 610                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 611                         M_BEQ(2 + 3);                                          /* 6 bytes */
 612
 613                         emit_cqto(cd);                                         /* 2 bytes */
 614                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 615
 616                         M_INTMOVE(RAX, d);
 617                         emit_store_dst(jd, iptr, d);
 618                         dst = VAROP(iptr->dst);
 619                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 620                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 621                         break;
 622
 623                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 624
 625                         s1 = emit_load_s1(jd, iptr, RAX);
 626                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 627                         d = codegen_reg_of_dst(jd, iptr, RDX);
 628
 629                         M_INTMOVE(s1, RAX);
 630                         M_INTMOVE(s2, REG_ITMP3);
 631                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 632
 633                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 634
 635                         /* check as described in jvm spec */
 636                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 637                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 638                         M_BNE(3 + 4 + 6);
 639                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 640                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 641                         M_BEQ(2 + 3);                                          /* 6 bytes */
 642
 643                         emit_cqto(cd);                                         /* 2 bytes */
 644                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 645
 646                         M_INTMOVE(RDX, d);
 647                         emit_store_dst(jd, iptr, d);
 648                         dst = VAROP(iptr->dst);
 649                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 650                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 651                         break;
 652
 653                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 654                                       /* sx.val.i = constant                             */
 655
 656                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 657                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 658                         M_INTMOVE(s1, REG_ITMP1);
 659                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 660                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 661                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 662                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 663                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 664                         emit_store_dst(jd, iptr, d);
 665                         break;
 666
 667                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 668                                       /* sx.val.l = constant                             */
 669
 670                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 671                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 672                         M_INTMOVE(s1, REG_ITMP1);
 673                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 674                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 675                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 676                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 677                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 678                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 679                         emit_store_dst(jd, iptr, d);
 680                         break;
 681
 682                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 683
 684                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 685                         emit_ishift(jd, SHIFT_SHL, iptr);
 686                         break;
 687
 688                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 689                                       /* sx.val.i = constant                             */
 690
 691                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 692                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 693                         M_INTMOVE(s1, d);
 694                         M_ISLL_IMM(iptr->sx.val.i, d);
 695                         emit_store_dst(jd, iptr, d);
 696                         break;
 697
 698                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 699
 700                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 701                         emit_ishift(jd, SHIFT_SAR, iptr);
 702                         break;
 703
 704                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 705                                       /* sx.val.i = constant                             */
 706
 707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 708                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 709                         M_INTMOVE(s1, d);
 710                         M_ISRA_IMM(iptr->sx.val.i, d);
 711                         emit_store_dst(jd, iptr, d);
 712                         break;
 713
 714                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 715
 716                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 717                         emit_ishift(jd, SHIFT_SHR, iptr);
 718                         break;
 719
 720                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 721                                       /* sx.val.i = constant                             */
 722
 723                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 724                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 725                         M_INTMOVE(s1, d);
 726                         M_ISRL_IMM(iptr->sx.val.i, d);
 727                         emit_store_dst(jd, iptr, d);
 728                         break;
 729
 730                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 731
 732                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 733                         emit_lshift(jd, SHIFT_SHL, iptr);
 734                         break;
 735
 736         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 737                                           /* sx.val.i = constant                             */
 738
 739                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 740                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 741                         M_INTMOVE(s1, d);
 742                         M_LSLL_IMM(iptr->sx.val.i, d);
 743                         emit_store_dst(jd, iptr, d);
 744                         break;
 745
 746                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 747
 748                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 749                         emit_lshift(jd, SHIFT_SAR, iptr);
 750                         break;
 751
 752                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 753                                       /* sx.val.i = constant                             */
 754
 755                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 756                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 757                         M_INTMOVE(s1, d);
 758                         M_LSRA_IMM(iptr->sx.val.i, d);
 759                         emit_store_dst(jd, iptr, d);
 760                         break;
 761
 762                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 763
 764                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 765                         emit_lshift(jd, SHIFT_SHR, iptr);
 766                         break;
 767
 768                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 769                                       /* sx.val.l = constant                             */
 770
 771                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 772                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 773                         M_INTMOVE(s1, d);
 774                         M_LSRL_IMM(iptr->sx.val.i, d);
 775                         emit_store_dst(jd, iptr, d);
 776                         break;
 777
 778                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
 779
 780                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 781                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 782                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 783                         if (s2 == d)
 784                                 M_IAND(s1, d);
 785                         else {
 786                                 M_INTMOVE(s1, d);
 787                                 M_IAND(s2, d);
 788                         }
 789                         emit_store_dst(jd, iptr, d);
 790                         break;
 791
 792                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
 793                                       /* sx.val.i = constant                             */
 794
 795                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 796                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 797                         M_INTMOVE(s1, d);
 798                         M_IAND_IMM(iptr->sx.val.i, d);
 799                         emit_store_dst(jd, iptr, d);
 800                         break;
 801
 802                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
 803
 804                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 805                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 806                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 807                         if (s2 == d)
 808                                 M_LAND(s1, d);
 809                         else {
 810                                 M_INTMOVE(s1, d);
 811                                 M_LAND(s2, d);
 812                         }
 813                         emit_store_dst(jd, iptr, d);
 814                         break;
 815
 816                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
 817                                       /* sx.val.l = constant                             */
 818
 819                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 820                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 821                         M_INTMOVE(s1, d);
 822                         if (IS_IMM32(iptr->sx.val.l))
 823                                 M_LAND_IMM(iptr->sx.val.l, d);
 824                         else {
 825                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 826                                 M_LAND(REG_ITMP2, d);
 827                         }
 828                         emit_store_dst(jd, iptr, d);
 829                         break;
 830
 831                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
 832
 833                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 834                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 835                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 836                         if (s2 == d)
 837                                 M_IOR(s1, d);
 838                         else {
 839                                 M_INTMOVE(s1, d);
 840                                 M_IOR(s2, d);
 841                         }
 842                         emit_store_dst(jd, iptr, d);
 843                         break;
 844
 845                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
 846                                       /* sx.val.i = constant                             */
 847
 848                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 849                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 850                         M_INTMOVE(s1, d);
 851                         M_IOR_IMM(iptr->sx.val.i, d);
 852                         emit_store_dst(jd, iptr, d);
 853                         break;
 854
 855                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
 856
 857                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 858                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 859                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 860                         if (s2 == d)
 861                                 M_LOR(s1, d);
 862                         else {
 863                                 M_INTMOVE(s1, d);
 864                                 M_LOR(s2, d);
 865                         }
 866                         emit_store_dst(jd, iptr, d);
 867                         break;
 868
 869                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
 870                                       /* sx.val.l = constant                             */
 871
 872                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 873                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 874                         M_INTMOVE(s1, d);
 875                         if (IS_IMM32(iptr->sx.val.l))
 876                                 M_LOR_IMM(iptr->sx.val.l, d);
 877                         else {
 878                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 879                                 M_LOR(REG_ITMP2, d);
 880                         }
 881                         emit_store_dst(jd, iptr, d);
 882                         break;
 883
 884                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
 885
 886                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 887                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 888                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 889                         if (s2 == d)
 890                                 M_IXOR(s1, d);
 891                         else {
 892                                 M_INTMOVE(s1, d);
 893                                 M_IXOR(s2, d);
 894                         }
 895                         emit_store_dst(jd, iptr, d);
 896                         break;
 897
 898                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
 899                                       /* sx.val.i = constant                             */
 900
 901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 902                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 903                         M_INTMOVE(s1, d);
 904                         M_IXOR_IMM(iptr->sx.val.i, d);
 905                         emit_store_dst(jd, iptr, d);
 906                         break;
 907
 908                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
 909
 910                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 911                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 912                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 913                         if (s2 == d)
 914                                 M_LXOR(s1, d);
 915                         else {
 916                                 M_INTMOVE(s1, d);
 917                                 M_LXOR(s2, d);
 918                         }
 919                         emit_store_dst(jd, iptr, d);
 920                         break;
 921
 922                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
 923                                       /* sx.val.l = constant                             */
 924
 925                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 926                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 927                         M_INTMOVE(s1, d);
 928                         if (IS_IMM32(iptr->sx.val.l))
 929                                 M_LXOR_IMM(iptr->sx.val.l, d);
 930                         else {
 931                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 932                                 M_LXOR(REG_ITMP2, d);
 933                         }
 934                         emit_store_dst(jd, iptr, d);
 935                         break;
 936
 937
 938                 /* floating operations ************************************************/
 939
 940                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
 941
 942                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 943                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 944                         disp = dseg_add_s4(cd, 0x80000000);
 945                         emit_fmove(cd, s1, d);
 946                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
 947                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
 948                         emit_store_dst(jd, iptr, d);
 949                         break;
 950
 951                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
 952
 953                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 954                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 955                         disp = dseg_add_s8(cd, 0x8000000000000000);
 956                         emit_fmove(cd, s1, d);
 957                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
 958                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
 959                         emit_store_dst(jd, iptr, d);
 960                         break;
 961
 962                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 963
 964                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 965                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 966                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 967                         if (s2 == d)
 968                                 M_FADD(s1, d);
 969                         else {
 970                                 emit_fmove(cd, s1, d);
 971                                 M_FADD(s2, d);
 972                         }
 973                         emit_store_dst(jd, iptr, d);
 974                         break;
 975
 976                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 977
 978                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 979                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 980                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 981                         if (s2 == d)
 982                                 M_DADD(s1, d);
 983                         else {
 984                                 emit_fmove(cd, s1, d);
 985                                 M_DADD(s2, d);
 986                         }
 987                         emit_store_dst(jd, iptr, d);
 988                         break;
 989
 990                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 991
 992                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 993                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 994                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 995                         if (s2 == d) {
 996                                 emit_fmove(cd, s2, REG_FTMP2);
 997                                 s2 = REG_FTMP2;
 998                         }
 999                         emit_fmove(cd, s1, d);
1000                         M_FSUB(s2, d);
1001                         emit_store_dst(jd, iptr, d);
1002                         break;
1003
1004                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1005
1006                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1007                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1008                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1009                         if (s2 == d) {
1010                                 emit_fmove(cd, s2, REG_FTMP2);
1011                                 s2 = REG_FTMP2;
1012                         }
1013                         emit_fmove(cd, s1, d);
1014                         M_DSUB(s2, d);
1015                         emit_store_dst(jd, iptr, d);
1016                         break;
1017
1018                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1019
1020                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1021                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1022                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1023                         if (s2 == d)
1024                                 M_FMUL(s1, d);
1025                         else {
1026                                 emit_fmove(cd, s1, d);
1027                                 M_FMUL(s2, d);
1028                         }
1029                         emit_store_dst(jd, iptr, d);
1030                         break;
1031
1032                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1033
1034                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1035                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1036                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1037                         if (s2 == d)
1038                                 M_DMUL(s1, d);
1039                         else {
1040                                 emit_fmove(cd, s1, d);
1041                                 M_DMUL(s2, d);
1042                         }
1043                         emit_store_dst(jd, iptr, d);
1044                         break;
1045
1046                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1047
1048                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1049                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1050                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1051                         if (s2 == d) {
1052                                 emit_fmove(cd, s2, REG_FTMP2);
1053                                 s2 = REG_FTMP2;
1054                         }
1055                         emit_fmove(cd, s1, d);
1056                         M_FDIV(s2, d);
1057                         emit_store_dst(jd, iptr, d);
1058                         break;
1059
1060                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1061
1062                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1063                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1064                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1065                         if (s2 == d) {
1066                                 emit_fmove(cd, s2, REG_FTMP2);
1067                                 s2 = REG_FTMP2;
1068                         }
1069                         emit_fmove(cd, s1, d);
1070                         M_DDIV(s2, d);
1071                         emit_store_dst(jd, iptr, d);
1072                         break;
1073
1074                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1075
1076                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1077                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1078                         M_CVTIF(s1, d);
1079                         emit_store_dst(jd, iptr, d);
1080                         break;
1081
1082                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1083
1084                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1085                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1086                         M_CVTID(s1, d);
1087                         emit_store_dst(jd, iptr, d);
1088                         break;
1089
1090                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1091
1092                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1093                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1094                         M_CVTLF(s1, d);
1095                         emit_store_dst(jd, iptr, d);
1096                         break;
1097
1098                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1099
1100                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1101                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1102                         M_CVTLD(s1, d);
1103                         emit_store_dst(jd, iptr, d);
1104                         break;
1105
1106                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1107
1108                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1109                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1110                         M_CVTFI(s1, d);
1111                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1112                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1113                                 ((REG_RESULT == d) ? 0 : 3);
1114                         M_BNE(disp);
1115                         emit_fmove(cd, s1, REG_FTMP1);
1116                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1117                         M_CALL(REG_ITMP2);
1118                         M_INTMOVE(REG_RESULT, d);
1119                         emit_store_dst(jd, iptr, d);
1120                         break;
1121
1122                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1123
1124                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1125                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1126                         M_CVTDI(s1, d);
1127                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1128                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1129                                 ((REG_RESULT == d) ? 0 : 3);
1130                         M_BNE(disp);
1131                         emit_fmove(cd, s1, REG_FTMP1);
1132                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1133                         M_CALL(REG_ITMP2);
1134                         M_INTMOVE(REG_RESULT, d);
1135                         emit_store_dst(jd, iptr, d);
1136                         break;
1137
1138                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1139
1140                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1141                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1142                         M_CVTFL(s1, d);
1143                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1144                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1145                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1146                                 ((REG_RESULT == d) ? 0 : 3);
1147                         M_BNE(disp);
1148                         emit_fmove(cd, s1, REG_FTMP1);
1149                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1150                         M_CALL(REG_ITMP2);
1151                         M_INTMOVE(REG_RESULT, d);
1152                         emit_store_dst(jd, iptr, d);
1153                         break;
1154
1155                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1156
1157                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1158                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1159                         M_CVTDL(s1, d);
1160                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1161                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1162                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1163                                 ((REG_RESULT == d) ? 0 : 3);
1164                         M_BNE(disp);
1165                         emit_fmove(cd, s1, REG_FTMP1);
1166                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1167                         M_CALL(REG_ITMP2);
1168                         M_INTMOVE(REG_RESULT, d);
1169                         emit_store_dst(jd, iptr, d);
1170                         break;
1171
1172                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1173
1174                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1175                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1176                         M_CVTFD(s1, d);
1177                         emit_store_dst(jd, iptr, d);
1178                         break;
1179
1180                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1181
1182                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1183                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1184                         M_CVTDF(s1, d);
1185                         emit_store_dst(jd, iptr, d);
1186                         break;
1187
1188                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1189                                           /* == => 0, < => 1, > => -1 */
1190
1191                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1192                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1193                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1194                         M_CLR(d);
1195                         M_MOV_IMM(1, REG_ITMP1);
1196                         M_MOV_IMM(-1, REG_ITMP2);
1197                         emit_ucomiss_reg_reg(cd, s1, s2);
1198                         M_CMOVULT(REG_ITMP1, d);
1199                         M_CMOVUGT(REG_ITMP2, d);
1200                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1201                         emit_store_dst(jd, iptr, d);
1202                         break;
1203
1204                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1205                                           /* == => 0, < => 1, > => -1 */
1206
1207                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1208                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1209                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1210                         M_CLR(d);
1211                         M_MOV_IMM(1, REG_ITMP1);
1212                         M_MOV_IMM(-1, REG_ITMP2);
1213                         emit_ucomiss_reg_reg(cd, s1, s2);
1214                         M_CMOVULT(REG_ITMP1, d);
1215                         M_CMOVUGT(REG_ITMP2, d);
1216                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1217                         emit_store_dst(jd, iptr, d);
1218                         break;
1219
1220                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1221                                           /* == => 0, < => 1, > => -1 */
1222
1223                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1224                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1225                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1226                         M_CLR(d);
1227                         M_MOV_IMM(1, REG_ITMP1);
1228                         M_MOV_IMM(-1, REG_ITMP2);
1229                         emit_ucomisd_reg_reg(cd, s1, s2);
1230                         M_CMOVULT(REG_ITMP1, d);
1231                         M_CMOVUGT(REG_ITMP2, d);
1232                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1233                         emit_store_dst(jd, iptr, d);
1234                         break;
1235
1236                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1237                                           /* == => 0, < => 1, > => -1 */
1238
1239                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1240                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1241                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1242                         M_CLR(d);
1243                         M_MOV_IMM(1, REG_ITMP1);
1244                         M_MOV_IMM(-1, REG_ITMP2);
1245                         emit_ucomisd_reg_reg(cd, s1, s2);
1246                         M_CMOVULT(REG_ITMP1, d);
1247                         M_CMOVUGT(REG_ITMP2, d);
1248                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1249                         emit_store_dst(jd, iptr, d);
1250                         break;
1251
1252
1253                 /* memory operations **************************************************/
1254
1255                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1256
1257                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1258                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1259                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1260                         /* implicit null-pointer check */
1261                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1262                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1263                         emit_store_dst(jd, iptr, d);
1264                         break;
1265
1266                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1267
1268                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1269                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1270                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1271                         /* implicit null-pointer check */
1272                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1273                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1274                         emit_store_dst(jd, iptr, d);
1275                         break;
1276
1277                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1278
1279                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1280                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1281                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1282                         /* implicit null-pointer check */
1283                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1284                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1285                         emit_store_dst(jd, iptr, d);
1286                         break;
1287
1288                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1289
1290                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1291                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1292                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1293                         /* implicit null-pointer check */
1294                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1295                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1296                         emit_store_dst(jd, iptr, d);
1297                         break;
1298
1299                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1300
1301                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1302                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1303                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1304                         /* implicit null-pointer check */
1305                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1306                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1307                         emit_store_dst(jd, iptr, d);
1308                         break;
1309
1310                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1311
1312                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1313                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1314                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1315                         /* implicit null-pointer check */
1316                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1317                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1318                         emit_store_dst(jd, iptr, d);
1319                         break;
1320
1321                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1322
1323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1324                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1325                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1326                         /* implicit null-pointer check */
1327                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1328                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1329                         emit_store_dst(jd, iptr, d);
1330                         break;
1331
1332                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1333
1334                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1335                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1336                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1337                         /* implicit null-pointer check */
1338                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1339                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1340                         emit_store_dst(jd, iptr, d);
1341                         break;
1342
1343
1344                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1345
1346                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1347                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1348                         /* implicit null-pointer check */
1349                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1350                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1351                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1352                         break;
1353
1354                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1355
1356                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1357                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1358                         /* implicit null-pointer check */
1359                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1360                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1361                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1362                         break;
1363
1364                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1365
1366                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1367                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1368                         /* implicit null-pointer check */
1369                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1370                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1371                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1372                         break;
1373
1374                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1375
1376                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1377                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1378                         /* implicit null-pointer check */
1379                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1380                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1381                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1382                         break;
1383
1384                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1385
1386                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1387                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1388                         /* implicit null-pointer check */
1389                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1390                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1391                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1392                         break;
1393
1394                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1395
1396                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1397                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1398                         /* implicit null-pointer check */
1399                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1400                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1401                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1402                         break;
1403
1404                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1405
1406                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1407                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1408                         /* implicit null-pointer check */
1409                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1410                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1411                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1412                         break;
1413
1414                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1415
1416                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1417                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1418                         /* implicit null-pointer check */
1419                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1420                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1421
1422                         M_MOV(s1, REG_A0);
1423                         M_MOV(s3, REG_A1);
1424                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1425                         M_CALL(REG_ITMP1);
1426                         emit_arraystore_check(cd, iptr);
1427
1428                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1429                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1430                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1431                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1432                         break;
1433
1434
1435                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1436
1437                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1438                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1439                         /* implicit null-pointer check */
1440                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1441                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1442                         break;
1443
1444                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1445
1446                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1447                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1448                         /* implicit null-pointer check */
1449                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1450                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1451                         break;
1452
1453                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1454
1455                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1456                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1457                         /* implicit null-pointer check */
1458                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1459                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1460                         break;
1461
1462                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1463
1464                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1465                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1466                         /* implicit null-pointer check */
1467                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1468                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1469                         break;
1470
1471                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1472
1473                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1474                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1475                         /* implicit null-pointer check */
1476                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1477
1478                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1479                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1480                         }
1481                         else {
1482                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1483                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1484                         }
1485                         break;
1486
1487                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1488
1489                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1490                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1491                         /* implicit null-pointer check */
1492                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1493                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1494                         break;
1495
1496                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1497                                           /* val = value (in current instruction)     */
1498                                           /* following NOP)                           */
1499
1500                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1501                                 uf        = iptr->sx.s23.s3.uf;
1502                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1503                                 disp      = dseg_add_unique_address(cd, uf);
1504
1505 /*                              PROFILE_CYCLE_STOP; */
1506
1507                                 pr = patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1508
1509 /*                              PROFILE_CYCLE_START; */
1510
1511                                 fi = NULL;              /* Silence compiler warning */
1512                         }
1513                         else {
1514                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1515                                 fieldtype = fi->type;
1516                                 disp      = dseg_add_address(cd, fi->value);
1517
1518                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1519                                         //PROFILE_CYCLE_STOP;
1520
1521                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1522                                                                                   fi->clazz, 0);
1523
1524                                         //PROFILE_CYCLE_START;
1525                                 }
1526
1527                                 pr = NULL;              /* Silence compiler warning */
1528                         }
1529
1530                         /* This approach is much faster than moving the field
1531                            address inline into a register. */
1532
1533                         M_ALD(REG_ITMP1, RIP, disp);
1534
1535                         switch (fieldtype) {
1536                         case TYPE_INT:
1537                         case TYPE_FLT:
1538                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1539                                 break;
1540                         case TYPE_LNG:
1541                         case TYPE_ADR:
1542                         case TYPE_DBL:
1543                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1544                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1545                                 else {
1546                                         M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1547                                         M_LST(REG_ITMP2, REG_ITMP1, 0);
1548                                 }
1549                                 break;
1550                         }
1551                         codegen_emit_patchable_barrier(iptr, cd, pr, fi);
1552                         break;
1553
1554                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1555
1556                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1557
1558                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1559                                 uf        = iptr->sx.s23.s3.uf;
1560                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1561                                 disp      = 0;
1562
1563 /*                              PROFILE_CYCLE_STOP; */
1564
1565                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1566
1567 /*                              PROFILE_CYCLE_START; */
1568
1569                                 fi = NULL;              /* Silence compiler warning */
1570                         }
1571                         else {
1572                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1573                                 fieldtype = fi->type;
1574                                 disp      = fi->offset;
1575                         }
1576
1577                         /* implicit null-pointer check */
1578                         switch (fieldtype) {
1579                         case TYPE_INT:
1580                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1581                                 M_ILD32(d, s1, disp);
1582                                 break;
1583                         case TYPE_LNG:
1584                         case TYPE_ADR:
1585                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1586                                 M_LLD32(d, s1, disp);
1587                                 break;
1588                         case TYPE_FLT:
1589                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1590                                 M_FLD32(d, s1, disp);
1591                                 break;
1592                         case TYPE_DBL:
1593                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1594                                 M_DLD32(d, s1, disp);
1595                                 break;
1596                         default:
1597                                 // Silence compiler warning.
1598                                 d = 0;
1599                         }
1600                         emit_store_dst(jd, iptr, d);
1601                         break;
1602
1603                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1604
1605                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1606                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1607
1608                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1609                                 uf        = iptr->sx.s23.s3.uf;
1610                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1611                                 disp      = 0;
1612
1613 /*                              PROFILE_CYCLE_STOP; */
1614
1615                                 pr = patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1616
1617 /*                              PROFILE_CYCLE_START; */
1618
1619                                 fi = NULL;              /* Silence compiler warning */
1620                         }
1621                         else {
1622                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1623                                 fieldtype = fi->type;
1624                                 disp      = fi->offset;
1625
1626                                 pr = NULL;              /* Silence compiler warning */
1627                         }
1628
1629                         /* implicit null-pointer check */
1630                         switch (fieldtype) {
1631                         case TYPE_INT:
1632                                 M_IST32(s2, s1, disp);
1633                                 break;
1634                         case TYPE_LNG:
1635                         case TYPE_ADR:
1636                                 M_LST32(s2, s1, disp);
1637                                 break;
1638                         case TYPE_FLT:
1639                                 M_FST32(s2, s1, disp);
1640                                 break;
1641                         case TYPE_DBL:
1642                                 M_DST32(s2, s1, disp);
1643                                 break;
1644                         }
1645                         codegen_emit_patchable_barrier(iptr, cd, pr, fi);
1646                         break;
1647
1648                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1649                                           /* val = value (in current instruction)     */
1650                                           /* following NOP)                           */
1651
1652                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1653
1654                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1655                                 uf        = iptr->sx.s23.s3.uf;
1656                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1657                                 disp      = 0;
1658
1659 /*                              PROFILE_CYCLE_STOP; */
1660
1661                                 pr = patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1662
1663 /*                              PROFILE_CYCLE_START; */
1664
1665                                 fi = NULL;              /* Silence compiler warning */
1666                         }
1667                         else {
1668                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1669                                 fieldtype = fi->type;
1670                                 disp      = fi->offset;
1671
1672                                 pr = NULL;              /* Silence compiler warning */
1673                         }
1674
1675                         /* implicit null-pointer check */
1676                         switch (fieldtype) {
1677                         case TYPE_INT:
1678                         case TYPE_FLT:
1679                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
1680                                 break;
1681                         case TYPE_LNG:
1682                         case TYPE_ADR:
1683                         case TYPE_DBL:
1684                                 /* XXX why no check for IS_IMM32? -- probably because of the patcher */
1685                                 M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1686                                 if (disp)  /* resolved, disp can never be 0 */
1687                                         M_LST(REG_ITMP2, s1, disp);
1688                                 else       /* unresolved */
1689                                         M_LST32(REG_ITMP2, s1, disp);
1690                                 break;
1691                         }
1692                         codegen_emit_patchable_barrier(iptr, cd, pr, fi);
1693                         break;
1694
1695
1696                 /* branch operations **************************************************/
1697
1698                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
1699
1700                         M_CALL_IMM(0);                            /* passing exception pc */
1701                         M_POP(REG_ITMP2_XPC);
1702
1703                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
1704                         M_JMP(REG_ITMP3);
1705                         break;
1706
1707                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
1708                 case ICMD_IF_LNE:
1709                 case ICMD_IF_LLT:
1710                 case ICMD_IF_LGE:
1711                 case ICMD_IF_LGT:
1712                 case ICMD_IF_LLE:
1713
1714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1715                         if (IS_IMM32(iptr->sx.val.l))
1716                                 M_LCMP_IMM(iptr->sx.val.l, s1);
1717                         else {
1718                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1719                                 M_LCMP(REG_ITMP2, s1);
1720                         }
1721                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
1722                         break;
1723
1724                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
1725                 case ICMD_IF_LCMPNE:
1726                 case ICMD_IF_LCMPLT:
1727                 case ICMD_IF_LCMPGE:
1728                 case ICMD_IF_LCMPGT:
1729                 case ICMD_IF_LCMPLE:
1730
1731                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1732                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1733                         M_LCMP(s2, s1);
1734                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
1735                         break;
1736
1737                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
1738                         {
1739                                 s4 i, l;
1740                                 branch_target_t *table;
1741
1742                                 table = iptr->dst.table;
1743
1744                                 l = iptr->sx.s23.s2.tablelow;
1745                                 i = iptr->sx.s23.s3.tablehigh;
1746
1747                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1748                                 M_INTMOVE(s1, REG_ITMP1);
1749
1750                                 if (l != 0)
1751                                         M_ISUB_IMM(l, REG_ITMP1);
1752
1753                                 /* number of targets */
1754                                 i = i - l + 1;
1755
1756                 /* range check */
1757
1758                                 M_ICMP_IMM(i - 1, REG_ITMP1);
1759                                 emit_bugt(cd, table[0].block);
1760
1761                                 /* build jump table top down and use address of lowest entry */
1762
1763                                 table += i;
1764
1765                                 while (--i >= 0) {
1766                                         dseg_add_target(cd, table->block);
1767                                         --table;
1768                                 }
1769
1770                                 /* length of dataseg after last dseg_add_target is used
1771                                    by load */
1772
1773                                 M_MOV_IMM(0, REG_ITMP2);
1774                                 dseg_adddata(cd);
1775                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
1776                                 M_JMP(REG_ITMP1);
1777                         }
1778                         break;
1779
1780                 case ICMD_BUILTIN:
1781                         bte = iptr->sx.s23.s3.bte;
1782                         if (bte->stub == NULL) {
1783                                 M_MOV_IMM(bte->fp, REG_ITMP1);
1784                         }
1785                         else {
1786                                 M_MOV_IMM(bte->stub, REG_ITMP1);
1787                         }
1788                         M_CALL(REG_ITMP1);
1789                         break;
1790
1791                 case ICMD_INVOKESPECIAL:
1792                         emit_nullpointer_check(cd, iptr, REG_A0);
1793                         /* fall through */
1794
1795                 case ICMD_INVOKESTATIC:
1796                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1797                                 um = iptr->sx.s23.s3.um;
1798                                 disp = dseg_add_unique_address(cd, um);
1799
1800                                 patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
1801                                                                           um, disp);
1802                         }
1803                         else {
1804                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1805                                 disp = dseg_add_functionptr(cd, lm->stubroutine);
1806                         }
1807
1808                         M_ALD(REG_ITMP2, RIP, disp);
1809                         M_CALL(REG_ITMP2);
1810                         break;
1811
1812                 case ICMD_INVOKEVIRTUAL:
1813                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1814                                 um = iptr->sx.s23.s3.um;
1815                                 patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
1816
1817                                 s1 = 0;
1818                         }
1819                         else {
1820                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1821                                 s1 = OFFSET(vftbl_t, table[0]) +
1822                                         sizeof(methodptr) * lm->vftblindex;
1823                         }
1824
1825                         /* implicit null-pointer check */
1826                         M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
1827                         M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
1828                         M_CALL(REG_ITMP3);
1829                         break;
1830
1831                 case ICMD_INVOKEINTERFACE:
1832                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1833                                 um = iptr->sx.s23.s3.um;
1834                                 patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
1835
1836                                 s1 = 0;
1837                                 s2 = 0;
1838                         }
1839                         else {
1840                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1841                                 s1 = OFFSET(vftbl_t, interfacetable[0]) -
1842                                         sizeof(methodptr) * lm->clazz->index;
1843
1844                                 s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
1845                         }
1846
1847                         /* implicit null-pointer check */
1848                         M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
1849                         M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
1850                         M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
1851                         M_CALL(REG_ITMP3);
1852                         break;
1853
1854                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
1855
1856                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
1857                                 /* object type cast-check */
1858
1859                                 classinfo *super;
1860                                 s4         superindex;
1861
1862                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1863                                         super      = NULL;
1864                                         superindex = 0;
1865                                 }
1866                                 else {
1867                                         super      = iptr->sx.s23.s3.c.cls;
1868                                         superindex = super->index;
1869                                 }
1870
1871                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1872
1873                                 /* if class is not resolved, check which code to call */
1874
1875                                 if (super == NULL) {
1876                                         M_TEST(s1);
1877                                         emit_label_beq(cd, BRANCH_LABEL_1);
1878
1879                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
1880                                                                                   iptr->sx.s23.s3.c.ref, 0);
1881
1882                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
1883                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
1884                                         emit_label_beq(cd, BRANCH_LABEL_2);
1885                                 }
1886
1887                                 /* interface checkcast code */
1888
1889                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
1890                                         if (super != NULL) {
1891                                                 M_TEST(s1);
1892                                                 emit_label_beq(cd, BRANCH_LABEL_3);
1893                                         }
1894
1895                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
1896
1897                                         if (super == NULL) {
1898                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
1899                                                                                           iptr->sx.s23.s3.c.ref,
1900                                                                                           0);
1901                                         }
1902
1903                                         M_ILD32(REG_ITMP3,
1904                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
1905                                         M_ICMP_IMM32(superindex, REG_ITMP3);
1906                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
1907
1908                                         M_ALD32(REG_ITMP3, REG_ITMP2,
1909                                                         OFFSET(vftbl_t, interfacetable[0]) -
1910                                                         superindex * sizeof(methodptr*));
1911                                         M_TEST(REG_ITMP3);
1912                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
1913
1914                                         if (super == NULL)
1915                                                 emit_label_br(cd, BRANCH_LABEL_4);
1916                                         else
1917                                                 emit_label(cd, BRANCH_LABEL_3);
1918                                 }
1919
1920                                 /* class checkcast code */
1921
1922                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
1923                                         if (super == NULL) {
1924                                                 emit_label(cd, BRANCH_LABEL_2);
1925
1926                                                 constant_classref *cr = iptr->sx.s23.s3.c.ref;
1927                                                 disp = dseg_add_unique_address(cd, cr);
1928
1929                                                 patcher_add_patch_ref(jd,
1930                                                                                           PATCHER_resolve_classref_to_vftbl,
1931                                                                                           cr, disp);
1932                                         }
1933                                         else {
1934                                                 M_TEST(s1);
1935                                                 emit_label_beq(cd, BRANCH_LABEL_5);
1936
1937                                                 disp = dseg_add_address(cd, super->vftbl);
1938                                         }
1939
1940                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
1941                                         M_ALD(REG_ITMP3, RIP, disp);
1942
1943                                         if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
1944                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
1945                                                 M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
1946                                                 emit_label_beq(cd, BRANCH_LABEL_6);  /* good */
1947
1948                                                 if (super == NULL) {
1949                                                         M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
1950                                                         emit_label_bne(cd, BRANCH_LABEL_10);  /* throw */
1951                                                 }
1952
1953                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
1954                                                 M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
1955                                                 emit_label_bgt(cd, BRANCH_LABEL_9);  /* throw */
1956
1957                                                 M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
1958                                                 M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
1959                                                 emit_label_beq(cd, BRANCH_LABEL_7);  /* good */
1960
1961                                                 emit_label(cd, BRANCH_LABEL_9);
1962                                                 if (super == NULL)
1963                                                         emit_label(cd, BRANCH_LABEL_10);
1964
1965                                                 /* reload s1, might have been destroyed */
1966                                                 emit_load_s1(jd, iptr, REG_ITMP1);
1967                                                 M_ALD_MEM(s1, TRAP_ClassCastException);
1968
1969                                                 emit_label(cd, BRANCH_LABEL_7);
1970                                                 emit_label(cd, BRANCH_LABEL_6);
1971                                                 /* reload s1, might have been destroyed */
1972                                                 emit_load_s1(jd, iptr, REG_ITMP1);
1973                                         }
1974                                         else {
1975                                                 M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
1976                                                 emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
1977                                         }
1978
1979                                         if (super != NULL)
1980                                                 emit_label(cd, BRANCH_LABEL_5);
1981                                 }
1982
1983                                 if (super == NULL) {
1984                                         emit_label(cd, BRANCH_LABEL_1);
1985                                         emit_label(cd, BRANCH_LABEL_4);
1986                                 }
1987
1988                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1989                         }
1990                         else {
1991                                 /* array type cast-check */
1992
1993                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1994                                 M_INTMOVE(s1, REG_A0);
1995
1996                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1997                                         constant_classref *cr = iptr->sx.s23.s3.c.ref;
1998                                         disp = dseg_add_unique_address(cd, cr);
1999
2000                                         patcher_add_patch_ref(jd,
2001                                                                                   PATCHER_resolve_classref_to_classinfo,
2002                                                                                   cr, disp);
2003                                 }
2004                                 else {
2005                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2006                                 }
2007
2008                                 M_ALD(REG_A1, RIP, disp);
2009                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2010                                 M_CALL(REG_ITMP1);
2011
2012                                 /* s1 may have been destroyed over the function call */
2013                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2014                                 M_TEST(REG_RESULT);
2015                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2016
2017                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2018                         }
2019
2020                         M_INTMOVE(s1, d);
2021                         emit_store_dst(jd, iptr, d);
2022                         break;
2023
2024                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2025
2026                         {
2027                         classinfo *super;
2028                         s4         superindex;
2029
2030                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2031                                 super      = NULL;
2032                                 superindex = 0;
2033                         }
2034                         else {
2035                                 super      = iptr->sx.s23.s3.c.cls;
2036                                 superindex = super->index;
2037                         }
2038
2039                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2040                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2041
2042                         if (s1 == d) {
2043                                 M_INTMOVE(s1, REG_ITMP1);
2044                                 s1 = REG_ITMP1;
2045                         }
2046
2047                         M_CLR(d);
2048
2049                         /* if class is not resolved, check which code to call */
2050
2051                         if (super == NULL) {
2052                                 M_TEST(s1);
2053                                 emit_label_beq(cd, BRANCH_LABEL_1);
2054
2055                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2056                                                                           iptr->sx.s23.s3.c.ref, 0);
2057
2058                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2059                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2060                                 emit_label_beq(cd, BRANCH_LABEL_2);
2061                         }
2062
2063                         /* interface instanceof code */
2064
2065                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2066                                 if (super != NULL) {
2067                                         M_TEST(s1);
2068                                         emit_label_beq(cd, BRANCH_LABEL_3);
2069                                 }
2070
2071                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2072
2073                                 if (super == NULL) {
2074                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2075                                                                                   iptr->sx.s23.s3.c.ref, 0);
2076                                 }
2077
2078                                 M_ILD32(REG_ITMP3,
2079                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2080                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2081
2082                                 int a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2083
2084                                 M_BLE(a);
2085                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2086                                                 OFFSET(vftbl_t, interfacetable[0]) -
2087                                                 superindex * sizeof(methodptr*));
2088                                 M_TEST(REG_ITMP1);
2089                                 M_SETNE(d);
2090
2091                                 if (super == NULL)
2092                                         emit_label_br(cd, BRANCH_LABEL_4);
2093                                 else
2094                                         emit_label(cd, BRANCH_LABEL_3);
2095                         }
2096
2097                         /* class instanceof code */
2098
2099                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2100                                 if (super == NULL) {
2101                                         emit_label(cd, BRANCH_LABEL_2);
2102
2103                                         constant_classref *cr = iptr->sx.s23.s3.c.ref;
2104                                         disp = dseg_add_unique_address(cd, cr);
2105
2106                                         patcher_add_patch_ref(jd,
2107                                                                                   PATCHER_resolve_classref_to_vftbl,
2108                                                                                   cr, disp);
2109                                 }
2110                                 else {
2111                                         M_TEST(s1);
2112                                         emit_label_beq(cd, BRANCH_LABEL_5);
2113
2114                                         disp = dseg_add_address(cd, super->vftbl);
2115                                 }
2116
2117                                 M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2118                                 M_ALD(REG_ITMP3, RIP, disp);
2119
2120                                 if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2121                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2122                                         M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
2123                                         emit_label_bne(cd, BRANCH_LABEL_8); /* jump over INC/SETE */
2124                                         if (d == REG_ITMP2) {
2125                                                 M_SETE(d);
2126                                                 M_BSEXT(d, d);
2127                                         } else
2128                                                 M_LINC(d);
2129                                         emit_label_br(cd, BRANCH_LABEL_6);  /* true */
2130                                         emit_label(cd, BRANCH_LABEL_8);
2131
2132                                         if (super == NULL) {
2133                                                 M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2134                                                 emit_label_bne(cd, BRANCH_LABEL_10);  /* false */
2135                                         }
2136
2137                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
2138                                         M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
2139                                         emit_label_bgt(cd, BRANCH_LABEL_9);  /* false */
2140
2141                                         M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2142                                         M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
2143                                         M_SETE(d);
2144                                         if (d == REG_ITMP2) {
2145                                                 M_BSEXT(d, d);
2146
2147                                                 emit_label_br(cd, BRANCH_LABEL_7); /* jump over M_CLR */
2148                                         }
2149
2150                                         emit_label(cd, BRANCH_LABEL_9);
2151                                         if (super == NULL)
2152                                                 emit_label(cd, BRANCH_LABEL_10);
2153                                         if (d == REG_ITMP2) {
2154                                                 M_CLR(d);
2155
2156                                                 emit_label(cd, BRANCH_LABEL_7);
2157                                         }
2158                                         emit_label(cd, BRANCH_LABEL_6);
2159                                 }
2160                                 else {
2161                                         M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
2162                                         M_SETE(d);
2163                                         if (d == REG_ITMP2)
2164                                                 M_BSEXT(d, d);
2165                                 }
2166
2167                                 if (super != NULL)
2168                                         emit_label(cd, BRANCH_LABEL_5);
2169                         }
2170
2171                         if (super == NULL) {
2172                                 emit_label(cd, BRANCH_LABEL_1);
2173                                 emit_label(cd, BRANCH_LABEL_4);
2174                         }
2175
2176                         emit_store_dst(jd, iptr, d);
2177                         }
2178                         break;
2179
2180                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2181
2182                         /* check for negative sizes and copy sizes to stack if necessary  */
2183
2184                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2185
2186                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2187
2188                                 /* copy SAVEDVAR sizes to stack */
2189                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2190
2191                                 /* Already Preallocated? */
2192                                 if (!(var->flags & PREALLOC)) {
2193                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2194                                         M_LST(s2, REG_SP, s1 * 8);
2195                                 }
2196                         }
2197
2198                         /* a0 = dimension count */
2199
2200                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2201
2202                         /* is a patcher function set? */
2203
2204                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2205                                 constant_classref *cr = iptr->sx.s23.s3.c.ref;
2206                                 disp = dseg_add_unique_address(cd, cr);
2207
2208                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2209                                                                           cr, disp);
2210                         }
2211                         else {
2212                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2213                         }
2214
2215                         /* a1 = classinfo */
2216
2217                         M_ALD(REG_A1, RIP, disp);
2218
2219                         /* a2 = pointer to dimensions = stack pointer */
2220
2221                         M_MOV(REG_SP, REG_A2);
2222
2223                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2224                         M_CALL(REG_ITMP1);
2225
2226                         /* check for exception before result assignment */
2227
2228                         emit_exception_check(cd, iptr);
2229
2230                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2231                         M_INTMOVE(REG_RESULT, s1);
2232                         emit_store_dst(jd, iptr, s1);
2233                         break;
2234
2235                 default:
2236                         vm_abort("Unknown ICMD %d during code generation", iptr->opc);
2237         } /* switch */
2238 }
2239
2240
2241 /* codegen_emit_stub_native ****************************************************
2242
2243    Emits a stub routine which calls a native method.
2244
2245 *******************************************************************************/
2246
2247 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2248 {
2249         methodinfo  *m;
2250         codeinfo    *code;
2251         codegendata *cd;
2252         methoddesc  *md;
2253         int          i, j;
2254         int          s1, s2;
2255         int          disp;
2256
2257         /* Sanity check. */
2258
2259         assert(f != NULL);
2260
2261         /* Get required compiler data. */
2262
2263         m    = jd->m;
2264         code = jd->code;
2265         cd   = jd->cd;
2266
2267         /* initialize variables */
2268
2269         md = m->parseddesc;
2270
2271         /* calculate stack frame size */
2272
2273         cd->stackframesize =
2274                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2275                 sizeof(localref_table) / SIZEOF_VOID_P +
2276                 md->paramcount +
2277                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2278                 nmd->memuse;
2279
2280         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2281
2282         /* create method header */
2283
2284         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2285         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2286         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2287         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2288         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2289
2290 #if defined(ENABLE_PROFILING)
2291         /* generate native method profiling code */
2292
2293         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2294                 /* count frequency */
2295
2296                 M_MOV_IMM(code, REG_ITMP3);
2297                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2298         }
2299 #endif
2300
2301         /* generate stub code */
2302
2303         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2304
2305 #if defined(ENABLE_GC_CACAO)
2306         /* Save callee saved integer registers in stackframeinfo (GC may
2307            need to recover them during a collection). */
2308
2309         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2310                 OFFSET(stackframeinfo_t, intregs);
2311
2312         for (i = 0; i < INT_SAV_CNT; i++)
2313                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2314 #endif
2315
2316         /* save integer and float argument registers */
2317
2318         for (i = 0; i < md->paramcount; i++) {
2319                 if (!md->params[i].inmemory) {
2320                         s1 = md->params[i].regoff;
2321
2322                         switch (md->paramtypes[i].type) {
2323                         case TYPE_INT:
2324                         case TYPE_LNG:
2325                         case TYPE_ADR:
2326                                 M_LST(s1, REG_SP, i * 8);
2327                                 break;
2328                         case TYPE_FLT:
2329                         case TYPE_DBL:
2330                                 M_DST(s1, REG_SP, i * 8);
2331                                 break;
2332                         }
2333                 }
2334         }
2335
2336         /* create dynamic stack info */
2337
2338         M_MOV(REG_SP, REG_A0);
2339         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2340         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2341         M_CALL(REG_ITMP1);
2342
2343         /* remember class argument */
2344
2345         if (m->flags & ACC_STATIC)
2346                 M_MOV(REG_RESULT, REG_ITMP2);
2347
2348         /* restore integer and float argument registers */
2349
2350         for (i = 0; i < md->paramcount; i++) {
2351                 if (!md->params[i].inmemory) {
2352                         s1 = md->params[i].regoff;
2353
2354                         switch (md->paramtypes[i].type) {
2355                         case TYPE_INT:
2356                         case TYPE_LNG:
2357                         case TYPE_ADR:
2358                                 M_LLD(s1, REG_SP, i * 8);
2359                                 break;
2360                         case TYPE_FLT:
2361                         case TYPE_DBL:
2362                                 M_DLD(s1, REG_SP, i * 8);
2363                                 break;
2364                         }
2365                 }
2366         }
2367
2368         /* Copy or spill arguments to new locations. */
2369
2370         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
2371                 s2 = nmd->params[j].regoff;
2372
2373                 switch (md->paramtypes[i].type) {
2374                 case TYPE_INT:
2375                 case TYPE_LNG:
2376                 case TYPE_ADR:
2377                         if (!md->params[i].inmemory) {
2378                                 s1 = md->params[i].regoff;
2379
2380                                 if (!nmd->params[j].inmemory)
2381                                         M_INTMOVE(s1, s2);
2382                                 else
2383                                         M_LST(s1, REG_SP, s2);
2384                         }
2385                         else {
2386                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2387                                 M_LLD(REG_ITMP1, REG_SP, s1);
2388                                 M_LST(REG_ITMP1, REG_SP, s2);
2389                         }
2390                         break;
2391                 case TYPE_FLT:
2392                         /* We only copy spilled float arguments, as the float
2393                            argument registers keep unchanged. */
2394
2395                         if (md->params[i].inmemory) {
2396                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2397
2398                                 M_FLD(REG_FTMP1, REG_SP, s1);
2399                                 M_FST(REG_FTMP1, REG_SP, s2);
2400                         }
2401                         break;
2402                 case TYPE_DBL:
2403                         if (md->params[i].inmemory) {
2404                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2405                                 M_DLD(REG_FTMP1, REG_SP, s1);
2406                                 M_DST(REG_FTMP1, REG_SP, s2);
2407                         }
2408                         break;
2409                 }
2410         }
2411
2412         /* Handle native Java methods. */
2413
2414         if (m->flags & ACC_NATIVE) {
2415                 /* put class into second argument register */
2416
2417                 if (m->flags & ACC_STATIC)
2418                         M_MOV(REG_ITMP2, REG_A1);
2419
2420                 /* put env into first argument register */
2421
2422                 M_MOV_IMM(VM_get_jnienv(), REG_A0);
2423         }
2424
2425         /* Call the native function. */
2426
2427         disp = dseg_add_functionptr(cd, f);
2428         M_ALD(REG_ITMP1, RIP, disp);
2429         M_CALL(REG_ITMP1);
2430
2431         /* save return value */
2432
2433         switch (md->returntype.type) {
2434         case TYPE_INT:
2435         case TYPE_LNG:
2436         case TYPE_ADR:
2437                 switch (md->returntype.primitivetype) {
2438                 case PRIMITIVETYPE_BOOLEAN:
2439                         M_BZEXT(REG_RESULT, REG_RESULT);
2440                         break;
2441                 case PRIMITIVETYPE_BYTE:
2442                         M_BSEXT(REG_RESULT, REG_RESULT);
2443                         break;
2444                 case PRIMITIVETYPE_CHAR:
2445                         M_CZEXT(REG_RESULT, REG_RESULT);
2446                         break;
2447                 case PRIMITIVETYPE_SHORT:
2448                         M_SSEXT(REG_RESULT, REG_RESULT);
2449                         break;
2450                 }
2451                 M_LST(REG_RESULT, REG_SP, 0 * 8);
2452                 break;
2453         case TYPE_FLT:
2454         case TYPE_DBL:
2455                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
2456                 break;
2457         case TYPE_VOID:
2458                 break;
2459         }
2460
2461         /* remove native stackframe info */
2462
2463         M_MOV(REG_SP, REG_A0);
2464         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2465         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
2466         M_CALL(REG_ITMP1);
2467         M_MOV(REG_RESULT, REG_ITMP3);
2468
2469         /* restore return value */
2470
2471         switch (md->returntype.type) {
2472         case TYPE_INT:
2473         case TYPE_LNG:
2474         case TYPE_ADR:
2475                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
2476                 break;
2477         case TYPE_FLT:
2478         case TYPE_DBL:
2479                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
2480                 break;
2481         case TYPE_VOID:
2482                 break;
2483         }
2484
2485 #if defined(ENABLE_GC_CACAO)
2486         /* Restore callee saved integer registers from stackframeinfo (GC
2487            might have modified them during a collection). */
2488
2489         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2490                 OFFSET(stackframeinfo_t, intregs);
2491
2492         for (i = 0; i < INT_SAV_CNT; i++)
2493                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2494 #endif
2495
2496         /* remove stackframe */
2497
2498         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2499
2500         /* test for exception */
2501
2502         M_TEST(REG_ITMP3);
2503         M_BNE(1);
2504         M_RET;
2505
2506         /* handle exception */
2507
2508         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
2509         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
2510         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
2511
2512         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
2513         M_JMP(REG_ITMP3);
2514 }
2515
2516
2517 /*
2518  * These are local overrides for various environment variables in Emacs.
2519  * Please do not remove this and leave it at the end of the file, where
2520  * Emacs will automagically detect them.
2521  * ---------------------------------------------------------------------
2522  * Local variables:
2523  * mode: c
2524  * indent-tabs-mode: t
2525  * c-basic-offset: 4
2526  * tab-width: 4
2527  * End:
2528  * vim:noexpandtab:sw=4:ts=4:
2529  */