src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007, 2008, 2009
   4    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
   5
   6    This file is part of CACAO.
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2, or (at
  11    your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful, but
  14    WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16    General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, write to the Free Software
  20    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  21    02110-1301, USA.
  22
  23 */
  24
  25
  26 #include "config.h"
  27
  28 #include <assert.h>
  29 #include <stdio.h>
  30 #include <stdint.h>
  31
  32 #include "vm/types.h"
  33 #include "vm/os.hpp"
  34
  35 #include "md-abi.h"
  36
  37 #include "vm/jit/x86_64/arch.h"
  38 #include "vm/jit/x86_64/codegen.h"
  39 #include "vm/jit/x86_64/emit.h"
  40
  41 #include "mm/memory.hpp"
  42
  43 #include "native/localref.hpp"
  44 #include "native/native.hpp"
  45
  46 #include "threads/lock.hpp"
  47
  48 #include "vm/jit/builtin.hpp"
  49 #include "vm/exceptions.hpp"
  50 #include "vm/global.h"
  51 #include "vm/loader.hpp"
  52 #include "vm/options.h"
  53 #include "vm/primitive.hpp"
  54 #include "vm/statistics.h"
  55 #include "vm/string.hpp"
  56 #include "vm/vm.hpp"
  57
  58 #include "vm/jit/abi.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/code.hpp"
  61 #include "vm/jit/codegen-common.hpp"
  62 #include "vm/jit/dseg.h"
  63 #include "vm/jit/emit-common.hpp"
  64 #include "vm/jit/jit.hpp"
  65 #include "vm/jit/linenumbertable.hpp"
  66 #include "vm/jit/methodheader.h"
  67 #include "vm/jit/parse.hpp"
  68 #include "vm/jit/patcher-common.hpp"
  69 #include "vm/jit/reg.h"
  70 #include "vm/jit/stacktrace.hpp"
  71 #include "vm/jit/trap.hpp"
  72
  73
  74 /**
  75  * Generates machine code for the method prolog.
  76  */
  77 void codegen_emit_prolog(jitdata* jd)
  78 {
  79         varinfo*    var;
  80         methoddesc* md;
  81         int32_t     s1;
  82         int32_t     p, t, l;
  83         int32_t     varindex;
  84         int         i;
  85
  86         // Get required compiler data.
  87         methodinfo*   m  = jd->m;
  88         codegendata*  cd = jd->cd;
  89         registerdata* rd = jd->rd;
  90
  91         /* create stack frame (if necessary) */
  92
  93         if (cd->stackframesize)
  94                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
  95
  96         /* save used callee saved registers */
  97
  98         p = cd->stackframesize;
  99         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 100                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 101         }
 102         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 103                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 104         }
 105
 106         /* take arguments out of register or stack frame */
 107
 108         md = m->parseddesc;
 109
 110         for (p = 0, l = 0; p < md->paramcount; p++) {
 111                 t = md->paramtypes[p].type;
 112
 113                 varindex = jd->local_map[l * 5 + t];
 114
 115                 l++;
 116                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 117                         l++;
 118
 119                 if (varindex == UNUSED)
 120                         continue;
 121
 122                 var = VAR(varindex);
 123
 124                 s1 = md->params[p].regoff;
 125
 126                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 127                         if (!md->params[p].inmemory) {           /* register arguments    */
 128                                 if (!IS_INMEMORY(var->flags))
 129                                         M_INTMOVE(s1, var->vv.regoff);
 130                                 else
 131                                     M_LST(s1, REG_SP, var->vv.regoff);
 132                         }
 133                         else {                                 /* stack arguments       */
 134                                 if (!IS_INMEMORY(var->flags))
 135                                         /* + 8 for return address */
 136                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 137                                 else
 138                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 139                         }
 140                 }
 141                 else {                                     /* floating args         */
 142                         if (!md->params[p].inmemory) {           /* register arguments    */
 143                                 if (!IS_INMEMORY(var->flags))
 144                                         emit_fmove(cd, s1, var->vv.regoff);
 145                                 else
 146                                         M_DST(s1, REG_SP, var->vv.regoff);
 147                         }
 148                         else {                                 /* stack arguments       */
 149                                 if (!IS_INMEMORY(var->flags))
 150                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 151                                 else
 152                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 153                         }
 154                 }
 155         }
 156 }
 157
 158
 159 /**
 160  * Generates machine code for the method epilog.
 161  */
 162 void codegen_emit_epilog(jitdata* jd)
 163 {
 164         int32_t p;
 165         int i;
 166
 167         // Get required compiler data.
 168         codegendata*  cd = jd->cd;
 169         registerdata* rd = jd->rd;
 170
 171         p = cd->stackframesize;
 172
 173         /* restore saved registers */
 174
 175         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 176                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
 177         }
 178         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 179                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
 180         }
 181
 182         /* deallocate stack */
 183
 184         if (cd->stackframesize)
 185                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
 186
 187         M_RET;
 188 }
 189
 190 /**
 191  * Generates a memory barrier to be used after volatile writes. It can be
 192  * patched out later if the field turns out not to be volatile.
 193  */
 194 void codegen_emit_patchable_barrier(instruction *iptr, codegendata *cd, patchref_t *pr, fieldinfo *fi)
 195 {
 196         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 197                 /* Align on word boundary */
 198                 if ((((intptr_t) cd->mcodeptr) & 3) >= 2)
 199                         emit_nop(cd, 4 - (((intptr_t) cd->mcodeptr) & 3));
 200                 /* Displacement for patching out MFENCE */
 201                 pr->disp_mb = (cd->mcodeptr - cd->mcodebase - pr->mpc);
 202         }
 203         if (INSTRUCTION_IS_UNRESOLVED(iptr) || fi->flags & ACC_VOLATILE)
 204                 M_MFENCE;
 205 }
 206
 207 /**
 208  * Generates machine code for one ICMD.
 209  */
 210 void codegen_emit_instruction(jitdata* jd, instruction* iptr)
 211 {
 212         varinfo*            var;
 213         varinfo*            dst;
 214         builtintable_entry* bte;
 215         methodinfo*         lm;             // Local methodinfo for ICMD_INVOKE*.
 216         unresolved_method*  um;
 217         fieldinfo*          fi;
 218         unresolved_field*   uf;
 219         patchref_t*         pr;
 220         int32_t             fieldtype;
 221         int32_t             s1, s2, s3, d;
 222         int32_t             disp;
 223
 224         // Get required compiler data.
 225         codegendata*  cd = jd->cd;
 226
 227         switch (iptr->opc) {
 228
 229                 /* constant operations ************************************************/
 230
 231                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 232
 233                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 234                         disp = dseg_add_float(cd, iptr->sx.val.f);
 235                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 236                         emit_store_dst(jd, iptr, d);
 237                         break;
 238
 239                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 240
 241                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 242                         disp = dseg_add_double(cd, iptr->sx.val.d);
 243                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 244                         emit_store_dst(jd, iptr, d);
 245                         break;
 246
 247                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 248
 249                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 250
 251                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 252                                 constant_classref *cr = iptr->sx.val.c.ref;
 253                                 disp = dseg_add_unique_address(cd, cr);
 254
 255 /*                              PROFILE_CYCLE_STOP; */
 256
 257                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 258                                                                           cr, disp);
 259
 260 /*                              PROFILE_CYCLE_START; */
 261
 262                                 M_ALD(d, RIP, disp);
 263                         }
 264                         else {
 265                                 if (iptr->sx.val.anyptr == 0) {
 266                                         M_CLR(d);
 267                                 }
 268                                 else {
 269                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 270                                         M_ALD(d, RIP, disp);
 271                                 }
 272                         }
 273                         emit_store_dst(jd, iptr, d);
 274                         break;
 275
 276
 277                 /* integer operations *************************************************/
 278
 279                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 280
 281                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 282                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 283                         M_INTMOVE(s1, d);
 284                         M_INEG(d);
 285                         emit_store_dst(jd, iptr, d);
 286                         break;
 287
 288                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 289
 290                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 291                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 292                         M_INTMOVE(s1, d);
 293                         M_LNEG(d);
 294                         emit_store_dst(jd, iptr, d);
 295                         break;
 296
 297                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 298
 299                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 300                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 301                         M_ISEXT(s1, d);
 302                         emit_store_dst(jd, iptr, d);
 303                         break;
 304
 305                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 306
 307                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 308                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 309                         M_IMOV(s1, d);
 310                         emit_store_dst(jd, iptr, d);
 311                         break;
 312
 313                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 314
 315                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 316                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 317                         M_BSEXT(s1, d);
 318                         emit_store_dst(jd, iptr, d);
 319                         break;
 320
 321                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 322
 323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 324                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 325                         M_CZEXT(s1, d);
 326                         emit_store_dst(jd, iptr, d);
 327                         break;
 328
 329                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 330
 331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 332                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 333                         M_SSEXT(s1, d);
 334                         emit_store_dst(jd, iptr, d);
 335                         break;
 336
 337
 338                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 339
 340                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 341                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 342                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 343                         if (s2 == d)
 344                                 M_IADD(s1, d);
 345                         else {
 346                                 M_INTMOVE(s1, d);
 347                                 M_IADD(s2, d);
 348                         }
 349                         emit_store_dst(jd, iptr, d);
 350                         break;
 351
 352                 case ICMD_IINC:
 353                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 354                                       /* sx.val.i = constant                             */
 355
 356                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 357                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 358
 359                         /* Using inc and dec is not faster than add (tested with
 360                            sieve). */
 361
 362                         M_INTMOVE(s1, d);
 363                         M_IADD_IMM(iptr->sx.val.i, d);
 364                         emit_store_dst(jd, iptr, d);
 365                         break;
 366
 367                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 368
 369                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 370                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 371                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 372                         if (s2 == d)
 373                                 M_LADD(s1, d);
 374                         else {
 375                                 M_INTMOVE(s1, d);
 376                                 M_LADD(s2, d);
 377                         }
 378                         emit_store_dst(jd, iptr, d);
 379                         break;
 380
 381                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 382                                       /* sx.val.l = constant                             */
 383
 384                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 385                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 386                         M_INTMOVE(s1, d);
 387                         if (IS_IMM32(iptr->sx.val.l))
 388                                 M_LADD_IMM(iptr->sx.val.l, d);
 389                         else {
 390                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 391                                 M_LADD(REG_ITMP2, d);
 392                         }
 393                         emit_store_dst(jd, iptr, d);
 394                         break;
 395
 396                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 397
 398                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 399                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 400                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 401                         if (s2 == d) {
 402                                 M_INTMOVE(s1, REG_ITMP1);
 403                                 M_ISUB(s2, REG_ITMP1);
 404                                 M_INTMOVE(REG_ITMP1, d);
 405                         } else {
 406                                 M_INTMOVE(s1, d);
 407                                 M_ISUB(s2, d);
 408                         }
 409                         emit_store_dst(jd, iptr, d);
 410                         break;
 411
 412                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 413                                       /* sx.val.i = constant                             */
 414
 415                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 416                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 417                         M_INTMOVE(s1, d);
 418                         M_ISUB_IMM(iptr->sx.val.i, d);
 419                         emit_store_dst(jd, iptr, d);
 420                         break;
 421
 422                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 423
 424                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 425                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 426                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 427                         if (s2 == d) {
 428                                 M_INTMOVE(s1, REG_ITMP1);
 429                                 M_LSUB(s2, REG_ITMP1);
 430                                 M_INTMOVE(REG_ITMP1, d);
 431                         } else {
 432                                 M_INTMOVE(s1, d);
 433                                 M_LSUB(s2, d);
 434                         }
 435                         emit_store_dst(jd, iptr, d);
 436                         break;
 437
 438                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 439                                       /* sx.val.l = constant                             */
 440
 441                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 442                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 443                         M_INTMOVE(s1, d);
 444                         if (IS_IMM32(iptr->sx.val.l))
 445                                 M_LSUB_IMM(iptr->sx.val.l, d);
 446                         else {
 447                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 448                                 M_LSUB(REG_ITMP2, d);
 449                         }
 450                         emit_store_dst(jd, iptr, d);
 451                         break;
 452
 453                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 454
 455                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 456                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 457                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 458                         if (s2 == d)
 459                                 M_IMUL(s1, d);
 460                         else {
 461                                 M_INTMOVE(s1, d);
 462                                 M_IMUL(s2, d);
 463                         }
 464                         emit_store_dst(jd, iptr, d);
 465                         break;
 466
 467                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 468                                       /* sx.val.i = constant                             */
 469
 470                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 471                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 472                         if (iptr->sx.val.i == 2) {
 473                                 M_INTMOVE(s1, d);
 474                                 M_ISLL_IMM(1, d);
 475                         } else
 476                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 477                         emit_store_dst(jd, iptr, d);
 478                         break;
 479
 480                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 481
 482                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 483                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 484                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 485                         if (s2 == d)
 486                                 M_LMUL(s1, d);
 487                         else {
 488                                 M_INTMOVE(s1, d);
 489                                 M_LMUL(s2, d);
 490                         }
 491                         emit_store_dst(jd, iptr, d);
 492                         break;
 493
 494                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 495                                       /* sx.val.l = constant                             */
 496
 497                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 498                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 499                         if (IS_IMM32(iptr->sx.val.l))
 500                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 501                         else {
 502                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 503                                 M_INTMOVE(s1, d);
 504                                 M_LMUL(REG_ITMP2, d);
 505                         }
 506                         emit_store_dst(jd, iptr, d);
 507                         break;
 508
 509                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 510
 511                         s1 = emit_load_s1(jd, iptr, RAX);
 512                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 513                         d = codegen_reg_of_dst(jd, iptr, RAX);
 514
 515                         M_INTMOVE(s1, RAX);
 516                         M_INTMOVE(s2, REG_ITMP3);
 517                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 518
 519                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 520
 521                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 522                         M_BNE(4 + 6);
 523                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 524                         M_BEQ(1 + 3);                                          /* 6 bytes */
 525
 526                         emit_cltd(cd);                                         /* 1 byte  */
 527                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 528
 529                         M_INTMOVE(RAX, d);
 530                         emit_store_dst(jd, iptr, d);
 531                         dst = VAROP(iptr->dst);
 532                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 533                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 534                         break;
 535
 536                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 537
 538                         s1 = emit_load_s1(jd, iptr, RAX);
 539                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 540                         d = codegen_reg_of_dst(jd, iptr, RDX);
 541
 542                         M_INTMOVE(s1, RAX);
 543                         M_INTMOVE(s2, REG_ITMP3);
 544                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 545
 546                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 547
 548                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 549                         M_BNE(3 + 4 + 6);
 550                         M_CLR(RDX);                                            /* 3 bytes */
 551                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 552                         M_BEQ(1 + 3);                                          /* 6 bytes */
 553
 554                         emit_cltd(cd);                                         /* 1 byte  */
 555                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 556
 557                         M_INTMOVE(RDX, d);
 558                         emit_store_dst(jd, iptr, d);
 559                         dst = VAROP(iptr->dst);
 560                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 561                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 562                         break;
 563
 564                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 565                                       /* sx.val.i = constant                             */
 566
 567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 568                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 569                         M_INTMOVE(s1, REG_ITMP1);
 570                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 571                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 572                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 573                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 574                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 575                         emit_store_dst(jd, iptr, d);
 576                         break;
 577
 578                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 579                                       /* sx.val.i = constant                             */
 580
 581                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 582                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 583                         M_INTMOVE(s1, REG_ITMP1);
 584                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 585                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 586                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 587                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 588                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 589                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 590                         emit_store_dst(jd, iptr, d);
 591                         break;
 592
 593
 594                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 595
 596                         s1 = emit_load_s1(jd, iptr, RAX);
 597                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 598                         d = codegen_reg_of_dst(jd, iptr, RAX);
 599
 600                         M_INTMOVE(s1, RAX);
 601                         M_INTMOVE(s2, REG_ITMP3);
 602                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 603
 604                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 605
 606                         /* check as described in jvm spec */
 607                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 608                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 609                         M_BNE(4 + 6);
 610                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 611                         M_BEQ(2 + 3);                                          /* 6 bytes */
 612
 613                         emit_cqto(cd);                                         /* 2 bytes */
 614                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 615
 616                         M_INTMOVE(RAX, d);
 617                         emit_store_dst(jd, iptr, d);
 618                         dst = VAROP(iptr->dst);
 619                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 620                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 621                         break;
 622
 623                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 624
 625                         s1 = emit_load_s1(jd, iptr, RAX);
 626                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 627                         d = codegen_reg_of_dst(jd, iptr, RDX);
 628
 629                         M_INTMOVE(s1, RAX);
 630                         M_INTMOVE(s2, REG_ITMP3);
 631                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 632
 633                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 634
 635                         /* check as described in jvm spec */
 636                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 637                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 638                         M_BNE(3 + 4 + 6);
 639                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 640                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 641                         M_BEQ(2 + 3);                                          /* 6 bytes */
 642
 643                         emit_cqto(cd);                                         /* 2 bytes */
 644                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 645
 646                         M_INTMOVE(RDX, d);
 647                         emit_store_dst(jd, iptr, d);
 648                         dst = VAROP(iptr->dst);
 649                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 650                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 651                         break;
 652
 653                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 654                                       /* sx.val.i = constant                             */
 655
 656                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 657                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 658                         M_INTMOVE(s1, REG_ITMP1);
 659                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 660                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 661                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 662                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 663                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 664                         emit_store_dst(jd, iptr, d);
 665                         break;
 666
 667                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 668                                       /* sx.val.l = constant                             */
 669
 670                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 671                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 672                         M_INTMOVE(s1, REG_ITMP1);
 673                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 674                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 675                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 676                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 677                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 678                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 679                         emit_store_dst(jd, iptr, d);
 680                         break;
 681
 682                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 683
 684                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 685                         emit_ishift(jd, SHIFT_SHL, iptr);
 686                         break;
 687
 688                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 689                                       /* sx.val.i = constant                             */
 690
 691                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 692                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 693                         M_INTMOVE(s1, d);
 694                         M_ISLL_IMM(iptr->sx.val.i, d);
 695                         emit_store_dst(jd, iptr, d);
 696                         break;
 697
 698                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 699
 700                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 701                         emit_ishift(jd, SHIFT_SAR, iptr);
 702                         break;
 703
 704                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 705                                       /* sx.val.i = constant                             */
 706
 707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 708                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 709                         M_INTMOVE(s1, d);
 710                         M_ISRA_IMM(iptr->sx.val.i, d);
 711                         emit_store_dst(jd, iptr, d);
 712                         break;
 713
 714                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 715
 716                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 717                         emit_ishift(jd, SHIFT_SHR, iptr);
 718                         break;
 719
 720                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 721                                       /* sx.val.i = constant                             */
 722
 723                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 724                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 725                         M_INTMOVE(s1, d);
 726                         M_ISRL_IMM(iptr->sx.val.i, d);
 727                         emit_store_dst(jd, iptr, d);
 728                         break;
 729
 730                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 731
 732                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 733                         emit_lshift(jd, SHIFT_SHL, iptr);
 734                         break;
 735
 736         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 737                                           /* sx.val.i = constant                             */
 738
 739                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 740                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 741                         M_INTMOVE(s1, d);
 742                         M_LSLL_IMM(iptr->sx.val.i, d);
 743                         emit_store_dst(jd, iptr, d);
 744                         break;
 745
 746                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 747
 748                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 749                         emit_lshift(jd, SHIFT_SAR, iptr);
 750                         break;
 751
 752                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 753                                       /* sx.val.i = constant                             */
 754
 755                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 756                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 757                         M_INTMOVE(s1, d);
 758                         M_LSRA_IMM(iptr->sx.val.i, d);
 759                         emit_store_dst(jd, iptr, d);
 760                         break;
 761
 762                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 763
 764                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 765                         emit_lshift(jd, SHIFT_SHR, iptr);
 766                         break;
 767
 768                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 769                                       /* sx.val.l = constant                             */
 770
 771                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 772                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 773                         M_INTMOVE(s1, d);
 774                         M_LSRL_IMM(iptr->sx.val.i, d);
 775                         emit_store_dst(jd, iptr, d);
 776                         break;
 777
 778                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
 779
 780                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 781                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 782                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 783                         if (s2 == d)
 784                                 M_IAND(s1, d);
 785                         else {
 786                                 M_INTMOVE(s1, d);
 787                                 M_IAND(s2, d);
 788                         }
 789                         emit_store_dst(jd, iptr, d);
 790                         break;
 791
 792                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
 793                                       /* sx.val.i = constant                             */
 794
 795                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 796                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 797                         M_INTMOVE(s1, d);
 798                         M_IAND_IMM(iptr->sx.val.i, d);
 799                         emit_store_dst(jd, iptr, d);
 800                         break;
 801
 802                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
 803
 804                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 805                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 806                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 807                         if (s2 == d)
 808                                 M_LAND(s1, d);
 809                         else {
 810                                 M_INTMOVE(s1, d);
 811                                 M_LAND(s2, d);
 812                         }
 813                         emit_store_dst(jd, iptr, d);
 814                         break;
 815
 816                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
 817                                       /* sx.val.l = constant                             */
 818
 819                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 820                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 821                         M_INTMOVE(s1, d);
 822                         if (IS_IMM32(iptr->sx.val.l))
 823                                 M_LAND_IMM(iptr->sx.val.l, d);
 824                         else {
 825                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 826                                 M_LAND(REG_ITMP2, d);
 827                         }
 828                         emit_store_dst(jd, iptr, d);
 829                         break;
 830
 831                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
 832
 833                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 834                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 835                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 836                         if (s2 == d)
 837                                 M_IOR(s1, d);
 838                         else {
 839                                 M_INTMOVE(s1, d);
 840                                 M_IOR(s2, d);
 841                         }
 842                         emit_store_dst(jd, iptr, d);
 843                         break;
 844
 845                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
 846                                       /* sx.val.i = constant                             */
 847
 848                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 849                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 850                         M_INTMOVE(s1, d);
 851                         M_IOR_IMM(iptr->sx.val.i, d);
 852                         emit_store_dst(jd, iptr, d);
 853                         break;
 854
 855                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
 856
 857                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 858                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 859                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 860                         if (s2 == d)
 861                                 M_LOR(s1, d);
 862                         else {
 863                                 M_INTMOVE(s1, d);
 864                                 M_LOR(s2, d);
 865                         }
 866                         emit_store_dst(jd, iptr, d);
 867                         break;
 868
 869                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
 870                                       /* sx.val.l = constant                             */
 871
 872                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 873                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 874                         M_INTMOVE(s1, d);
 875                         if (IS_IMM32(iptr->sx.val.l))
 876                                 M_LOR_IMM(iptr->sx.val.l, d);
 877                         else {
 878                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 879                                 M_LOR(REG_ITMP2, d);
 880                         }
 881                         emit_store_dst(jd, iptr, d);
 882                         break;
 883
 884                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
 885
 886                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 887                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 888                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 889                         if (s2 == d)
 890                                 M_IXOR(s1, d);
 891                         else {
 892                                 M_INTMOVE(s1, d);
 893                                 M_IXOR(s2, d);
 894                         }
 895                         emit_store_dst(jd, iptr, d);
 896                         break;
 897
 898                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
 899                                       /* sx.val.i = constant                             */
 900
 901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 902                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 903                         M_INTMOVE(s1, d);
 904                         M_IXOR_IMM(iptr->sx.val.i, d);
 905                         emit_store_dst(jd, iptr, d);
 906                         break;
 907
 908                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
 909
 910                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 911                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 912                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 913                         if (s2 == d)
 914                                 M_LXOR(s1, d);
 915                         else {
 916                                 M_INTMOVE(s1, d);
 917                                 M_LXOR(s2, d);
 918                         }
 919                         emit_store_dst(jd, iptr, d);
 920                         break;
 921
 922                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
 923                                       /* sx.val.l = constant                             */
 924
 925                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 926                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 927                         M_INTMOVE(s1, d);
 928                         if (IS_IMM32(iptr->sx.val.l))
 929                                 M_LXOR_IMM(iptr->sx.val.l, d);
 930                         else {
 931                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 932                                 M_LXOR(REG_ITMP2, d);
 933                         }
 934                         emit_store_dst(jd, iptr, d);
 935                         break;
 936
 937
 938                 /* floating operations ************************************************/
 939
 940                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
 941
 942                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 943                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 944                         disp = dseg_add_s4(cd, 0x80000000);
 945                         emit_fmove(cd, s1, d);
 946                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
 947                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
 948                         emit_store_dst(jd, iptr, d);
 949                         break;
 950
 951                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
 952
 953                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 954                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 955                         disp = dseg_add_s8(cd, 0x8000000000000000);
 956                         emit_fmove(cd, s1, d);
 957                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
 958                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
 959                         emit_store_dst(jd, iptr, d);
 960                         break;
 961
 962                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 963
 964                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 965                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 966                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 967                         if (s2 == d)
 968                                 M_FADD(s1, d);
 969                         else {
 970                                 emit_fmove(cd, s1, d);
 971                                 M_FADD(s2, d);
 972                         }
 973                         emit_store_dst(jd, iptr, d);
 974                         break;
 975
 976                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 977
 978                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 979                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 980                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 981                         if (s2 == d)
 982                                 M_DADD(s1, d);
 983                         else {
 984                                 emit_fmove(cd, s1, d);
 985                                 M_DADD(s2, d);
 986                         }
 987                         emit_store_dst(jd, iptr, d);
 988                         break;
 989
 990                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 991
 992                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 993                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 994                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 995                         if (s2 == d) {
 996                                 emit_fmove(cd, s2, REG_FTMP2);
 997                                 s2 = REG_FTMP2;
 998                         }
 999                         emit_fmove(cd, s1, d);
1000                         M_FSUB(s2, d);
1001                         emit_store_dst(jd, iptr, d);
1002                         break;
1003
1004                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1005
1006                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1007                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1008                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1009                         if (s2 == d) {
1010                                 emit_fmove(cd, s2, REG_FTMP2);
1011                                 s2 = REG_FTMP2;
1012                         }
1013                         emit_fmove(cd, s1, d);
1014                         M_DSUB(s2, d);
1015                         emit_store_dst(jd, iptr, d);
1016                         break;
1017
1018                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1019
1020                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1021                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1022                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1023                         if (s2 == d)
1024                                 M_FMUL(s1, d);
1025                         else {
1026                                 emit_fmove(cd, s1, d);
1027                                 M_FMUL(s2, d);
1028                         }
1029                         emit_store_dst(jd, iptr, d);
1030                         break;
1031
1032                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1033
1034                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1035                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1036                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1037                         if (s2 == d)
1038                                 M_DMUL(s1, d);
1039                         else {
1040                                 emit_fmove(cd, s1, d);
1041                                 M_DMUL(s2, d);
1042                         }
1043                         emit_store_dst(jd, iptr, d);
1044                         break;
1045
1046                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1047
1048                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1049                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1050                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1051                         if (s2 == d) {
1052                                 emit_fmove(cd, s2, REG_FTMP2);
1053                                 s2 = REG_FTMP2;
1054                         }
1055                         emit_fmove(cd, s1, d);
1056                         M_FDIV(s2, d);
1057                         emit_store_dst(jd, iptr, d);
1058                         break;
1059
1060                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1061
1062                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1063                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1064                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1065                         if (s2 == d) {
1066                                 emit_fmove(cd, s2, REG_FTMP2);
1067                                 s2 = REG_FTMP2;
1068                         }
1069                         emit_fmove(cd, s1, d);
1070                         M_DDIV(s2, d);
1071                         emit_store_dst(jd, iptr, d);
1072                         break;
1073
1074                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1075
1076                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1077                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1078                         M_CVTIF(s1, d);
1079                         emit_store_dst(jd, iptr, d);
1080                         break;
1081
1082                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1083
1084                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1085                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1086                         M_CVTID(s1, d);
1087                         emit_store_dst(jd, iptr, d);
1088                         break;
1089
1090                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1091
1092                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1093                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1094                         M_CVTLF(s1, d);
1095                         emit_store_dst(jd, iptr, d);
1096                         break;
1097
1098                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1099
1100                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1101                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1102                         M_CVTLD(s1, d);
1103                         emit_store_dst(jd, iptr, d);
1104                         break;
1105
1106                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1107
1108                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1109                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1110                         M_CVTFI(s1, d);
1111                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1112                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1113                                 ((REG_RESULT == d) ? 0 : 3);
1114                         M_BNE(disp);
1115                         emit_fmove(cd, s1, REG_FTMP1);
1116                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1117                         M_CALL(REG_ITMP2);
1118                         M_INTMOVE(REG_RESULT, d);
1119                         emit_store_dst(jd, iptr, d);
1120                         break;
1121
1122                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1123
1124                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1125                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1126                         M_CVTDI(s1, d);
1127                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1128                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1129                                 ((REG_RESULT == d) ? 0 : 3);
1130                         M_BNE(disp);
1131                         emit_fmove(cd, s1, REG_FTMP1);
1132                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1133                         M_CALL(REG_ITMP2);
1134                         M_INTMOVE(REG_RESULT, d);
1135                         emit_store_dst(jd, iptr, d);
1136                         break;
1137
1138                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1139
1140                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1141                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1142                         M_CVTFL(s1, d);
1143                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1144                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1145                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1146                                 ((REG_RESULT == d) ? 0 : 3);
1147                         M_BNE(disp);
1148                         emit_fmove(cd, s1, REG_FTMP1);
1149                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1150                         M_CALL(REG_ITMP2);
1151                         M_INTMOVE(REG_RESULT, d);
1152                         emit_store_dst(jd, iptr, d);
1153                         break;
1154
1155                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1156
1157                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1158                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1159                         M_CVTDL(s1, d);
1160                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1161                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1162                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1163                                 ((REG_RESULT == d) ? 0 : 3);
1164                         M_BNE(disp);
1165                         emit_fmove(cd, s1, REG_FTMP1);
1166                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1167                         M_CALL(REG_ITMP2);
1168                         M_INTMOVE(REG_RESULT, d);
1169                         emit_store_dst(jd, iptr, d);
1170                         break;
1171
1172                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1173
1174                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1175                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1176                         M_CVTFD(s1, d);
1177                         emit_store_dst(jd, iptr, d);
1178                         break;
1179
1180                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1181
1182                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1183                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1184                         M_CVTDF(s1, d);
1185                         emit_store_dst(jd, iptr, d);
1186                         break;
1187
1188                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1189                                           /* == => 0, < => 1, > => -1 */
1190
1191                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1192                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1193                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1194                         M_CLR(d);
1195                         M_MOV_IMM(1, REG_ITMP1);
1196                         M_MOV_IMM(-1, REG_ITMP2);
1197                         emit_ucomiss_reg_reg(cd, s1, s2);
1198                         M_CMOVULT(REG_ITMP1, d);
1199                         M_CMOVUGT(REG_ITMP2, d);
1200                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1201                         emit_store_dst(jd, iptr, d);
1202                         break;
1203
1204                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1205                                           /* == => 0, < => 1, > => -1 */
1206
1207                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1208                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1209                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1210                         M_CLR(d);
1211                         M_MOV_IMM(1, REG_ITMP1);
1212                         M_MOV_IMM(-1, REG_ITMP2);
1213                         emit_ucomiss_reg_reg(cd, s1, s2);
1214                         M_CMOVULT(REG_ITMP1, d);
1215                         M_CMOVUGT(REG_ITMP2, d);
1216                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1217                         emit_store_dst(jd, iptr, d);
1218                         break;
1219
1220                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1221                                           /* == => 0, < => 1, > => -1 */
1222
1223                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1224                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1225                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1226                         M_CLR(d);
1227                         M_MOV_IMM(1, REG_ITMP1);
1228                         M_MOV_IMM(-1, REG_ITMP2);
1229                         emit_ucomisd_reg_reg(cd, s1, s2);
1230                         M_CMOVULT(REG_ITMP1, d);
1231                         M_CMOVUGT(REG_ITMP2, d);
1232                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1233                         emit_store_dst(jd, iptr, d);
1234                         break;
1235
1236                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1237                                           /* == => 0, < => 1, > => -1 */
1238
1239                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1240                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1241                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1242                         M_CLR(d);
1243                         M_MOV_IMM(1, REG_ITMP1);
1244                         M_MOV_IMM(-1, REG_ITMP2);
1245                         emit_ucomisd_reg_reg(cd, s1, s2);
1246                         M_CMOVULT(REG_ITMP1, d);
1247                         M_CMOVUGT(REG_ITMP2, d);
1248                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1249                         emit_store_dst(jd, iptr, d);
1250                         break;
1251
1252
1253                 /* memory operations **************************************************/
1254
1255                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1256
1257                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1258                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1259                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1260                         /* implicit null-pointer check */
1261                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1262                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1263                         emit_store_dst(jd, iptr, d);
1264                         break;
1265
1266                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1267
1268                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1269                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1270                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1271                         /* implicit null-pointer check */
1272                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1273                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1274                         emit_store_dst(jd, iptr, d);
1275                         break;
1276
1277                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1278
1279                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1280                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1281                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1282                         /* implicit null-pointer check */
1283                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1284                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1285                         emit_store_dst(jd, iptr, d);
1286                         break;
1287
1288                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1289
1290                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1291                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1292                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1293                         /* implicit null-pointer check */
1294                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1295                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1296                         emit_store_dst(jd, iptr, d);
1297                         break;
1298
1299                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1300
1301                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1302                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1303                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1304                         /* implicit null-pointer check */
1305                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1306                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1307                         emit_store_dst(jd, iptr, d);
1308                         break;
1309
1310                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1311
1312                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1313                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1314                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1315                         /* implicit null-pointer check */
1316                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1317                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1318                         emit_store_dst(jd, iptr, d);
1319                         break;
1320
1321                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1322
1323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1324                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1325                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1326                         /* implicit null-pointer check */
1327                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1328                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1329                         emit_store_dst(jd, iptr, d);
1330                         break;
1331
1332                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1333
1334                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1335                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1336                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1337                         /* implicit null-pointer check */
1338                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1339                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1340                         emit_store_dst(jd, iptr, d);
1341                         break;
1342
1343
1344                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1345
1346                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1347                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1348                         /* implicit null-pointer check */
1349                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1350                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1351                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1352                         break;
1353
1354                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1355
1356                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1357                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1358                         /* implicit null-pointer check */
1359                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1360                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1361                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1362                         break;
1363
1364                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1365
1366                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1367                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1368                         /* implicit null-pointer check */
1369                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1370                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1371                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1372                         break;
1373
1374                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1375
1376                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1377                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1378                         /* implicit null-pointer check */
1379                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1380                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1381                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1382                         break;
1383
1384                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1385
1386                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1387                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1388                         /* implicit null-pointer check */
1389                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1390                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1391                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1392                         break;
1393
1394                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1395
1396                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1397                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1398                         /* implicit null-pointer check */
1399                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1400                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1401                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1402                         break;
1403
1404                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1405
1406                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1407                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1408                         /* implicit null-pointer check */
1409                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1410                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1411                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1412                         break;
1413
1414                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1415
1416                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1417                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1418                         /* implicit null-pointer check */
1419                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1420                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1421
1422                         M_MOV(s1, REG_A0);
1423                         M_MOV(s3, REG_A1);
1424                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1425                         M_CALL(REG_ITMP1);
1426                         emit_arraystore_check(cd, iptr);
1427
1428                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1429                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1430                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1431                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1432                         break;
1433
1434
1435                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1436
1437                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1438                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1439                         /* implicit null-pointer check */
1440                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1441                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1442                         break;
1443
1444                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1445
1446                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1447                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1448                         /* implicit null-pointer check */
1449                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1450                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1451                         break;
1452
1453                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1454
1455                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1456                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1457                         /* implicit null-pointer check */
1458                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1459                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1460                         break;
1461
1462                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1463
1464                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1465                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1466                         /* implicit null-pointer check */
1467                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1468                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1469                         break;
1470
1471                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1472
1473                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1474                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1475                         /* implicit null-pointer check */
1476                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1477
1478                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1479                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1480                         }
1481                         else {
1482                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1483                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1484                         }
1485                         break;
1486
1487                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1488
1489                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1490                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1491                         /* implicit null-pointer check */
1492                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1493                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1494                         break;
1495
1496                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1497                                           /* val = value (in current instruction)     */
1498                                           /* following NOP)                           */
1499
1500                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1501                                 uf        = iptr->sx.s23.s3.uf;
1502                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1503                                 disp      = dseg_add_unique_address(cd, uf);
1504
1505 /*                              PROFILE_CYCLE_STOP; */
1506
1507                                 pr = patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1508
1509 /*                              PROFILE_CYCLE_START; */
1510                         }
1511                         else {
1512                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1513                                 fieldtype = fi->type;
1514                                 disp      = dseg_add_address(cd, fi->value);
1515
1516                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1517                                         //PROFILE_CYCLE_STOP;
1518
1519                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1520                                                                                   fi->clazz, 0);
1521
1522                                         //PROFILE_CYCLE_START;
1523                                 }
1524                         }
1525
1526                         /* This approach is much faster than moving the field
1527                            address inline into a register. */
1528
1529                         M_ALD(REG_ITMP1, RIP, disp);
1530
1531                         switch (fieldtype) {
1532                         case TYPE_INT:
1533                         case TYPE_FLT:
1534                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1535                                 break;
1536                         case TYPE_LNG:
1537                         case TYPE_ADR:
1538                         case TYPE_DBL:
1539                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1540                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1541                                 else {
1542                                         M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1543                                         M_LST(REG_ITMP2, REG_ITMP1, 0);
1544                                 }
1545                                 break;
1546                         }
1547                         codegen_emit_patchable_barrier(iptr, cd, pr, fi);
1548                         break;
1549
1550                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1551
1552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1553
1554                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1555                                 uf        = iptr->sx.s23.s3.uf;
1556                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1557                                 disp      = 0;
1558
1559 /*                              PROFILE_CYCLE_STOP; */
1560
1561                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1562
1563 /*                              PROFILE_CYCLE_START; */
1564                         }
1565                         else {
1566                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1567                                 fieldtype = fi->type;
1568                                 disp      = fi->offset;
1569                         }
1570
1571                         /* implicit null-pointer check */
1572                         switch (fieldtype) {
1573                         case TYPE_INT:
1574                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1575                                 M_ILD32(d, s1, disp);
1576                                 break;
1577                         case TYPE_LNG:
1578                         case TYPE_ADR:
1579                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1580                                 M_LLD32(d, s1, disp);
1581                                 break;
1582                         case TYPE_FLT:
1583                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1584                                 M_FLD32(d, s1, disp);
1585                                 break;
1586                         case TYPE_DBL:
1587                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1588                                 M_DLD32(d, s1, disp);
1589                                 break;
1590                         }
1591                         emit_store_dst(jd, iptr, d);
1592                         break;
1593
1594                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1595
1596                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1597                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1598
1599                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1600                                 uf        = iptr->sx.s23.s3.uf;
1601                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1602                                 disp      = 0;
1603
1604 /*                              PROFILE_CYCLE_STOP; */
1605
1606                                 pr = patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1607
1608 /*                              PROFILE_CYCLE_START; */
1609                         }
1610                         else {
1611                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1612                                 fieldtype = fi->type;
1613                                 disp      = fi->offset;
1614                         }
1615
1616                         /* implicit null-pointer check */
1617                         switch (fieldtype) {
1618                         case TYPE_INT:
1619                                 M_IST32(s2, s1, disp);
1620                                 break;
1621                         case TYPE_LNG:
1622                         case TYPE_ADR:
1623                                 M_LST32(s2, s1, disp);
1624                                 break;
1625                         case TYPE_FLT:
1626                                 M_FST32(s2, s1, disp);
1627                                 break;
1628                         case TYPE_DBL:
1629                                 M_DST32(s2, s1, disp);
1630                                 break;
1631                         }
1632                         codegen_emit_patchable_barrier(iptr, cd, pr, fi);
1633                         break;
1634
1635                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1636                                           /* val = value (in current instruction)     */
1637                                           /* following NOP)                           */
1638
1639                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1640
1641                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1642                                 uf        = iptr->sx.s23.s3.uf;
1643                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1644                                 disp      = 0;
1645
1646 /*                              PROFILE_CYCLE_STOP; */
1647
1648                                 pr = patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1649
1650 /*                              PROFILE_CYCLE_START; */
1651                         }
1652                         else {
1653                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1654                                 fieldtype = fi->type;
1655                                 disp      = fi->offset;
1656                         }
1657
1658                         /* implicit null-pointer check */
1659                         switch (fieldtype) {
1660                         case TYPE_INT:
1661                         case TYPE_FLT:
1662                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
1663                                 break;
1664                         case TYPE_LNG:
1665                         case TYPE_ADR:
1666                         case TYPE_DBL:
1667                                 /* XXX why no check for IS_IMM32? -- probably because of the patcher */
1668                                 M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1669                                 if (disp)  /* resolved, disp can never be 0 */
1670                                         M_LST(REG_ITMP2, s1, disp);
1671                                 else       /* unresolved */
1672                                         M_LST32(REG_ITMP2, s1, disp);
1673                                 break;
1674                         }
1675                         codegen_emit_patchable_barrier(iptr, cd, pr, fi);
1676                         break;
1677
1678
1679                 /* branch operations **************************************************/
1680
1681                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
1682
1683                         M_CALL_IMM(0);                            /* passing exception pc */
1684                         M_POP(REG_ITMP2_XPC);
1685
1686                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
1687                         M_JMP(REG_ITMP3);
1688                         break;
1689
1690                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
1691                 case ICMD_IF_LNE:
1692                 case ICMD_IF_LLT:
1693                 case ICMD_IF_LGE:
1694                 case ICMD_IF_LGT:
1695                 case ICMD_IF_LLE:
1696
1697                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1698                         if (IS_IMM32(iptr->sx.val.l))
1699                                 M_LCMP_IMM(iptr->sx.val.l, s1);
1700                         else {
1701                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1702                                 M_LCMP(REG_ITMP2, s1);
1703                         }
1704                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
1705                         break;
1706
1707                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
1708                 case ICMD_IF_LCMPNE:
1709                 case ICMD_IF_LCMPLT:
1710                 case ICMD_IF_LCMPGE:
1711                 case ICMD_IF_LCMPGT:
1712                 case ICMD_IF_LCMPLE:
1713
1714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1715                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1716                         M_LCMP(s2, s1);
1717                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
1718                         break;
1719
1720                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
1721                         {
1722                                 s4 i, l;
1723                                 branch_target_t *table;
1724
1725                                 table = iptr->dst.table;
1726
1727                                 l = iptr->sx.s23.s2.tablelow;
1728                                 i = iptr->sx.s23.s3.tablehigh;
1729
1730                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1731                                 M_INTMOVE(s1, REG_ITMP1);
1732
1733                                 if (l != 0)
1734                                         M_ISUB_IMM(l, REG_ITMP1);
1735
1736                                 /* number of targets */
1737                                 i = i - l + 1;
1738
1739                 /* range check */
1740
1741                                 M_ICMP_IMM(i - 1, REG_ITMP1);
1742                                 emit_bugt(cd, table[0].block);
1743
1744                                 /* build jump table top down and use address of lowest entry */
1745
1746                                 table += i;
1747
1748                                 while (--i >= 0) {
1749                                         dseg_add_target(cd, table->block);
1750                                         --table;
1751                                 }
1752
1753                                 /* length of dataseg after last dseg_add_target is used
1754                                    by load */
1755
1756                                 M_MOV_IMM(0, REG_ITMP2);
1757                                 dseg_adddata(cd);
1758                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
1759                                 M_JMP(REG_ITMP1);
1760                         }
1761                         break;
1762
1763                 case ICMD_BUILTIN:
1764                         bte = iptr->sx.s23.s3.bte;
1765                         if (bte->stub == NULL) {
1766                                 M_MOV_IMM(bte->fp, REG_ITMP1);
1767                         }
1768                         else {
1769                                 M_MOV_IMM(bte->stub, REG_ITMP1);
1770                         }
1771                         M_CALL(REG_ITMP1);
1772                         break;
1773
1774                 case ICMD_INVOKESPECIAL:
1775                         emit_nullpointer_check(cd, iptr, REG_A0);
1776                         /* fall through */
1777
1778                 case ICMD_INVOKESTATIC:
1779                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1780                                 um = iptr->sx.s23.s3.um;
1781                                 disp = dseg_add_unique_address(cd, um);
1782
1783                                 patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
1784                                                                           um, disp);
1785                         }
1786                         else {
1787                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1788                                 disp = dseg_add_functionptr(cd, lm->stubroutine);
1789                         }
1790
1791                         M_ALD(REG_ITMP2, RIP, disp);
1792                         M_CALL(REG_ITMP2);
1793                         break;
1794
1795                 case ICMD_INVOKEVIRTUAL:
1796                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1797                                 um = iptr->sx.s23.s3.um;
1798                                 patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
1799
1800                                 s1 = 0;
1801                         }
1802                         else {
1803                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1804                                 s1 = OFFSET(vftbl_t, table[0]) +
1805                                         sizeof(methodptr) * lm->vftblindex;
1806                         }
1807
1808                         /* implicit null-pointer check */
1809                         M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
1810                         M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
1811                         M_CALL(REG_ITMP3);
1812                         break;
1813
1814                 case ICMD_INVOKEINTERFACE:
1815                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1816                                 um = iptr->sx.s23.s3.um;
1817                                 patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
1818
1819                                 s1 = 0;
1820                                 s2 = 0;
1821                         }
1822                         else {
1823                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1824                                 s1 = OFFSET(vftbl_t, interfacetable[0]) -
1825                                         sizeof(methodptr) * lm->clazz->index;
1826
1827                                 s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
1828                         }
1829
1830                         /* implicit null-pointer check */
1831                         M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
1832                         M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
1833                         M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
1834                         M_CALL(REG_ITMP3);
1835                         break;
1836
1837                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
1838
1839                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
1840                                 /* object type cast-check */
1841
1842                                 classinfo *super;
1843                                 s4         superindex;
1844
1845                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1846                                         super      = NULL;
1847                                         superindex = 0;
1848                                 }
1849                                 else {
1850                                         super      = iptr->sx.s23.s3.c.cls;
1851                                         superindex = super->index;
1852                                 }
1853
1854                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1855
1856                                 /* if class is not resolved, check which code to call */
1857
1858                                 if (super == NULL) {
1859                                         M_TEST(s1);
1860                                         emit_label_beq(cd, BRANCH_LABEL_1);
1861
1862                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
1863                                                                                   iptr->sx.s23.s3.c.ref, 0);
1864
1865                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
1866                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
1867                                         emit_label_beq(cd, BRANCH_LABEL_2);
1868                                 }
1869
1870                                 /* interface checkcast code */
1871
1872                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
1873                                         if (super != NULL) {
1874                                                 M_TEST(s1);
1875                                                 emit_label_beq(cd, BRANCH_LABEL_3);
1876                                         }
1877
1878                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
1879
1880                                         if (super == NULL) {
1881                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
1882                                                                                           iptr->sx.s23.s3.c.ref,
1883                                                                                           0);
1884                                         }
1885
1886                                         M_ILD32(REG_ITMP3,
1887                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
1888                                         M_ICMP_IMM32(superindex, REG_ITMP3);
1889                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
1890
1891                                         M_ALD32(REG_ITMP3, REG_ITMP2,
1892                                                         OFFSET(vftbl_t, interfacetable[0]) -
1893                                                         superindex * sizeof(methodptr*));
1894                                         M_TEST(REG_ITMP3);
1895                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
1896
1897                                         if (super == NULL)
1898                                                 emit_label_br(cd, BRANCH_LABEL_4);
1899                                         else
1900                                                 emit_label(cd, BRANCH_LABEL_3);
1901                                 }
1902
1903                                 /* class checkcast code */
1904
1905                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
1906                                         if (super == NULL) {
1907                                                 emit_label(cd, BRANCH_LABEL_2);
1908
1909                                                 constant_classref *cr = iptr->sx.s23.s3.c.ref;
1910                                                 disp = dseg_add_unique_address(cd, cr);
1911
1912                                                 patcher_add_patch_ref(jd,
1913                                                                                           PATCHER_resolve_classref_to_vftbl,
1914                                                                                           cr, disp);
1915                                         }
1916                                         else {
1917                                                 M_TEST(s1);
1918                                                 emit_label_beq(cd, BRANCH_LABEL_5);
1919
1920                                                 disp = dseg_add_address(cd, super->vftbl);
1921                                         }
1922
1923                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
1924                                         M_ALD(REG_ITMP3, RIP, disp);
1925
1926                                         if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
1927                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
1928                                                 M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
1929                                                 emit_label_beq(cd, BRANCH_LABEL_6);  /* good */
1930
1931                                                 if (super == NULL) {
1932                                                         M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
1933                                                         emit_label_bne(cd, BRANCH_LABEL_10);  /* throw */
1934                                                 }
1935
1936                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
1937                                                 M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
1938                                                 emit_label_bgt(cd, BRANCH_LABEL_9);  /* throw */
1939
1940                                                 M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
1941                                                 M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
1942                                                 emit_label_beq(cd, BRANCH_LABEL_7);  /* good */
1943
1944                                                 emit_label(cd, BRANCH_LABEL_9);
1945                                                 if (super == NULL)
1946                                                         emit_label(cd, BRANCH_LABEL_10);
1947
1948                                                 /* reload s1, might have been destroyed */
1949                                                 emit_load_s1(jd, iptr, REG_ITMP1);
1950                                                 M_ALD_MEM(s1, TRAP_ClassCastException);
1951
1952                                                 emit_label(cd, BRANCH_LABEL_7);
1953                                                 emit_label(cd, BRANCH_LABEL_6);
1954                                                 /* reload s1, might have been destroyed */
1955                                                 emit_load_s1(jd, iptr, REG_ITMP1);
1956                                         }
1957                                         else {
1958                                                 M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
1959                                                 emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
1960                                         }
1961
1962                                         if (super != NULL)
1963                                                 emit_label(cd, BRANCH_LABEL_5);
1964                                 }
1965
1966                                 if (super == NULL) {
1967                                         emit_label(cd, BRANCH_LABEL_1);
1968                                         emit_label(cd, BRANCH_LABEL_4);
1969                                 }
1970
1971                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1972                         }
1973                         else {
1974                                 /* array type cast-check */
1975
1976                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1977                                 M_INTMOVE(s1, REG_A0);
1978
1979                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1980                                         constant_classref *cr = iptr->sx.s23.s3.c.ref;
1981                                         disp = dseg_add_unique_address(cd, cr);
1982
1983                                         patcher_add_patch_ref(jd,
1984                                                                                   PATCHER_resolve_classref_to_classinfo,
1985                                                                                   cr, disp);
1986                                 }
1987                                 else {
1988                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
1989                                 }
1990
1991                                 M_ALD(REG_A1, RIP, disp);
1992                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
1993                                 M_CALL(REG_ITMP1);
1994
1995                                 /* s1 may have been destroyed over the function call */
1996                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1997                                 M_TEST(REG_RESULT);
1998                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
1999
2000                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2001                         }
2002
2003                         M_INTMOVE(s1, d);
2004                         emit_store_dst(jd, iptr, d);
2005                         break;
2006
2007                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2008
2009                         {
2010                         classinfo *super;
2011                         s4         superindex;
2012
2013                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2014                                 super      = NULL;
2015                                 superindex = 0;
2016                         }
2017                         else {
2018                                 super      = iptr->sx.s23.s3.c.cls;
2019                                 superindex = super->index;
2020                         }
2021
2022                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2023                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2024
2025                         if (s1 == d) {
2026                                 M_INTMOVE(s1, REG_ITMP1);
2027                                 s1 = REG_ITMP1;
2028                         }
2029
2030                         M_CLR(d);
2031
2032                         /* if class is not resolved, check which code to call */
2033
2034                         if (super == NULL) {
2035                                 M_TEST(s1);
2036                                 emit_label_beq(cd, BRANCH_LABEL_1);
2037
2038                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2039                                                                           iptr->sx.s23.s3.c.ref, 0);
2040
2041                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2042                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2043                                 emit_label_beq(cd, BRANCH_LABEL_2);
2044                         }
2045
2046                         /* interface instanceof code */
2047
2048                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2049                                 if (super != NULL) {
2050                                         M_TEST(s1);
2051                                         emit_label_beq(cd, BRANCH_LABEL_3);
2052                                 }
2053
2054                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2055
2056                                 if (super == NULL) {
2057                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2058                                                                                   iptr->sx.s23.s3.c.ref, 0);
2059                                 }
2060
2061                                 M_ILD32(REG_ITMP3,
2062                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2063                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2064
2065                                 int a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2066
2067                                 M_BLE(a);
2068                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2069                                                 OFFSET(vftbl_t, interfacetable[0]) -
2070                                                 superindex * sizeof(methodptr*));
2071                                 M_TEST(REG_ITMP1);
2072                                 M_SETNE(d);
2073
2074                                 if (super == NULL)
2075                                         emit_label_br(cd, BRANCH_LABEL_4);
2076                                 else
2077                                         emit_label(cd, BRANCH_LABEL_3);
2078                         }
2079
2080                         /* class instanceof code */
2081
2082                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2083                                 if (super == NULL) {
2084                                         emit_label(cd, BRANCH_LABEL_2);
2085
2086                                         constant_classref *cr = iptr->sx.s23.s3.c.ref;
2087                                         disp = dseg_add_unique_address(cd, cr);
2088
2089                                         patcher_add_patch_ref(jd,
2090                                                                                   PATCHER_resolve_classref_to_vftbl,
2091                                                                                   cr, disp);
2092                                 }
2093                                 else {
2094                                         M_TEST(s1);
2095                                         emit_label_beq(cd, BRANCH_LABEL_5);
2096
2097                                         disp = dseg_add_address(cd, super->vftbl);
2098                                 }
2099
2100                                 M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2101                                 M_ALD(REG_ITMP3, RIP, disp);
2102
2103                                 if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2104                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2105                                         M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
2106                                         emit_label_bne(cd, BRANCH_LABEL_8); /* jump over INC/SETE */
2107                                         if (d == REG_ITMP2) {
2108                                                 M_SETE(d);
2109                                                 M_BSEXT(d, d);
2110                                         } else
2111                                                 M_LINC(d);
2112                                         emit_label_br(cd, BRANCH_LABEL_6);  /* true */
2113                                         emit_label(cd, BRANCH_LABEL_8);
2114
2115                                         if (super == NULL) {
2116                                                 M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2117                                                 emit_label_bne(cd, BRANCH_LABEL_10);  /* false */
2118                                         }
2119
2120                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
2121                                         M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
2122                                         emit_label_bgt(cd, BRANCH_LABEL_9);  /* false */
2123
2124                                         M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2125                                         M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
2126                                         M_SETE(d);
2127                                         if (d == REG_ITMP2) {
2128                                                 M_BSEXT(d, d);
2129
2130                                                 emit_label_br(cd, BRANCH_LABEL_7); /* jump over M_CLR */
2131                                         }
2132
2133                                         emit_label(cd, BRANCH_LABEL_9);
2134                                         if (super == NULL)
2135                                                 emit_label(cd, BRANCH_LABEL_10);
2136                                         if (d == REG_ITMP2) {
2137                                                 M_CLR(d);
2138
2139                                                 emit_label(cd, BRANCH_LABEL_7);
2140                                         }
2141                                         emit_label(cd, BRANCH_LABEL_6);
2142                                 }
2143                                 else {
2144                                         M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
2145                                         M_SETE(d);
2146                                         if (d == REG_ITMP2)
2147                                                 M_BSEXT(d, d);
2148                                 }
2149
2150                                 if (super != NULL)
2151                                         emit_label(cd, BRANCH_LABEL_5);
2152                         }
2153
2154                         if (super == NULL) {
2155                                 emit_label(cd, BRANCH_LABEL_1);
2156                                 emit_label(cd, BRANCH_LABEL_4);
2157                         }
2158
2159                         emit_store_dst(jd, iptr, d);
2160                         }
2161                         break;
2162
2163                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2164
2165                         /* check for negative sizes and copy sizes to stack if necessary  */
2166
2167                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2168
2169                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2170
2171                                 /* copy SAVEDVAR sizes to stack */
2172                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2173
2174                                 /* Already Preallocated? */
2175                                 if (!(var->flags & PREALLOC)) {
2176                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2177                                         M_LST(s2, REG_SP, s1 * 8);
2178                                 }
2179                         }
2180
2181                         /* a0 = dimension count */
2182
2183                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2184
2185                         /* is a patcher function set? */
2186
2187                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2188                                 constant_classref *cr = iptr->sx.s23.s3.c.ref;
2189                                 disp = dseg_add_unique_address(cd, cr);
2190
2191                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2192                                                                           cr, disp);
2193                         }
2194                         else {
2195                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2196                         }
2197
2198                         /* a1 = classinfo */
2199
2200                         M_ALD(REG_A1, RIP, disp);
2201
2202                         /* a2 = pointer to dimensions = stack pointer */
2203
2204                         M_MOV(REG_SP, REG_A2);
2205
2206                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2207                         M_CALL(REG_ITMP1);
2208
2209                         /* check for exception before result assignment */
2210
2211                         emit_exception_check(cd, iptr);
2212
2213                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2214                         M_INTMOVE(REG_RESULT, s1);
2215                         emit_store_dst(jd, iptr, s1);
2216                         break;
2217
2218                 default:
2219                         vm_abort("Unknown ICMD %d during code generation", iptr->opc);
2220         } /* switch */
2221 }
2222
2223
2224 /* codegen_emit_stub_native ****************************************************
2225
2226    Emits a stub routine which calls a native method.
2227
2228 *******************************************************************************/
2229
2230 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2231 {
2232         methodinfo  *m;
2233         codeinfo    *code;
2234         codegendata *cd;
2235         methoddesc  *md;
2236         int          i, j;
2237         int          s1, s2;
2238         int          disp;
2239
2240         /* Sanity check. */
2241
2242         assert(f != NULL);
2243
2244         /* Get required compiler data. */
2245
2246         m    = jd->m;
2247         code = jd->code;
2248         cd   = jd->cd;
2249
2250         /* initialize variables */
2251
2252         md = m->parseddesc;
2253
2254         /* calculate stack frame size */
2255
2256         cd->stackframesize =
2257                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2258                 sizeof(localref_table) / SIZEOF_VOID_P +
2259                 md->paramcount +
2260                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2261                 nmd->memuse;
2262
2263         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2264
2265         /* create method header */
2266
2267         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2268         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2269         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2270         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2271         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2272
2273 #if defined(ENABLE_PROFILING)
2274         /* generate native method profiling code */
2275
2276         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2277                 /* count frequency */
2278
2279                 M_MOV_IMM(code, REG_ITMP3);
2280                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2281         }
2282 #endif
2283
2284         /* generate stub code */
2285
2286         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2287
2288 #if defined(ENABLE_GC_CACAO)
2289         /* Save callee saved integer registers in stackframeinfo (GC may
2290            need to recover them during a collection). */
2291
2292         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2293                 OFFSET(stackframeinfo_t, intregs);
2294
2295         for (i = 0; i < INT_SAV_CNT; i++)
2296                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2297 #endif
2298
2299         /* save integer and float argument registers */
2300
2301         for (i = 0; i < md->paramcount; i++) {
2302                 if (!md->params[i].inmemory) {
2303                         s1 = md->params[i].regoff;
2304
2305                         switch (md->paramtypes[i].type) {
2306                         case TYPE_INT:
2307                         case TYPE_LNG:
2308                         case TYPE_ADR:
2309                                 M_LST(s1, REG_SP, i * 8);
2310                                 break;
2311                         case TYPE_FLT:
2312                         case TYPE_DBL:
2313                                 M_DST(s1, REG_SP, i * 8);
2314                                 break;
2315                         }
2316                 }
2317         }
2318
2319         /* create dynamic stack info */
2320
2321         M_MOV(REG_SP, REG_A0);
2322         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2323         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2324         M_CALL(REG_ITMP1);
2325
2326         /* remember class argument */
2327
2328         if (m->flags & ACC_STATIC)
2329                 M_MOV(REG_RESULT, REG_ITMP2);
2330
2331         /* restore integer and float argument registers */
2332
2333         for (i = 0; i < md->paramcount; i++) {
2334                 if (!md->params[i].inmemory) {
2335                         s1 = md->params[i].regoff;
2336
2337                         switch (md->paramtypes[i].type) {
2338                         case TYPE_INT:
2339                         case TYPE_LNG:
2340                         case TYPE_ADR:
2341                                 M_LLD(s1, REG_SP, i * 8);
2342                                 break;
2343                         case TYPE_FLT:
2344                         case TYPE_DBL:
2345                                 M_DLD(s1, REG_SP, i * 8);
2346                                 break;
2347                         }
2348                 }
2349         }
2350
2351         /* Copy or spill arguments to new locations. */
2352
2353         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
2354                 s2 = nmd->params[j].regoff;
2355
2356                 switch (md->paramtypes[i].type) {
2357                 case TYPE_INT:
2358                 case TYPE_LNG:
2359                 case TYPE_ADR:
2360                         if (!md->params[i].inmemory) {
2361                                 s1 = md->params[i].regoff;
2362
2363                                 if (!nmd->params[j].inmemory)
2364                                         M_INTMOVE(s1, s2);
2365                                 else
2366                                         M_LST(s1, REG_SP, s2);
2367                         }
2368                         else {
2369                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2370                                 M_LLD(REG_ITMP1, REG_SP, s1);
2371                                 M_LST(REG_ITMP1, REG_SP, s2);
2372                         }
2373                         break;
2374                 case TYPE_FLT:
2375                         /* We only copy spilled float arguments, as the float
2376                            argument registers keep unchanged. */
2377
2378                         if (md->params[i].inmemory) {
2379                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2380
2381                                 M_FLD(REG_FTMP1, REG_SP, s1);
2382                                 M_FST(REG_FTMP1, REG_SP, s2);
2383                         }
2384                         break;
2385                 case TYPE_DBL:
2386                         if (md->params[i].inmemory) {
2387                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2388                                 M_DLD(REG_FTMP1, REG_SP, s1);
2389                                 M_DST(REG_FTMP1, REG_SP, s2);
2390                         }
2391                         break;
2392                 }
2393         }
2394
2395         /* Handle native Java methods. */
2396
2397         if (m->flags & ACC_NATIVE) {
2398                 /* put class into second argument register */
2399
2400                 if (m->flags & ACC_STATIC)
2401                         M_MOV(REG_ITMP2, REG_A1);
2402
2403                 /* put env into first argument register */
2404
2405                 M_MOV_IMM(VM_get_jnienv(), REG_A0);
2406         }
2407
2408         /* Call the native function. */
2409
2410         disp = dseg_add_functionptr(cd, f);
2411         M_ALD(REG_ITMP1, RIP, disp);
2412         M_CALL(REG_ITMP1);
2413
2414         /* save return value */
2415
2416         switch (md->returntype.type) {
2417         case TYPE_INT:
2418         case TYPE_LNG:
2419         case TYPE_ADR:
2420                 switch (md->returntype.primitivetype) {
2421                 case PRIMITIVETYPE_BOOLEAN:
2422                         M_BZEXT(REG_RESULT, REG_RESULT);
2423                         break;
2424                 case PRIMITIVETYPE_BYTE:
2425                         M_BSEXT(REG_RESULT, REG_RESULT);
2426                         break;
2427                 case PRIMITIVETYPE_CHAR:
2428                         M_CZEXT(REG_RESULT, REG_RESULT);
2429                         break;
2430                 case PRIMITIVETYPE_SHORT:
2431                         M_SSEXT(REG_RESULT, REG_RESULT);
2432                         break;
2433                 }
2434                 M_LST(REG_RESULT, REG_SP, 0 * 8);
2435                 break;
2436         case TYPE_FLT:
2437         case TYPE_DBL:
2438                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
2439                 break;
2440         case TYPE_VOID:
2441                 break;
2442         }
2443
2444         /* remove native stackframe info */
2445
2446         M_MOV(REG_SP, REG_A0);
2447         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2448         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
2449         M_CALL(REG_ITMP1);
2450         M_MOV(REG_RESULT, REG_ITMP3);
2451
2452         /* restore return value */
2453
2454         switch (md->returntype.type) {
2455         case TYPE_INT:
2456         case TYPE_LNG:
2457         case TYPE_ADR:
2458                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
2459                 break;
2460         case TYPE_FLT:
2461         case TYPE_DBL:
2462                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
2463                 break;
2464         case TYPE_VOID:
2465                 break;
2466         }
2467
2468 #if defined(ENABLE_GC_CACAO)
2469         /* Restore callee saved integer registers from stackframeinfo (GC
2470            might have modified them during a collection). */
2471
2472         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2473                 OFFSET(stackframeinfo_t, intregs);
2474
2475         for (i = 0; i < INT_SAV_CNT; i++)
2476                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2477 #endif
2478
2479         /* remove stackframe */
2480
2481         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2482
2483         /* test for exception */
2484
2485         M_TEST(REG_ITMP3);
2486         M_BNE(1);
2487         M_RET;
2488
2489         /* handle exception */
2490
2491         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
2492         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
2493         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
2494
2495         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
2496         M_JMP(REG_ITMP3);
2497 }
2498
2499
2500 /*
2501  * These are local overrides for various environment variables in Emacs.
2502  * Please do not remove this and leave it at the end of the file, where
2503  * Emacs will automagically detect them.
2504  * ---------------------------------------------------------------------
2505  * Local variables:
2506  * mode: c
2507  * indent-tabs-mode: t
2508  * c-basic-offset: 4
2509  * tab-width: 4
2510  * End:
2511  * vim:noexpandtab:sw=4:ts=4:
2512  */