src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2011
   4    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
   5
   6    This file is part of CACAO.
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2, or (at
  11    your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful, but
  14    WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16    General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, write to the Free Software
  20    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  21    02110-1301, USA.
  22
  23 */
  24
  25
  26 #include "config.h"
  27
  28 #include <assert.h>
  29 #include <stdio.h>
  30 #include <stdint.h>
  31
  32 #include "vm/types.h"
  33 #include "vm/os.hpp"
  34
  35 #include "md-abi.h"
  36
  37 #include "vm/jit/x86_64/arch.h"
  38 #include "vm/jit/x86_64/codegen.h"
  39 #include "vm/jit/x86_64/emit.h"
  40
  41 #include "mm/memory.hpp"
  42
  43 #include "native/localref.hpp"
  44 #include "native/native.hpp"
  45
  46 #include "threads/lock.hpp"
  47
  48 #include "vm/jit/builtin.hpp"
  49 #include "vm/exceptions.hpp"
  50 #include "vm/global.h"
  51 #include "vm/loader.hpp"
  52 #include "vm/options.h"
  53 #include "vm/primitive.hpp"
  54 #include "vm/statistics.h"
  55 #include "vm/string.hpp"
  56 #include "vm/vm.hpp"
  57
  58 #include "vm/jit/abi.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/code.hpp"
  61 #include "vm/jit/codegen-common.hpp"
  62 #include "vm/jit/dseg.h"
  63 #include "vm/jit/emit-common.hpp"
  64 #include "vm/jit/jit.hpp"
  65 #include "vm/jit/linenumbertable.hpp"
  66 #include "vm/jit/methodheader.h"
  67 #include "vm/jit/parse.hpp"
  68 #include "vm/jit/patcher-common.hpp"
  69 #include "vm/jit/reg.h"
  70 #include "vm/jit/stacktrace.hpp"
  71 #include "vm/jit/trap.hpp"
  72
  73
  74 /**
  75  * Generates machine code for the method prolog.
  76  */
  77 void codegen_emit_prolog(jitdata* jd)
  78 {
  79         varinfo*    var;
  80         methoddesc* md;
  81         int32_t     s1;
  82         int32_t     p, t, l;
  83         int32_t     varindex;
  84         int         i;
  85
  86         // Get required compiler data.
  87         methodinfo*   m  = jd->m;
  88         codegendata*  cd = jd->cd;
  89         registerdata* rd = jd->rd;
  90
  91         /* create stack frame (if necessary) */
  92
  93         if (cd->stackframesize)
  94                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
  95
  96         /* save used callee saved registers */
  97
  98         p = cd->stackframesize;
  99         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 100                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 101         }
 102         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 103                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 104         }
 105
 106         /* take arguments out of register or stack frame */
 107
 108         md = m->parseddesc;
 109
 110         for (p = 0, l = 0; p < md->paramcount; p++) {
 111                 t = md->paramtypes[p].type;
 112
 113                 varindex = jd->local_map[l * 5 + t];
 114
 115                 l++;
 116                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 117                         l++;
 118
 119                 if (varindex == UNUSED)
 120                         continue;
 121
 122                 var = VAR(varindex);
 123
 124                 s1 = md->params[p].regoff;
 125
 126                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 127                         if (!md->params[p].inmemory) {           /* register arguments    */
 128                                 if (!IS_INMEMORY(var->flags))
 129                                         M_INTMOVE(s1, var->vv.regoff);
 130                                 else
 131                                     M_LST(s1, REG_SP, var->vv.regoff);
 132                         }
 133                         else {                                 /* stack arguments       */
 134                                 if (!IS_INMEMORY(var->flags))
 135                                         /* + 8 for return address */
 136                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 137                                 else
 138                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 139                         }
 140                 }
 141                 else {                                     /* floating args         */
 142                         if (!md->params[p].inmemory) {           /* register arguments    */
 143                                 if (!IS_INMEMORY(var->flags))
 144                                         emit_fmove(cd, s1, var->vv.regoff);
 145                                 else
 146                                         M_DST(s1, REG_SP, var->vv.regoff);
 147                         }
 148                         else {                                 /* stack arguments       */
 149                                 if (!IS_INMEMORY(var->flags))
 150                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 151                                 else
 152                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 153                         }
 154                 }
 155         }
 156 }
 157
 158
 159 /**
 160  * Generates machine code for the method epilog.
 161  */
 162 void codegen_emit_epilog(jitdata* jd)
 163 {
 164         int32_t p;
 165         int i;
 166
 167         // Get required compiler data.
 168         codegendata*  cd = jd->cd;
 169         registerdata* rd = jd->rd;
 170
 171         p = cd->stackframesize;
 172
 173         /* restore saved registers */
 174
 175         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 176                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
 177         }
 178         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 179                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
 180         }
 181
 182         /* deallocate stack */
 183
 184         if (cd->stackframesize)
 185                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
 186
 187         M_RET;
 188 }
 189
 190 /**
 191  * Generates a memory barrier to be used after volatile writes. It can be
 192  * patched out later if the field turns out not to be volatile.
 193  */
 194 void codegen_emit_patchable_barrier(instruction *iptr, codegendata *cd, patchref_t *pr, fieldinfo *fi)
 195 {
 196         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 197                 /* Align on word boundary */
 198                 if ((((intptr_t) cd->mcodeptr) & 3) >= 2)
 199                         emit_nop(cd, 4 - (((intptr_t) cd->mcodeptr) & 3));
 200                 /* Displacement for patching out MFENCE */
 201                 pr->disp_mb = (cd->mcodeptr - cd->mcodebase - pr->mpc);
 202         }
 203         if (INSTRUCTION_IS_UNRESOLVED(iptr) || fi->flags & ACC_VOLATILE)
 204                 M_MFENCE;
 205 }
 206
 207 /**
 208  * Ensures that the patched location (an int32_t) is aligned.
 209  */
 210 static void codegen_fixup_alignment(codegendata *cd, patchref_t *pr, u1 *mcodeptr_save)
 211 {
 212         u1 *codeptr = cd->mcodeptr;
 213         int disp = PATCH_ALIGNMENT((uintptr_t) cd->mcodeptr, 0, sizeof(int32_t));
 214         memmove(mcodeptr_save + disp, mcodeptr_save, cd->mcodeptr - mcodeptr_save);
 215         pr->patch_align += cd->mcodeptr - mcodeptr_save + disp;
 216
 217         cd->mcodeptr = mcodeptr_save;
 218         emit_arbitrary_nop(cd, disp);
 219         cd->mcodeptr = codeptr + disp;
 220 }
 221
 222 /**
 223  * Generates machine code for one ICMD.
 224  */
 225 void codegen_emit_instruction(jitdata* jd, instruction* iptr)
 226 {
 227         varinfo*            var;
 228         varinfo*            dst;
 229         builtintable_entry* bte;
 230         methodinfo*         lm;             // Local methodinfo for ICMD_INVOKE*.
 231         unresolved_method*  um;
 232         fieldinfo*          fi;
 233         unresolved_field*   uf;
 234         patchref_t*         pr;
 235         int32_t             fieldtype;
 236         int32_t             s1, s2, s3, d;
 237         int32_t             disp;
 238         u1*                 mcodeptr_save;
 239
 240         // Get required compiler data.
 241         codegendata*  cd = jd->cd;
 242
 243         switch (iptr->opc) {
 244
 245                 /* constant operations ************************************************/
 246
 247                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 248
 249                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 250                         disp = dseg_add_float(cd, iptr->sx.val.f);
 251                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 252                         emit_store_dst(jd, iptr, d);
 253                         break;
 254
 255                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 256
 257                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 258                         disp = dseg_add_double(cd, iptr->sx.val.d);
 259                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 260                         emit_store_dst(jd, iptr, d);
 261                         break;
 262
 263                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 264
 265                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 266
 267                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 268                                 constant_classref *cr = iptr->sx.val.c.ref;
 269                                 disp = dseg_add_unique_address(cd, cr);
 270
 271 /*                              PROFILE_CYCLE_STOP; */
 272
 273                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 274                                                                           cr, disp);
 275
 276 /*                              PROFILE_CYCLE_START; */
 277
 278                                 M_ALD(d, RIP, disp);
 279                         }
 280                         else {
 281                                 if (iptr->sx.val.anyptr == 0) {
 282                                         M_CLR(d);
 283                                 }
 284                                 else {
 285                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 286                                         M_ALD(d, RIP, disp);
 287                                 }
 288                         }
 289                         emit_store_dst(jd, iptr, d);
 290                         break;
 291
 292
 293                 /* integer operations *************************************************/
 294
 295                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 296
 297                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 298                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 299                         M_INTMOVE(s1, d);
 300                         M_INEG(d);
 301                         emit_store_dst(jd, iptr, d);
 302                         break;
 303
 304                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 305
 306                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 307                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 308                         M_INTMOVE(s1, d);
 309                         M_LNEG(d);
 310                         emit_store_dst(jd, iptr, d);
 311                         break;
 312
 313                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 314
 315                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 316                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 317                         M_ISEXT(s1, d);
 318                         emit_store_dst(jd, iptr, d);
 319                         break;
 320
 321                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 322
 323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 324                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 325                         M_IMOV(s1, d);
 326                         emit_store_dst(jd, iptr, d);
 327                         break;
 328
 329                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 330
 331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 332                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 333                         M_BSEXT(s1, d);
 334                         emit_store_dst(jd, iptr, d);
 335                         break;
 336
 337                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 338
 339                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 340                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 341                         M_CZEXT(s1, d);
 342                         emit_store_dst(jd, iptr, d);
 343                         break;
 344
 345                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 346
 347                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 348                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 349                         M_SSEXT(s1, d);
 350                         emit_store_dst(jd, iptr, d);
 351                         break;
 352
 353
 354                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 355
 356                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 357                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 358                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 359                         if (s2 == d)
 360                                 M_IADD(s1, d);
 361                         else {
 362                                 M_INTMOVE(s1, d);
 363                                 M_IADD(s2, d);
 364                         }
 365                         emit_store_dst(jd, iptr, d);
 366                         break;
 367
 368                 case ICMD_IINC:
 369                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 370                                       /* sx.val.i = constant                             */
 371
 372                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 373                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 374
 375                         /* Using inc and dec is not faster than add (tested with
 376                            sieve). */
 377
 378                         M_INTMOVE(s1, d);
 379                         M_IADD_IMM(iptr->sx.val.i, d);
 380                         emit_store_dst(jd, iptr, d);
 381                         break;
 382
 383                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 384
 385                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 386                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 387                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 388                         if (s2 == d)
 389                                 M_LADD(s1, d);
 390                         else {
 391                                 M_INTMOVE(s1, d);
 392                                 M_LADD(s2, d);
 393                         }
 394                         emit_store_dst(jd, iptr, d);
 395                         break;
 396
 397                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 398                                       /* sx.val.l = constant                             */
 399
 400                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 401                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 402                         M_INTMOVE(s1, d);
 403                         if (IS_IMM32(iptr->sx.val.l))
 404                                 M_LADD_IMM(iptr->sx.val.l, d);
 405                         else {
 406                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 407                                 M_LADD(REG_ITMP2, d);
 408                         }
 409                         emit_store_dst(jd, iptr, d);
 410                         break;
 411
 412                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 413
 414                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 415                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 416                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 417                         if (s2 == d) {
 418                                 M_INTMOVE(s1, REG_ITMP1);
 419                                 M_ISUB(s2, REG_ITMP1);
 420                                 M_INTMOVE(REG_ITMP1, d);
 421                         } else {
 422                                 M_INTMOVE(s1, d);
 423                                 M_ISUB(s2, d);
 424                         }
 425                         emit_store_dst(jd, iptr, d);
 426                         break;
 427
 428                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 429                                       /* sx.val.i = constant                             */
 430
 431                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 432                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 433                         M_INTMOVE(s1, d);
 434                         M_ISUB_IMM(iptr->sx.val.i, d);
 435                         emit_store_dst(jd, iptr, d);
 436                         break;
 437
 438                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 439
 440                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 441                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 442                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 443                         if (s2 == d) {
 444                                 M_INTMOVE(s1, REG_ITMP1);
 445                                 M_LSUB(s2, REG_ITMP1);
 446                                 M_INTMOVE(REG_ITMP1, d);
 447                         } else {
 448                                 M_INTMOVE(s1, d);
 449                                 M_LSUB(s2, d);
 450                         }
 451                         emit_store_dst(jd, iptr, d);
 452                         break;
 453
 454                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 455                                       /* sx.val.l = constant                             */
 456
 457                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 458                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 459                         M_INTMOVE(s1, d);
 460                         if (IS_IMM32(iptr->sx.val.l))
 461                                 M_LSUB_IMM(iptr->sx.val.l, d);
 462                         else {
 463                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 464                                 M_LSUB(REG_ITMP2, d);
 465                         }
 466                         emit_store_dst(jd, iptr, d);
 467                         break;
 468
 469                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 470
 471                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 472                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 473                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 474                         if (s2 == d)
 475                                 M_IMUL(s1, d);
 476                         else {
 477                                 M_INTMOVE(s1, d);
 478                                 M_IMUL(s2, d);
 479                         }
 480                         emit_store_dst(jd, iptr, d);
 481                         break;
 482
 483                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 484                                       /* sx.val.i = constant                             */
 485
 486                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 487                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 488                         if (iptr->sx.val.i == 2) {
 489                                 M_INTMOVE(s1, d);
 490                                 M_ISLL_IMM(1, d);
 491                         } else
 492                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 493                         emit_store_dst(jd, iptr, d);
 494                         break;
 495
 496                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 497
 498                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 499                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 500                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 501                         if (s2 == d)
 502                                 M_LMUL(s1, d);
 503                         else {
 504                                 M_INTMOVE(s1, d);
 505                                 M_LMUL(s2, d);
 506                         }
 507                         emit_store_dst(jd, iptr, d);
 508                         break;
 509
 510                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 511                                       /* sx.val.l = constant                             */
 512
 513                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 514                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 515                         if (IS_IMM32(iptr->sx.val.l))
 516                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 517                         else {
 518                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 519                                 M_INTMOVE(s1, d);
 520                                 M_LMUL(REG_ITMP2, d);
 521                         }
 522                         emit_store_dst(jd, iptr, d);
 523                         break;
 524
 525                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 526
 527                         s1 = emit_load_s1(jd, iptr, RAX);
 528                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 529                         d = codegen_reg_of_dst(jd, iptr, RAX);
 530
 531                         M_INTMOVE(s1, RAX);
 532                         M_INTMOVE(s2, REG_ITMP3);
 533                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 534
 535                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 536
 537                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 538                         M_BNE(4 + 6);
 539                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 540                         M_BEQ(1 + 3);                                          /* 6 bytes */
 541
 542                         emit_cltd(cd);                                         /* 1 byte  */
 543                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 544
 545                         M_INTMOVE(RAX, d);
 546                         emit_store_dst(jd, iptr, d);
 547                         dst = VAROP(iptr->dst);
 548                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 549                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 550                         break;
 551
 552                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 553
 554                         s1 = emit_load_s1(jd, iptr, RAX);
 555                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 556                         d = codegen_reg_of_dst(jd, iptr, RDX);
 557
 558                         M_INTMOVE(s1, RAX);
 559                         M_INTMOVE(s2, REG_ITMP3);
 560                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 561
 562                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 563
 564                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 565                         M_BNE(3 + 4 + 6);
 566                         M_CLR(RDX);                                            /* 3 bytes */
 567                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 568                         M_BEQ(1 + 3);                                          /* 6 bytes */
 569
 570                         emit_cltd(cd);                                         /* 1 byte  */
 571                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 572
 573                         M_INTMOVE(RDX, d);
 574                         emit_store_dst(jd, iptr, d);
 575                         dst = VAROP(iptr->dst);
 576                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 577                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 578                         break;
 579
 580                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 581                                       /* sx.val.i = constant                             */
 582
 583                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 584                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 585                         M_INTMOVE(s1, REG_ITMP1);
 586                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 587                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 588                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 589                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 590                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 591                         emit_store_dst(jd, iptr, d);
 592                         break;
 593
 594                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 595                                       /* sx.val.i = constant                             */
 596
 597                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 598                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 599                         M_INTMOVE(s1, REG_ITMP1);
 600                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 601                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 602                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 603                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 604                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 605                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 606                         emit_store_dst(jd, iptr, d);
 607                         break;
 608
 609
 610                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 611
 612                         s1 = emit_load_s1(jd, iptr, RAX);
 613                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 614                         d = codegen_reg_of_dst(jd, iptr, RAX);
 615
 616                         M_INTMOVE(s1, RAX);
 617                         M_INTMOVE(s2, REG_ITMP3);
 618                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 619
 620                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 621
 622                         /* check as described in jvm spec */
 623                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 624                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 625                         M_BNE(4 + 6);
 626                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 627                         M_BEQ(2 + 3);                                          /* 6 bytes */
 628
 629                         emit_cqto(cd);                                         /* 2 bytes */
 630                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 631
 632                         M_INTMOVE(RAX, d);
 633                         emit_store_dst(jd, iptr, d);
 634                         dst = VAROP(iptr->dst);
 635                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 636                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 637                         break;
 638
 639                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 640
 641                         s1 = emit_load_s1(jd, iptr, RAX);
 642                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 643                         d = codegen_reg_of_dst(jd, iptr, RDX);
 644
 645                         M_INTMOVE(s1, RAX);
 646                         M_INTMOVE(s2, REG_ITMP3);
 647                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 648
 649                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 650
 651                         /* check as described in jvm spec */
 652                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 653                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 654                         M_BNE(3 + 4 + 6);
 655                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 656                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 657                         M_BEQ(2 + 3);                                          /* 6 bytes */
 658
 659                         emit_cqto(cd);                                         /* 2 bytes */
 660                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 661
 662                         M_INTMOVE(RDX, d);
 663                         emit_store_dst(jd, iptr, d);
 664                         dst = VAROP(iptr->dst);
 665                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 666                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 667                         break;
 668
 669                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 670                                       /* sx.val.i = constant                             */
 671
 672                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 673                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 674                         M_INTMOVE(s1, REG_ITMP1);
 675                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 676                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 677                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 678                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 679                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 680                         emit_store_dst(jd, iptr, d);
 681                         break;
 682
 683                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 684                                       /* sx.val.l = constant                             */
 685
 686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 687                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 688                         M_INTMOVE(s1, REG_ITMP1);
 689                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 690                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 691                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 692                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 693                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 694                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 695                         emit_store_dst(jd, iptr, d);
 696                         break;
 697
 698                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 699
 700                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 701                         emit_ishift(jd, SHIFT_SHL, iptr);
 702                         break;
 703
 704                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 705                                       /* sx.val.i = constant                             */
 706
 707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 708                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 709                         M_INTMOVE(s1, d);
 710                         M_ISLL_IMM(iptr->sx.val.i, d);
 711                         emit_store_dst(jd, iptr, d);
 712                         break;
 713
 714                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 715
 716                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 717                         emit_ishift(jd, SHIFT_SAR, iptr);
 718                         break;
 719
 720                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 721                                       /* sx.val.i = constant                             */
 722
 723                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 724                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 725                         M_INTMOVE(s1, d);
 726                         M_ISRA_IMM(iptr->sx.val.i, d);
 727                         emit_store_dst(jd, iptr, d);
 728                         break;
 729
 730                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 731
 732                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 733                         emit_ishift(jd, SHIFT_SHR, iptr);
 734                         break;
 735
 736                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 737                                       /* sx.val.i = constant                             */
 738
 739                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 740                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 741                         M_INTMOVE(s1, d);
 742                         M_ISRL_IMM(iptr->sx.val.i, d);
 743                         emit_store_dst(jd, iptr, d);
 744                         break;
 745
 746                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 747
 748                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 749                         emit_lshift(jd, SHIFT_SHL, iptr);
 750                         break;
 751
 752         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 753                                           /* sx.val.i = constant                             */
 754
 755                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 756                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 757                         M_INTMOVE(s1, d);
 758                         M_LSLL_IMM(iptr->sx.val.i, d);
 759                         emit_store_dst(jd, iptr, d);
 760                         break;
 761
 762                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 763
 764                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 765                         emit_lshift(jd, SHIFT_SAR, iptr);
 766                         break;
 767
 768                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 769                                       /* sx.val.i = constant                             */
 770
 771                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 772                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 773                         M_INTMOVE(s1, d);
 774                         M_LSRA_IMM(iptr->sx.val.i, d);
 775                         emit_store_dst(jd, iptr, d);
 776                         break;
 777
 778                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 779
 780                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 781                         emit_lshift(jd, SHIFT_SHR, iptr);
 782                         break;
 783
 784                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 785                                       /* sx.val.l = constant                             */
 786
 787                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 788                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 789                         M_INTMOVE(s1, d);
 790                         M_LSRL_IMM(iptr->sx.val.i, d);
 791                         emit_store_dst(jd, iptr, d);
 792                         break;
 793
 794                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
 795
 796                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 797                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 798                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 799                         if (s2 == d)
 800                                 M_IAND(s1, d);
 801                         else {
 802                                 M_INTMOVE(s1, d);
 803                                 M_IAND(s2, d);
 804                         }
 805                         emit_store_dst(jd, iptr, d);
 806                         break;
 807
 808                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
 809                                       /* sx.val.i = constant                             */
 810
 811                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 812                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 813                         M_INTMOVE(s1, d);
 814                         M_IAND_IMM(iptr->sx.val.i, d);
 815                         emit_store_dst(jd, iptr, d);
 816                         break;
 817
 818                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
 819
 820                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 821                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 822                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 823                         if (s2 == d)
 824                                 M_LAND(s1, d);
 825                         else {
 826                                 M_INTMOVE(s1, d);
 827                                 M_LAND(s2, d);
 828                         }
 829                         emit_store_dst(jd, iptr, d);
 830                         break;
 831
 832                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
 833                                       /* sx.val.l = constant                             */
 834
 835                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 836                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 837                         M_INTMOVE(s1, d);
 838                         if (IS_IMM32(iptr->sx.val.l))
 839                                 M_LAND_IMM(iptr->sx.val.l, d);
 840                         else {
 841                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 842                                 M_LAND(REG_ITMP2, d);
 843                         }
 844                         emit_store_dst(jd, iptr, d);
 845                         break;
 846
 847                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
 848
 849                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 850                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 851                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 852                         if (s2 == d)
 853                                 M_IOR(s1, d);
 854                         else {
 855                                 M_INTMOVE(s1, d);
 856                                 M_IOR(s2, d);
 857                         }
 858                         emit_store_dst(jd, iptr, d);
 859                         break;
 860
 861                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
 862                                       /* sx.val.i = constant                             */
 863
 864                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 865                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 866                         M_INTMOVE(s1, d);
 867                         M_IOR_IMM(iptr->sx.val.i, d);
 868                         emit_store_dst(jd, iptr, d);
 869                         break;
 870
 871                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
 872
 873                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 874                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 875                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 876                         if (s2 == d)
 877                                 M_LOR(s1, d);
 878                         else {
 879                                 M_INTMOVE(s1, d);
 880                                 M_LOR(s2, d);
 881                         }
 882                         emit_store_dst(jd, iptr, d);
 883                         break;
 884
 885                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
 886                                       /* sx.val.l = constant                             */
 887
 888                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 889                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 890                         M_INTMOVE(s1, d);
 891                         if (IS_IMM32(iptr->sx.val.l))
 892                                 M_LOR_IMM(iptr->sx.val.l, d);
 893                         else {
 894                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 895                                 M_LOR(REG_ITMP2, d);
 896                         }
 897                         emit_store_dst(jd, iptr, d);
 898                         break;
 899
 900                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
 901
 902                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 903                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 904                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 905                         if (s2 == d)
 906                                 M_IXOR(s1, d);
 907                         else {
 908                                 M_INTMOVE(s1, d);
 909                                 M_IXOR(s2, d);
 910                         }
 911                         emit_store_dst(jd, iptr, d);
 912                         break;
 913
 914                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
 915                                       /* sx.val.i = constant                             */
 916
 917                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 918                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 919                         M_INTMOVE(s1, d);
 920                         M_IXOR_IMM(iptr->sx.val.i, d);
 921                         emit_store_dst(jd, iptr, d);
 922                         break;
 923
 924                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
 925
 926                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 927                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 928                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 929                         if (s2 == d)
 930                                 M_LXOR(s1, d);
 931                         else {
 932                                 M_INTMOVE(s1, d);
 933                                 M_LXOR(s2, d);
 934                         }
 935                         emit_store_dst(jd, iptr, d);
 936                         break;
 937
 938                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
 939                                       /* sx.val.l = constant                             */
 940
 941                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 942                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 943                         M_INTMOVE(s1, d);
 944                         if (IS_IMM32(iptr->sx.val.l))
 945                                 M_LXOR_IMM(iptr->sx.val.l, d);
 946                         else {
 947                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 948                                 M_LXOR(REG_ITMP2, d);
 949                         }
 950                         emit_store_dst(jd, iptr, d);
 951                         break;
 952
 953
 954                 /* floating operations ************************************************/
 955
 956                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
 957
 958                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 959                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 960                         disp = dseg_add_s4(cd, 0x80000000);
 961                         emit_fmove(cd, s1, d);
 962                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
 963                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
 964                         emit_store_dst(jd, iptr, d);
 965                         break;
 966
 967                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
 968
 969                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 970                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 971                         disp = dseg_add_s8(cd, 0x8000000000000000);
 972                         emit_fmove(cd, s1, d);
 973                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
 974                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
 975                         emit_store_dst(jd, iptr, d);
 976                         break;
 977
 978                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 979
 980                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 981                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 982                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 983                         if (s2 == d)
 984                                 M_FADD(s1, d);
 985                         else {
 986                                 emit_fmove(cd, s1, d);
 987                                 M_FADD(s2, d);
 988                         }
 989                         emit_store_dst(jd, iptr, d);
 990                         break;
 991
 992                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 993
 994                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 995                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 996                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 997                         if (s2 == d)
 998                                 M_DADD(s1, d);
 999                         else {
1000                                 emit_fmove(cd, s1, d);
1001                                 M_DADD(s2, d);
1002                         }
1003                         emit_store_dst(jd, iptr, d);
1004                         break;
1005
1006                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1007
1008                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1009                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1010                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1011                         if (s2 == d) {
1012                                 emit_fmove(cd, s2, REG_FTMP2);
1013                                 s2 = REG_FTMP2;
1014                         }
1015                         emit_fmove(cd, s1, d);
1016                         M_FSUB(s2, d);
1017                         emit_store_dst(jd, iptr, d);
1018                         break;
1019
1020                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1021
1022                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1023                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1024                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1025                         if (s2 == d) {
1026                                 emit_fmove(cd, s2, REG_FTMP2);
1027                                 s2 = REG_FTMP2;
1028                         }
1029                         emit_fmove(cd, s1, d);
1030                         M_DSUB(s2, d);
1031                         emit_store_dst(jd, iptr, d);
1032                         break;
1033
1034                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1035
1036                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1037                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1038                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1039                         if (s2 == d)
1040                                 M_FMUL(s1, d);
1041                         else {
1042                                 emit_fmove(cd, s1, d);
1043                                 M_FMUL(s2, d);
1044                         }
1045                         emit_store_dst(jd, iptr, d);
1046                         break;
1047
1048                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1049
1050                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1051                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1052                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1053                         if (s2 == d)
1054                                 M_DMUL(s1, d);
1055                         else {
1056                                 emit_fmove(cd, s1, d);
1057                                 M_DMUL(s2, d);
1058                         }
1059                         emit_store_dst(jd, iptr, d);
1060                         break;
1061
1062                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1063
1064                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1065                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1066                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1067                         if (s2 == d) {
1068                                 emit_fmove(cd, s2, REG_FTMP2);
1069                                 s2 = REG_FTMP2;
1070                         }
1071                         emit_fmove(cd, s1, d);
1072                         M_FDIV(s2, d);
1073                         emit_store_dst(jd, iptr, d);
1074                         break;
1075
1076                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1077
1078                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1079                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1080                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1081                         if (s2 == d) {
1082                                 emit_fmove(cd, s2, REG_FTMP2);
1083                                 s2 = REG_FTMP2;
1084                         }
1085                         emit_fmove(cd, s1, d);
1086                         M_DDIV(s2, d);
1087                         emit_store_dst(jd, iptr, d);
1088                         break;
1089
1090                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1091
1092                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1093                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1094                         M_CVTIF(s1, d);
1095                         emit_store_dst(jd, iptr, d);
1096                         break;
1097
1098                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1099
1100                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1101                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1102                         M_CVTID(s1, d);
1103                         emit_store_dst(jd, iptr, d);
1104                         break;
1105
1106                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1107
1108                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1109                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1110                         M_CVTLF(s1, d);
1111                         emit_store_dst(jd, iptr, d);
1112                         break;
1113
1114                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1115
1116                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1117                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1118                         M_CVTLD(s1, d);
1119                         emit_store_dst(jd, iptr, d);
1120                         break;
1121
1122                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1123
1124                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1125                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1126                         M_CVTFI(s1, d);
1127                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1128                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1129                                 ((REG_RESULT == d) ? 0 : 3);
1130                         M_BNE(disp);
1131                         emit_fmove(cd, s1, REG_FTMP1);
1132                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1133                         M_CALL(REG_ITMP2);
1134                         M_INTMOVE(REG_RESULT, d);
1135                         emit_store_dst(jd, iptr, d);
1136                         break;
1137
1138                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1139
1140                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1141                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1142                         M_CVTDI(s1, d);
1143                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1144                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1145                                 ((REG_RESULT == d) ? 0 : 3);
1146                         M_BNE(disp);
1147                         emit_fmove(cd, s1, REG_FTMP1);
1148                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1149                         M_CALL(REG_ITMP2);
1150                         M_INTMOVE(REG_RESULT, d);
1151                         emit_store_dst(jd, iptr, d);
1152                         break;
1153
1154                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1155
1156                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1157                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1158                         M_CVTFL(s1, d);
1159                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1160                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1161                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1162                                 ((REG_RESULT == d) ? 0 : 3);
1163                         M_BNE(disp);
1164                         emit_fmove(cd, s1, REG_FTMP1);
1165                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1166                         M_CALL(REG_ITMP2);
1167                         M_INTMOVE(REG_RESULT, d);
1168                         emit_store_dst(jd, iptr, d);
1169                         break;
1170
1171                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1172
1173                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1174                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1175                         M_CVTDL(s1, d);
1176                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1177                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1178                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1179                                 ((REG_RESULT == d) ? 0 : 3);
1180                         M_BNE(disp);
1181                         emit_fmove(cd, s1, REG_FTMP1);
1182                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1183                         M_CALL(REG_ITMP2);
1184                         M_INTMOVE(REG_RESULT, d);
1185                         emit_store_dst(jd, iptr, d);
1186                         break;
1187
1188                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1189
1190                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1191                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1192                         M_CVTFD(s1, d);
1193                         emit_store_dst(jd, iptr, d);
1194                         break;
1195
1196                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1197
1198                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1199                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1200                         M_CVTDF(s1, d);
1201                         emit_store_dst(jd, iptr, d);
1202                         break;
1203
1204                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1205                                           /* == => 0, < => 1, > => -1 */
1206
1207                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1208                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1209                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1210                         M_CLR(d);
1211                         M_MOV_IMM(1, REG_ITMP1);
1212                         M_MOV_IMM(-1, REG_ITMP2);
1213                         emit_ucomiss_reg_reg(cd, s1, s2);
1214                         M_CMOVULT(REG_ITMP1, d);
1215                         M_CMOVUGT(REG_ITMP2, d);
1216                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1217                         emit_store_dst(jd, iptr, d);
1218                         break;
1219
1220                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1221                                           /* == => 0, < => 1, > => -1 */
1222
1223                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1224                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1225                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1226                         M_CLR(d);
1227                         M_MOV_IMM(1, REG_ITMP1);
1228                         M_MOV_IMM(-1, REG_ITMP2);
1229                         emit_ucomiss_reg_reg(cd, s1, s2);
1230                         M_CMOVULT(REG_ITMP1, d);
1231                         M_CMOVUGT(REG_ITMP2, d);
1232                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1233                         emit_store_dst(jd, iptr, d);
1234                         break;
1235
1236                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1237                                           /* == => 0, < => 1, > => -1 */
1238
1239                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1240                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1241                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1242                         M_CLR(d);
1243                         M_MOV_IMM(1, REG_ITMP1);
1244                         M_MOV_IMM(-1, REG_ITMP2);
1245                         emit_ucomisd_reg_reg(cd, s1, s2);
1246                         M_CMOVULT(REG_ITMP1, d);
1247                         M_CMOVUGT(REG_ITMP2, d);
1248                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1249                         emit_store_dst(jd, iptr, d);
1250                         break;
1251
1252                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1253                                           /* == => 0, < => 1, > => -1 */
1254
1255                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1256                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1257                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1258                         M_CLR(d);
1259                         M_MOV_IMM(1, REG_ITMP1);
1260                         M_MOV_IMM(-1, REG_ITMP2);
1261                         emit_ucomisd_reg_reg(cd, s1, s2);
1262                         M_CMOVULT(REG_ITMP1, d);
1263                         M_CMOVUGT(REG_ITMP2, d);
1264                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1265                         emit_store_dst(jd, iptr, d);
1266                         break;
1267
1268
1269                 /* memory operations **************************************************/
1270
1271                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1272
1273                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1274                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1275                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1276                         /* implicit null-pointer check */
1277                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1278                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1279                         emit_store_dst(jd, iptr, d);
1280                         break;
1281
1282                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1283
1284                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1285                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1286                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1287                         /* implicit null-pointer check */
1288                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1289                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1290                         emit_store_dst(jd, iptr, d);
1291                         break;
1292
1293                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1294
1295                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1296                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1297                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1298                         /* implicit null-pointer check */
1299                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1300                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1301                         emit_store_dst(jd, iptr, d);
1302                         break;
1303
1304                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1305
1306                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1307                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1308                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1309                         /* implicit null-pointer check */
1310                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1311                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1312                         emit_store_dst(jd, iptr, d);
1313                         break;
1314
1315                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1316
1317                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1318                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1319                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1320                         /* implicit null-pointer check */
1321                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1322                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1323                         emit_store_dst(jd, iptr, d);
1324                         break;
1325
1326                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1327
1328                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1329                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1330                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1331                         /* implicit null-pointer check */
1332                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1333                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1340                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1341                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1342                         /* implicit null-pointer check */
1343                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1344                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1345                         emit_store_dst(jd, iptr, d);
1346                         break;
1347
1348                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1349
1350                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1351                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1352                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1353                         /* implicit null-pointer check */
1354                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1355                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1356                         emit_store_dst(jd, iptr, d);
1357                         break;
1358
1359
1360                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1361
1362                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1363                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1364                         /* implicit null-pointer check */
1365                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1366                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1367                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1368                         break;
1369
1370                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1371
1372                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1373                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1374                         /* implicit null-pointer check */
1375                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1376                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1377                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1378                         break;
1379
1380                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1381
1382                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1383                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1384                         /* implicit null-pointer check */
1385                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1386                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1387                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1388                         break;
1389
1390                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1391
1392                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1393                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1394                         /* implicit null-pointer check */
1395                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1396                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1397                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1398                         break;
1399
1400                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1401
1402                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1403                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1404                         /* implicit null-pointer check */
1405                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1406                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1407                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1408                         break;
1409
1410                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1411
1412                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1413                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1414                         /* implicit null-pointer check */
1415                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1416                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1417                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1418                         break;
1419
1420                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1421
1422                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1423                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1424                         /* implicit null-pointer check */
1425                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1426                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1427                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1428                         break;
1429
1430                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1431
1432                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1433                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1434                         /* implicit null-pointer check */
1435                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1436                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1437
1438                         M_MOV(s1, REG_A0);
1439                         M_MOV(s3, REG_A1);
1440                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1441                         M_CALL(REG_ITMP1);
1442                         emit_arraystore_check(cd, iptr);
1443
1444                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1445                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1446                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1447                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1448                         break;
1449
1450
1451                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1452
1453                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1454                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1455                         /* implicit null-pointer check */
1456                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1457                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1458                         break;
1459
1460                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1461
1462                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1463                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1464                         /* implicit null-pointer check */
1465                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1466                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1467                         break;
1468
1469                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1470
1471                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1472                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1473                         /* implicit null-pointer check */
1474                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1475                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1476                         break;
1477
1478                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1479
1480                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1481                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1482                         /* implicit null-pointer check */
1483                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1484                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1485                         break;
1486
1487                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1488
1489                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1490                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1491                         /* implicit null-pointer check */
1492                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1493
1494                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1495                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1496                         }
1497                         else {
1498                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1499                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1500                         }
1501                         break;
1502
1503                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1504
1505                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1506                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1507                         /* implicit null-pointer check */
1508                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1509                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1510                         break;
1511
1512                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1513                                           /* val = value (in current instruction)     */
1514                                           /* following NOP)                           */
1515
1516                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1517                                 uf        = iptr->sx.s23.s3.uf;
1518                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1519                                 disp      = dseg_add_unique_address(cd, uf);
1520
1521 /*                              PROFILE_CYCLE_STOP; */
1522
1523                                 pr = patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1524
1525 /*                              PROFILE_CYCLE_START; */
1526
1527                                 fi = NULL;              /* Silence compiler warning */
1528                         }
1529                         else {
1530                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1531                                 fieldtype = fi->type;
1532                                 disp      = dseg_add_address(cd, fi->value);
1533
1534                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1535                                         //PROFILE_CYCLE_STOP;
1536
1537                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1538                                                                                   fi->clazz, 0);
1539
1540                                         //PROFILE_CYCLE_START;
1541                                 }
1542
1543                                 pr = NULL;              /* Silence compiler warning */
1544                         }
1545
1546                         /* This approach is much faster than moving the field
1547                            address inline into a register. */
1548
1549                         M_ALD(REG_ITMP1, RIP, disp);
1550
1551                         switch (fieldtype) {
1552                         case TYPE_INT:
1553                         case TYPE_FLT:
1554                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1555                                 break;
1556                         case TYPE_LNG:
1557                         case TYPE_ADR:
1558                         case TYPE_DBL:
1559                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1560                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1561                                 else {
1562                                         M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1563                                         M_LST(REG_ITMP2, REG_ITMP1, 0);
1564                                 }
1565                                 break;
1566                         }
1567                         codegen_emit_patchable_barrier(iptr, cd, pr, fi);
1568                         break;
1569
1570                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1571
1572                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1573
1574                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1575                                 uf        = iptr->sx.s23.s3.uf;
1576                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1577                                 disp      = 0;
1578
1579 /*                              PROFILE_CYCLE_STOP; */
1580
1581                                 pr = patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1582                                 mcodeptr_save = cd->mcodeptr;
1583
1584 /*                              PROFILE_CYCLE_START; */
1585
1586                                 fi = NULL;              /* Silence compiler warning */
1587                         }
1588                         else {
1589                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1590                                 fieldtype = fi->type;
1591                                 disp      = fi->offset;
1592
1593                                 pr = 0;
1594                         }
1595
1596                         /* implicit null-pointer check */
1597                         switch (fieldtype) {
1598                         case TYPE_INT:
1599                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1600                                 M_ILD32(d, s1, disp);
1601                                 break;
1602                         case TYPE_LNG:
1603                         case TYPE_ADR:
1604                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1605                                 M_LLD32(d, s1, disp);
1606                                 break;
1607                         case TYPE_FLT:
1608                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1609                                 M_FLD32(d, s1, disp);
1610                                 break;
1611                         case TYPE_DBL:
1612                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1613                                 M_DLD32(d, s1, disp);
1614                                 break;
1615                         default:
1616                                 // Silence compiler warning.
1617                                 d = 0;
1618                         }
1619                         if (pr)
1620                                 codegen_fixup_alignment(cd, pr, mcodeptr_save);
1621                         emit_store_dst(jd, iptr, d);
1622                         break;
1623
1624                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1625
1626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1627                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1628
1629                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1630                                 uf        = iptr->sx.s23.s3.uf;
1631                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1632                                 disp      = 0;
1633
1634 /*                              PROFILE_CYCLE_STOP; */
1635
1636                                 pr = patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1637                                 mcodeptr_save = cd->mcodeptr;
1638
1639 /*                              PROFILE_CYCLE_START; */
1640
1641                                 fi = NULL;              /* Silence compiler warning */
1642                         }
1643                         else {
1644                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1645                                 fieldtype = fi->type;
1646                                 disp      = fi->offset;
1647
1648                                 pr = NULL;              /* Silence compiler warning */
1649                         }
1650
1651                         /* implicit null-pointer check */
1652                         switch (fieldtype) {
1653                         case TYPE_INT:
1654                                 M_IST32(s2, s1, disp);
1655                                 break;
1656                         case TYPE_LNG:
1657                         case TYPE_ADR:
1658                                 M_LST32(s2, s1, disp);
1659                                 break;
1660                         case TYPE_FLT:
1661                                 M_FST32(s2, s1, disp);
1662                                 break;
1663                         case TYPE_DBL:
1664                                 M_DST32(s2, s1, disp);
1665                                 break;
1666                         }
1667                         if (pr)
1668                                 codegen_fixup_alignment(cd, pr, mcodeptr_save);
1669                         codegen_emit_patchable_barrier(iptr, cd, pr, fi);
1670                         break;
1671
1672                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1673                                           /* val = value (in current instruction)     */
1674                                           /* following NOP)                           */
1675
1676                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1677
1678                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1679                                 uf        = iptr->sx.s23.s3.uf;
1680                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1681                                 disp      = 0;
1682
1683 /*                              PROFILE_CYCLE_STOP; */
1684
1685                                 pr = patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1686                                 mcodeptr_save = cd->mcodeptr;
1687
1688 /*                              PROFILE_CYCLE_START; */
1689
1690                                 fi = NULL;              /* Silence compiler warning */
1691                         }
1692                         else {
1693                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1694                                 fieldtype = fi->type;
1695                                 disp      = fi->offset;
1696
1697                                 pr = NULL;              /* Silence compiler warning */
1698                         }
1699
1700                         /* implicit null-pointer check */
1701                         switch (fieldtype) {
1702                         case TYPE_INT:
1703                         case TYPE_FLT:
1704                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
1705                                 break;
1706                         case TYPE_LNG:
1707                         case TYPE_ADR:
1708                         case TYPE_DBL:
1709                                 /* XXX why no check for IS_IMM32? -- probably because of the patcher */
1710                                 M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1711                                 if (disp)  /* resolved, disp can never be 0 */
1712                                         M_LST(REG_ITMP2, s1, disp);
1713                                 else {     /* unresolved */
1714                                         M_LST32(REG_ITMP2, s1, disp);
1715                                         pr->patch_align = 4;
1716                                 }
1717                                 break;
1718                         }
1719                         if (pr)
1720                                 codegen_fixup_alignment(cd, pr, mcodeptr_save);
1721                         codegen_emit_patchable_barrier(iptr, cd, pr, fi);
1722                         break;
1723
1724
1725                 /* branch operations **************************************************/
1726
1727                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
1728
1729                         M_CALL_IMM(0);                            /* passing exception pc */
1730                         M_POP(REG_ITMP2_XPC);
1731
1732                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
1733                         M_JMP(REG_ITMP3);
1734                         break;
1735
1736                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
1737                 case ICMD_IF_LNE:
1738                 case ICMD_IF_LLT:
1739                 case ICMD_IF_LGE:
1740                 case ICMD_IF_LGT:
1741                 case ICMD_IF_LLE:
1742
1743                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1744                         if (IS_IMM32(iptr->sx.val.l))
1745                                 M_LCMP_IMM(iptr->sx.val.l, s1);
1746                         else {
1747                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1748                                 M_LCMP(REG_ITMP2, s1);
1749                         }
1750                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
1751                         break;
1752
1753                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
1754                 case ICMD_IF_LCMPNE:
1755                 case ICMD_IF_LCMPLT:
1756                 case ICMD_IF_LCMPGE:
1757                 case ICMD_IF_LCMPGT:
1758                 case ICMD_IF_LCMPLE:
1759
1760                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1761                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1762                         M_LCMP(s2, s1);
1763                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
1764                         break;
1765
1766                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
1767                         {
1768                                 s4 i, l;
1769                                 branch_target_t *table;
1770
1771                                 table = iptr->dst.table;
1772
1773                                 l = iptr->sx.s23.s2.tablelow;
1774                                 i = iptr->sx.s23.s3.tablehigh;
1775
1776                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1777                                 M_INTMOVE(s1, REG_ITMP1);
1778
1779                                 if (l != 0)
1780                                         M_ISUB_IMM(l, REG_ITMP1);
1781
1782                                 /* number of targets */
1783                                 i = i - l + 1;
1784
1785                 /* range check */
1786
1787                                 M_ICMP_IMM(i - 1, REG_ITMP1);
1788                                 emit_bugt(cd, table[0].block);
1789
1790                                 /* build jump table top down and use address of lowest entry */
1791
1792                                 table += i;
1793
1794                                 while (--i >= 0) {
1795                                         dseg_add_target(cd, table->block);
1796                                         --table;
1797                                 }
1798
1799                                 /* length of dataseg after last dseg_add_target is used
1800                                    by load */
1801
1802                                 M_MOV_IMM(0, REG_ITMP2);
1803                                 dseg_adddata(cd);
1804                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
1805                                 M_JMP(REG_ITMP1);
1806                         }
1807                         break;
1808
1809                 case ICMD_BUILTIN:
1810                         bte = iptr->sx.s23.s3.bte;
1811                         if (bte->stub == NULL) {
1812                                 M_MOV_IMM(bte->fp, REG_ITMP1);
1813                         }
1814                         else {
1815                                 M_MOV_IMM(bte->stub, REG_ITMP1);
1816                         }
1817                         M_CALL(REG_ITMP1);
1818                         break;
1819
1820                 case ICMD_INVOKESPECIAL:
1821                         emit_nullpointer_check(cd, iptr, REG_A0);
1822                         /* fall through */
1823
1824                 case ICMD_INVOKESTATIC:
1825                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1826                                 um = iptr->sx.s23.s3.um;
1827                                 disp = dseg_add_unique_address(cd, um);
1828
1829                                 patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
1830                                                                           um, disp);
1831                         }
1832                         else {
1833                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1834                                 disp = dseg_add_functionptr(cd, lm->stubroutine);
1835                         }
1836
1837                         M_ALD(REG_ITMP2, RIP, disp);
1838                         M_CALL(REG_ITMP2);
1839                         break;
1840
1841                 case ICMD_INVOKEVIRTUAL:
1842                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1843                                 um = iptr->sx.s23.s3.um;
1844                                 patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
1845                                 emit_arbitrary_nop(cd, PATCH_ALIGNMENT((uintptr_t) cd->mcodeptr, 6, sizeof(int32_t)));
1846
1847                                 s1 = 0;
1848                         }
1849                         else {
1850                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1851                                 s1 = OFFSET(vftbl_t, table[0]) +
1852                                         sizeof(methodptr) * lm->vftblindex;
1853                         }
1854
1855                         /* implicit null-pointer check */
1856                         M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
1857                         M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
1858                         M_CALL(REG_ITMP3);
1859                         break;
1860
1861                 case ICMD_INVOKEINTERFACE:
1862                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1863                                 um = iptr->sx.s23.s3.um;
1864                                 patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
1865                                 emit_arbitrary_nop(cd, PATCH_ALIGNMENT((uintptr_t) cd->mcodeptr, 6, sizeof(int32_t)));
1866
1867                                 s1 = 0;
1868                                 s2 = 0;
1869                         }
1870                         else {
1871                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1872                                 s1 = OFFSET(vftbl_t, interfacetable[0]) -
1873                                         sizeof(methodptr) * lm->clazz->index;
1874
1875                                 s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
1876                         }
1877
1878                         /* implicit null-pointer check */
1879                         M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
1880                         M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
1881                         if (INSTRUCTION_IS_UNRESOLVED(iptr))
1882                                 emit_arbitrary_nop(cd, PATCH_ALIGNMENT((uintptr_t) cd->mcodeptr, 3, sizeof(int32_t)));
1883                         M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
1884                         M_CALL(REG_ITMP3);
1885                         break;
1886
1887                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
1888
1889                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
1890                                 /* object type cast-check */
1891
1892                                 classinfo *super;
1893                                 s4         superindex;
1894
1895                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1896                                         super      = NULL;
1897                                         superindex = 0;
1898                                 }
1899                                 else {
1900                                         super      = iptr->sx.s23.s3.c.cls;
1901                                         superindex = super->index;
1902                                 }
1903
1904                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1905
1906                                 /* if class is not resolved, check which code to call */
1907
1908                                 if (super == NULL) {
1909                                         M_TEST(s1);
1910                                         emit_label_beq(cd, BRANCH_LABEL_1);
1911
1912                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
1913                                                                                   iptr->sx.s23.s3.c.ref, 0);
1914
1915                                         emit_arbitrary_nop(cd, PATCH_ALIGNMENT((uintptr_t) cd->mcodeptr, 2, sizeof(int32_t)));
1916
1917                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
1918                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
1919                                         emit_label_beq(cd, BRANCH_LABEL_2);
1920                                 }
1921
1922                                 /* interface checkcast code */
1923
1924                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
1925                                         if (super != NULL) {
1926                                                 M_TEST(s1);
1927                                                 emit_label_beq(cd, BRANCH_LABEL_3);
1928                                         }
1929
1930                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
1931
1932                                         if (super == NULL) {
1933                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
1934                                                                                           iptr->sx.s23.s3.c.ref,
1935                                                                                           0);
1936                                                 emit_arbitrary_nop(cd, PATCH_ALIGNMENT((uintptr_t) cd->mcodeptr, 10, sizeof(int32_t)));
1937                                         }
1938
1939                                         M_ILD32(REG_ITMP3,
1940                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
1941                                         M_ICMP_IMM32(superindex, REG_ITMP3);
1942                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
1943
1944                                         if (super == NULL)
1945                                                 emit_arbitrary_nop(cd, PATCH_ALIGNMENT((uintptr_t) cd->mcodeptr, 3, sizeof(int32_t)));
1946                                         M_ALD32(REG_ITMP3, REG_ITMP2,
1947                                                         OFFSET(vftbl_t, interfacetable[0]) -
1948                                                         superindex * sizeof(methodptr*));
1949                                         M_TEST(REG_ITMP3);
1950                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
1951
1952                                         if (super == NULL)
1953                                                 emit_label_br(cd, BRANCH_LABEL_4);
1954                                         else
1955                                                 emit_label(cd, BRANCH_LABEL_3);
1956                                 }
1957
1958                                 /* class checkcast code */
1959
1960                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
1961                                         if (super == NULL) {
1962                                                 emit_label(cd, BRANCH_LABEL_2);
1963
1964                                                 constant_classref *cr = iptr->sx.s23.s3.c.ref;
1965                                                 disp = dseg_add_unique_address(cd, cr);
1966
1967                                                 patcher_add_patch_ref(jd,
1968                                                                                           PATCHER_resolve_classref_to_vftbl,
1969                                                                                           cr, disp);
1970                                         }
1971                                         else {
1972                                                 M_TEST(s1);
1973                                                 emit_label_beq(cd, BRANCH_LABEL_5);
1974
1975                                                 disp = dseg_add_address(cd, super->vftbl);
1976                                         }
1977
1978                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
1979                                         M_ALD(REG_ITMP3, RIP, disp);
1980
1981                                         if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
1982                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
1983                                                 M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
1984                                                 emit_label_beq(cd, BRANCH_LABEL_6);  /* good */
1985
1986                                                 if (super == NULL) {
1987                                                         M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
1988                                                         emit_label_bne(cd, BRANCH_LABEL_10);  /* throw */
1989                                                 }
1990
1991                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
1992                                                 M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
1993                                                 emit_label_bgt(cd, BRANCH_LABEL_9);  /* throw */
1994
1995                                                 M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
1996                                                 M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
1997                                                 emit_label_beq(cd, BRANCH_LABEL_7);  /* good */
1998
1999                                                 emit_label(cd, BRANCH_LABEL_9);
2000                                                 if (super == NULL)
2001                                                         emit_label(cd, BRANCH_LABEL_10);
2002
2003                                                 /* reload s1, might have been destroyed */
2004                                                 emit_load_s1(jd, iptr, REG_ITMP1);
2005                                                 M_ALD_MEM(s1, TRAP_ClassCastException);
2006
2007                                                 emit_label(cd, BRANCH_LABEL_7);
2008                                                 emit_label(cd, BRANCH_LABEL_6);
2009                                                 /* reload s1, might have been destroyed */
2010                                                 emit_load_s1(jd, iptr, REG_ITMP1);
2011                                         }
2012                                         else {
2013                                                 M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
2014                                                 emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
2015                                         }
2016
2017                                         if (super != NULL)
2018                                                 emit_label(cd, BRANCH_LABEL_5);
2019                                 }
2020
2021                                 if (super == NULL) {
2022                                         emit_label(cd, BRANCH_LABEL_1);
2023                                         emit_label(cd, BRANCH_LABEL_4);
2024                                 }
2025
2026                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2027                         }
2028                         else {
2029                                 /* array type cast-check */
2030
2031                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2032                                 M_INTMOVE(s1, REG_A0);
2033
2034                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2035                                         constant_classref *cr = iptr->sx.s23.s3.c.ref;
2036                                         disp = dseg_add_unique_address(cd, cr);
2037
2038                                         patcher_add_patch_ref(jd,
2039                                                                                   PATCHER_resolve_classref_to_classinfo,
2040                                                                                   cr, disp);
2041                                 }
2042                                 else {
2043                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2044                                 }
2045
2046                                 M_ALD(REG_A1, RIP, disp);
2047                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2048                                 M_CALL(REG_ITMP1);
2049
2050                                 /* s1 may have been destroyed over the function call */
2051                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2052                                 M_TEST(REG_RESULT);
2053                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2054
2055                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2056                         }
2057
2058                         M_INTMOVE(s1, d);
2059                         emit_store_dst(jd, iptr, d);
2060                         break;
2061
2062                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2063
2064                         {
2065                         classinfo *super;
2066                         s4         superindex;
2067
2068                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2069                                 super      = NULL;
2070                                 superindex = 0;
2071                         }
2072                         else {
2073                                 super      = iptr->sx.s23.s3.c.cls;
2074                                 superindex = super->index;
2075                         }
2076
2077                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2078                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2079
2080                         if (s1 == d) {
2081                                 M_INTMOVE(s1, REG_ITMP1);
2082                                 s1 = REG_ITMP1;
2083                         }
2084
2085                         M_CLR(d);
2086
2087                         /* if class is not resolved, check which code to call */
2088
2089                         if (super == NULL) {
2090                                 M_TEST(s1);
2091                                 emit_label_beq(cd, BRANCH_LABEL_1);
2092
2093                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2094                                                                           iptr->sx.s23.s3.c.ref, 0);
2095
2096                                 emit_arbitrary_nop(cd, PATCH_ALIGNMENT((uintptr_t) cd->mcodeptr, 2, sizeof(int32_t)));
2097
2098                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2099                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2100                                 emit_label_beq(cd, BRANCH_LABEL_2);
2101                         }
2102
2103                         /* interface instanceof code */
2104
2105                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2106                                 int nops;
2107                                 if (super != NULL) {
2108                                         M_TEST(s1);
2109                                         emit_label_beq(cd, BRANCH_LABEL_3);
2110                                 }
2111
2112                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2113
2114                                 if (super == NULL) {
2115                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2116                                                                                   iptr->sx.s23.s3.c.ref, 0);
2117                                         emit_arbitrary_nop(cd, PATCH_ALIGNMENT((uintptr_t) cd->mcodeptr, 10, sizeof(int32_t)));
2118                                 }
2119
2120                                 M_ILD32(REG_ITMP3,
2121                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2122                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2123
2124                                 nops = super == NULL ? PATCH_ALIGNMENT((uintptr_t) (cd->mcodeptr + 6), 3, sizeof(int32_t)) : 0;
2125
2126                                 int a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */ + nops;
2127
2128                                 M_BLE(a);
2129                                 emit_arbitrary_nop(cd, nops);
2130                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2131                                                 OFFSET(vftbl_t, interfacetable[0]) -
2132                                                 superindex * sizeof(methodptr*));
2133                                 M_TEST(REG_ITMP1);
2134                                 M_SETNE(d);
2135
2136                                 if (super == NULL)
2137                                         emit_label_br(cd, BRANCH_LABEL_4);
2138                                 else
2139                                         emit_label(cd, BRANCH_LABEL_3);
2140                         }
2141
2142                         /* class instanceof code */
2143
2144                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2145                                 if (super == NULL) {
2146                                         emit_label(cd, BRANCH_LABEL_2);
2147
2148                                         constant_classref *cr = iptr->sx.s23.s3.c.ref;
2149                                         disp = dseg_add_unique_address(cd, cr);
2150
2151                                         patcher_add_patch_ref(jd,
2152                                                                                   PATCHER_resolve_classref_to_vftbl,
2153                                                                                   cr, disp);
2154                                 }
2155                                 else {
2156                                         M_TEST(s1);
2157                                         emit_label_beq(cd, BRANCH_LABEL_5);
2158
2159                                         disp = dseg_add_address(cd, super->vftbl);
2160                                 }
2161
2162                                 M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2163                                 M_ALD(REG_ITMP3, RIP, disp);
2164
2165                                 if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2166                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2167                                         M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
2168                                         emit_label_bne(cd, BRANCH_LABEL_8); /* jump over INC/SETE */
2169                                         if (d == REG_ITMP2) {
2170                                                 M_SETE(d);
2171                                                 M_BSEXT(d, d);
2172                                         } else
2173                                                 M_LINC(d);
2174                                         emit_label_br(cd, BRANCH_LABEL_6);  /* true */
2175                                         emit_label(cd, BRANCH_LABEL_8);
2176
2177                                         if (super == NULL) {
2178                                                 M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2179                                                 emit_label_bne(cd, BRANCH_LABEL_10);  /* false */
2180                                         }
2181
2182                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
2183                                         M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
2184                                         emit_label_bgt(cd, BRANCH_LABEL_9);  /* false */
2185
2186                                         M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2187                                         M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
2188                                         M_SETE(d);
2189                                         if (d == REG_ITMP2) {
2190                                                 M_BSEXT(d, d);
2191
2192                                                 emit_label_br(cd, BRANCH_LABEL_7); /* jump over M_CLR */
2193                                         }
2194
2195                                         emit_label(cd, BRANCH_LABEL_9);
2196                                         if (super == NULL)
2197                                                 emit_label(cd, BRANCH_LABEL_10);
2198                                         if (d == REG_ITMP2) {
2199                                                 M_CLR(d);
2200
2201                                                 emit_label(cd, BRANCH_LABEL_7);
2202                                         }
2203                                         emit_label(cd, BRANCH_LABEL_6);
2204                                 }
2205                                 else {
2206                                         M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
2207                                         M_SETE(d);
2208                                         if (d == REG_ITMP2)
2209                                                 M_BSEXT(d, d);
2210                                 }
2211
2212                                 if (super != NULL)
2213                                         emit_label(cd, BRANCH_LABEL_5);
2214                         }
2215
2216                         if (super == NULL) {
2217                                 emit_label(cd, BRANCH_LABEL_1);
2218                                 emit_label(cd, BRANCH_LABEL_4);
2219                         }
2220
2221                         emit_store_dst(jd, iptr, d);
2222                         }
2223                         break;
2224
2225                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2226
2227                         /* check for negative sizes and copy sizes to stack if necessary  */
2228
2229                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2230
2231                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2232
2233                                 /* copy SAVEDVAR sizes to stack */
2234                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2235
2236                                 /* Already Preallocated? */
2237                                 if (!(var->flags & PREALLOC)) {
2238                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2239                                         M_LST(s2, REG_SP, s1 * 8);
2240                                 }
2241                         }
2242
2243                         /* a0 = dimension count */
2244
2245                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2246
2247                         /* is a patcher function set? */
2248
2249                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2250                                 constant_classref *cr = iptr->sx.s23.s3.c.ref;
2251                                 disp = dseg_add_unique_address(cd, cr);
2252
2253                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2254                                                                           cr, disp);
2255                         }
2256                         else {
2257                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2258                         }
2259
2260                         /* a1 = classinfo */
2261
2262                         M_ALD(REG_A1, RIP, disp);
2263
2264                         /* a2 = pointer to dimensions = stack pointer */
2265
2266                         M_MOV(REG_SP, REG_A2);
2267
2268                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2269                         M_CALL(REG_ITMP1);
2270
2271                         /* check for exception before result assignment */
2272
2273                         emit_exception_check(cd, iptr);
2274
2275                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2276                         M_INTMOVE(REG_RESULT, s1);
2277                         emit_store_dst(jd, iptr, s1);
2278                         break;
2279
2280                 default:
2281                         vm_abort("Unknown ICMD %d during code generation", iptr->opc);
2282         } /* switch */
2283 }
2284
2285
2286 /* codegen_emit_stub_native ****************************************************
2287
2288    Emits a stub routine which calls a native method.
2289
2290 *******************************************************************************/
2291
2292 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2293 {
2294         methodinfo  *m;
2295         codeinfo    *code;
2296         codegendata *cd;
2297         methoddesc  *md;
2298         int          i, j;
2299         int          s1, s2;
2300         int          disp;
2301
2302         /* Sanity check. */
2303
2304         assert(f != NULL);
2305
2306         /* Get required compiler data. */
2307
2308         m    = jd->m;
2309         code = jd->code;
2310         cd   = jd->cd;
2311
2312         /* initialize variables */
2313
2314         md = m->parseddesc;
2315
2316         /* calculate stack frame size */
2317
2318         cd->stackframesize =
2319                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2320                 sizeof(localref_table) / SIZEOF_VOID_P +
2321                 md->paramcount +
2322                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2323                 nmd->memuse;
2324
2325         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2326
2327         /* create method header */
2328
2329         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2330         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2331         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2332         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2333         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2334
2335 #if defined(ENABLE_PROFILING)
2336         /* generate native method profiling code */
2337
2338         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2339                 /* count frequency */
2340
2341                 M_MOV_IMM(code, REG_ITMP3);
2342                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2343         }
2344 #endif
2345
2346         /* generate stub code */
2347
2348         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2349
2350 #if defined(ENABLE_GC_CACAO)
2351         /* Save callee saved integer registers in stackframeinfo (GC may
2352            need to recover them during a collection). */
2353
2354         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2355                 OFFSET(stackframeinfo_t, intregs);
2356
2357         for (i = 0; i < INT_SAV_CNT; i++)
2358                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2359 #endif
2360
2361         /* save integer and float argument registers */
2362
2363         for (i = 0; i < md->paramcount; i++) {
2364                 if (!md->params[i].inmemory) {
2365                         s1 = md->params[i].regoff;
2366
2367                         switch (md->paramtypes[i].type) {
2368                         case TYPE_INT:
2369                         case TYPE_LNG:
2370                         case TYPE_ADR:
2371                                 M_LST(s1, REG_SP, i * 8);
2372                                 break;
2373                         case TYPE_FLT:
2374                         case TYPE_DBL:
2375                                 M_DST(s1, REG_SP, i * 8);
2376                                 break;
2377                         }
2378                 }
2379         }
2380
2381         /* create dynamic stack info */
2382
2383         M_MOV(REG_SP, REG_A0);
2384         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2385         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2386         M_CALL(REG_ITMP1);
2387
2388         /* remember class argument */
2389
2390         if (m->flags & ACC_STATIC)
2391                 M_MOV(REG_RESULT, REG_ITMP2);
2392
2393         /* restore integer and float argument registers */
2394
2395         for (i = 0; i < md->paramcount; i++) {
2396                 if (!md->params[i].inmemory) {
2397                         s1 = md->params[i].regoff;
2398
2399                         switch (md->paramtypes[i].type) {
2400                         case TYPE_INT:
2401                         case TYPE_LNG:
2402                         case TYPE_ADR:
2403                                 M_LLD(s1, REG_SP, i * 8);
2404                                 break;
2405                         case TYPE_FLT:
2406                         case TYPE_DBL:
2407                                 M_DLD(s1, REG_SP, i * 8);
2408                                 break;
2409                         }
2410                 }
2411         }
2412
2413         /* Copy or spill arguments to new locations. */
2414
2415         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
2416                 s2 = nmd->params[j].regoff;
2417
2418                 switch (md->paramtypes[i].type) {
2419                 case TYPE_INT:
2420                 case TYPE_LNG:
2421                 case TYPE_ADR:
2422                         if (!md->params[i].inmemory) {
2423                                 s1 = md->params[i].regoff;
2424
2425                                 if (!nmd->params[j].inmemory)
2426                                         M_INTMOVE(s1, s2);
2427                                 else
2428                                         M_LST(s1, REG_SP, s2);
2429                         }
2430                         else {
2431                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2432                                 M_LLD(REG_ITMP1, REG_SP, s1);
2433                                 M_LST(REG_ITMP1, REG_SP, s2);
2434                         }
2435                         break;
2436                 case TYPE_FLT:
2437                         /* We only copy spilled float arguments, as the float
2438                            argument registers keep unchanged. */
2439
2440                         if (md->params[i].inmemory) {
2441                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2442
2443                                 M_FLD(REG_FTMP1, REG_SP, s1);
2444                                 M_FST(REG_FTMP1, REG_SP, s2);
2445                         }
2446                         break;
2447                 case TYPE_DBL:
2448                         if (md->params[i].inmemory) {
2449                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2450                                 M_DLD(REG_FTMP1, REG_SP, s1);
2451                                 M_DST(REG_FTMP1, REG_SP, s2);
2452                         }
2453                         break;
2454                 }
2455         }
2456
2457         /* Handle native Java methods. */
2458
2459         if (m->flags & ACC_NATIVE) {
2460                 /* put class into second argument register */
2461
2462                 if (m->flags & ACC_STATIC)
2463                         M_MOV(REG_ITMP2, REG_A1);
2464
2465                 /* put env into first argument register */
2466
2467                 M_MOV_IMM(VM_get_jnienv(), REG_A0);
2468         }
2469
2470         /* Call the native function. */
2471
2472         disp = dseg_add_functionptr(cd, f);
2473         M_ALD(REG_ITMP1, RIP, disp);
2474         M_CALL(REG_ITMP1);
2475
2476         /* save return value */
2477
2478         switch (md->returntype.type) {
2479         case TYPE_INT:
2480         case TYPE_LNG:
2481         case TYPE_ADR:
2482                 switch (md->returntype.primitivetype) {
2483                 case PRIMITIVETYPE_BOOLEAN:
2484                         M_BZEXT(REG_RESULT, REG_RESULT);
2485                         break;
2486                 case PRIMITIVETYPE_BYTE:
2487                         M_BSEXT(REG_RESULT, REG_RESULT);
2488                         break;
2489                 case PRIMITIVETYPE_CHAR:
2490                         M_CZEXT(REG_RESULT, REG_RESULT);
2491                         break;
2492                 case PRIMITIVETYPE_SHORT:
2493                         M_SSEXT(REG_RESULT, REG_RESULT);
2494                         break;
2495                 }
2496                 M_LST(REG_RESULT, REG_SP, 0 * 8);
2497                 break;
2498         case TYPE_FLT:
2499         case TYPE_DBL:
2500                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
2501                 break;
2502         case TYPE_VOID:
2503                 break;
2504         }
2505
2506         /* remove native stackframe info */
2507
2508         M_MOV(REG_SP, REG_A0);
2509         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2510         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
2511         M_CALL(REG_ITMP1);
2512         M_MOV(REG_RESULT, REG_ITMP3);
2513
2514         /* restore return value */
2515
2516         switch (md->returntype.type) {
2517         case TYPE_INT:
2518         case TYPE_LNG:
2519         case TYPE_ADR:
2520                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
2521                 break;
2522         case TYPE_FLT:
2523         case TYPE_DBL:
2524                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
2525                 break;
2526         case TYPE_VOID:
2527                 break;
2528         }
2529
2530 #if defined(ENABLE_GC_CACAO)
2531         /* Restore callee saved integer registers from stackframeinfo (GC
2532            might have modified them during a collection). */
2533
2534         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2535                 OFFSET(stackframeinfo_t, intregs);
2536
2537         for (i = 0; i < INT_SAV_CNT; i++)
2538                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2539 #endif
2540
2541         /* remove stackframe */
2542
2543         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2544
2545         /* test for exception */
2546
2547         M_TEST(REG_ITMP3);
2548         M_BNE(1);
2549         M_RET;
2550
2551         /* handle exception */
2552
2553         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
2554         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
2555         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
2556
2557         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
2558         M_JMP(REG_ITMP3);
2559 }
2560
2561
2562 /*
2563  * These are local overrides for various environment variables in Emacs.
2564  * Please do not remove this and leave it at the end of the file, where
2565  * Emacs will automagically detect them.
2566  * ---------------------------------------------------------------------
2567  * Local variables:
2568  * mode: c
2569  * indent-tabs-mode: t
2570  * c-basic-offset: 4
2571  * tab-width: 4
2572  * End:
2573  * vim:noexpandtab:sw=4:ts=4:
2574  */