src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007, 2008, 2009
   4    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
   5
   6    This file is part of CACAO.
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2, or (at
  11    your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful, but
  14    WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16    General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, write to the Free Software
  20    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  21    02110-1301, USA.
  22
  23 */
  24
  25
  26 #include "config.h"
  27
  28 #include <assert.h>
  29 #include <stdio.h>
  30 #include <stdint.h>
  31
  32 #include "vm/types.h"
  33 #include "vm/os.hpp"
  34
  35 #include "md-abi.h"
  36
  37 #include "vm/jit/x86_64/arch.h"
  38 #include "vm/jit/x86_64/codegen.h"
  39 #include "vm/jit/x86_64/emit.h"
  40
  41 #include "mm/memory.hpp"
  42
  43 #include "native/localref.hpp"
  44 #include "native/native.hpp"
  45
  46 #include "threads/lock.hpp"
  47
  48 #include "vm/jit/builtin.hpp"
  49 #include "vm/exceptions.hpp"
  50 #include "vm/global.h"
  51 #include "vm/loader.hpp"
  52 #include "vm/options.h"
  53 #include "vm/primitive.hpp"
  54 #include "vm/statistics.h"
  55 #include "vm/string.hpp"
  56 #include "vm/vm.hpp"
  57
  58 #include "vm/jit/abi.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/code.hpp"
  61 #include "vm/jit/codegen-common.hpp"
  62 #include "vm/jit/dseg.h"
  63 #include "vm/jit/emit-common.hpp"
  64 #include "vm/jit/jit.hpp"
  65 #include "vm/jit/linenumbertable.hpp"
  66 #include "vm/jit/methodheader.h"
  67 #include "vm/jit/parse.hpp"
  68 #include "vm/jit/patcher-common.hpp"
  69 #include "vm/jit/reg.h"
  70 #include "vm/jit/stacktrace.hpp"
  71 #include "vm/jit/trap.hpp"
  72
  73
  74 /**
  75  * Generates machine code for the method prolog.
  76  */
  77 void codegen_emit_prolog(jitdata* jd)
  78 {
  79         varinfo*    var;
  80         methoddesc* md;
  81         int32_t     s1;
  82         int32_t     p, t, l;
  83         int32_t     varindex;
  84         int         i;
  85
  86         // Get required compiler data.
  87         methodinfo*   m  = jd->m;
  88         codegendata*  cd = jd->cd;
  89         registerdata* rd = jd->rd;
  90
  91         /* create stack frame (if necessary) */
  92
  93         if (cd->stackframesize)
  94                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
  95
  96         /* save used callee saved registers */
  97
  98         p = cd->stackframesize;
  99         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 100                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 101         }
 102         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 103                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 104         }
 105
 106         /* take arguments out of register or stack frame */
 107
 108         md = m->parseddesc;
 109
 110         for (p = 0, l = 0; p < md->paramcount; p++) {
 111                 t = md->paramtypes[p].type;
 112
 113                 varindex = jd->local_map[l * 5 + t];
 114
 115                 l++;
 116                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 117                         l++;
 118
 119                 if (varindex == UNUSED)
 120                         continue;
 121
 122                 var = VAR(varindex);
 123
 124                 s1 = md->params[p].regoff;
 125
 126                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 127                         if (!md->params[p].inmemory) {           /* register arguments    */
 128                                 if (!IS_INMEMORY(var->flags))
 129                                         M_INTMOVE(s1, var->vv.regoff);
 130                                 else
 131                                     M_LST(s1, REG_SP, var->vv.regoff);
 132                         }
 133                         else {                                 /* stack arguments       */
 134                                 if (!IS_INMEMORY(var->flags))
 135                                         /* + 8 for return address */
 136                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 137                                 else
 138                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 139                         }
 140                 }
 141                 else {                                     /* floating args         */
 142                         if (!md->params[p].inmemory) {           /* register arguments    */
 143                                 if (!IS_INMEMORY(var->flags))
 144                                         emit_fmove(cd, s1, var->vv.regoff);
 145                                 else
 146                                         M_DST(s1, REG_SP, var->vv.regoff);
 147                         }
 148                         else {                                 /* stack arguments       */
 149                                 if (!IS_INMEMORY(var->flags))
 150                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 151                                 else
 152                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 153                         }
 154                 }
 155         }
 156 }
 157
 158
 159 /**
 160  * Generates machine code for the method epilog.
 161  */
 162 void codegen_emit_epilog(jitdata* jd)
 163 {
 164         int32_t p;
 165         int i;
 166
 167         // Get required compiler data.
 168         codegendata*  cd = jd->cd;
 169         registerdata* rd = jd->rd;
 170
 171         p = cd->stackframesize;
 172
 173         /* restore saved registers */
 174
 175         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 176                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
 177         }
 178         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 179                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
 180         }
 181
 182         /* deallocate stack */
 183
 184         if (cd->stackframesize)
 185                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
 186
 187         M_RET;
 188 }
 189
 190
 191 /**
 192  * Generates machine code for one ICMD.
 193  */
 194 void codegen_emit_instruction(jitdata* jd, instruction* iptr)
 195 {
 196         varinfo*            var;
 197         varinfo*            dst;
 198         builtintable_entry* bte;
 199         methodinfo*         lm;             // Local methodinfo for ICMD_INVOKE*.
 200         unresolved_method*  um;
 201         fieldinfo*          fi;
 202         unresolved_field*   uf;
 203         int32_t             fieldtype;
 204         int32_t             s1, s2, s3, d;
 205         int32_t             disp;
 206
 207         // Get required compiler data.
 208         codegendata*  cd = jd->cd;
 209
 210         switch (iptr->opc) {
 211
 212                 /* constant operations ************************************************/
 213
 214                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 215
 216                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 217                         disp = dseg_add_float(cd, iptr->sx.val.f);
 218                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 219                         emit_store_dst(jd, iptr, d);
 220                         break;
 221
 222                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 223
 224                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 225                         disp = dseg_add_double(cd, iptr->sx.val.d);
 226                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 227                         emit_store_dst(jd, iptr, d);
 228                         break;
 229
 230                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 231
 232                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 233
 234                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 235                                 constant_classref *cr = iptr->sx.val.c.ref;
 236                                 disp = dseg_add_unique_address(cd, cr);
 237
 238 /*                              PROFILE_CYCLE_STOP; */
 239
 240                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 241                                                                           cr, disp);
 242
 243 /*                              PROFILE_CYCLE_START; */
 244
 245                                 M_ALD(d, RIP, disp);
 246                         }
 247                         else {
 248                                 if (iptr->sx.val.anyptr == 0) {
 249                                         M_CLR(d);
 250                                 }
 251                                 else {
 252                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 253                                         M_ALD(d, RIP, disp);
 254                                 }
 255                         }
 256                         emit_store_dst(jd, iptr, d);
 257                         break;
 258
 259
 260                 /* integer operations *************************************************/
 261
 262                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 263
 264                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 265                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 266                         M_INTMOVE(s1, d);
 267                         M_INEG(d);
 268                         emit_store_dst(jd, iptr, d);
 269                         break;
 270
 271                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 272
 273                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 274                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 275                         M_INTMOVE(s1, d);
 276                         M_LNEG(d);
 277                         emit_store_dst(jd, iptr, d);
 278                         break;
 279
 280                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 281
 282                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 283                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 284                         M_ISEXT(s1, d);
 285                         emit_store_dst(jd, iptr, d);
 286                         break;
 287
 288                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 289
 290                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 291                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 292                         M_IMOV(s1, d);
 293                         emit_store_dst(jd, iptr, d);
 294                         break;
 295
 296                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 297
 298                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 299                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 300                         M_BSEXT(s1, d);
 301                         emit_store_dst(jd, iptr, d);
 302                         break;
 303
 304                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 305
 306                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 307                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 308                         M_CZEXT(s1, d);
 309                         emit_store_dst(jd, iptr, d);
 310                         break;
 311
 312                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 313
 314                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 315                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 316                         M_SSEXT(s1, d);
 317                         emit_store_dst(jd, iptr, d);
 318                         break;
 319
 320
 321                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 322
 323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 324                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 325                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 326                         if (s2 == d)
 327                                 M_IADD(s1, d);
 328                         else {
 329                                 M_INTMOVE(s1, d);
 330                                 M_IADD(s2, d);
 331                         }
 332                         emit_store_dst(jd, iptr, d);
 333                         break;
 334
 335                 case ICMD_IINC:
 336                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 337                                       /* sx.val.i = constant                             */
 338
 339                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 340                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 341
 342                         /* Using inc and dec is not faster than add (tested with
 343                            sieve). */
 344
 345                         M_INTMOVE(s1, d);
 346                         M_IADD_IMM(iptr->sx.val.i, d);
 347                         emit_store_dst(jd, iptr, d);
 348                         break;
 349
 350                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 351
 352                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 353                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 354                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 355                         if (s2 == d)
 356                                 M_LADD(s1, d);
 357                         else {
 358                                 M_INTMOVE(s1, d);
 359                                 M_LADD(s2, d);
 360                         }
 361                         emit_store_dst(jd, iptr, d);
 362                         break;
 363
 364                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 365                                       /* sx.val.l = constant                             */
 366
 367                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 368                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 369                         M_INTMOVE(s1, d);
 370                         if (IS_IMM32(iptr->sx.val.l))
 371                                 M_LADD_IMM(iptr->sx.val.l, d);
 372                         else {
 373                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 374                                 M_LADD(REG_ITMP2, d);
 375                         }
 376                         emit_store_dst(jd, iptr, d);
 377                         break;
 378
 379                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 380
 381                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 382                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 383                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 384                         if (s2 == d) {
 385                                 M_INTMOVE(s1, REG_ITMP1);
 386                                 M_ISUB(s2, REG_ITMP1);
 387                                 M_INTMOVE(REG_ITMP1, d);
 388                         } else {
 389                                 M_INTMOVE(s1, d);
 390                                 M_ISUB(s2, d);
 391                         }
 392                         emit_store_dst(jd, iptr, d);
 393                         break;
 394
 395                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 396                                       /* sx.val.i = constant                             */
 397
 398                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 399                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 400                         M_INTMOVE(s1, d);
 401                         M_ISUB_IMM(iptr->sx.val.i, d);
 402                         emit_store_dst(jd, iptr, d);
 403                         break;
 404
 405                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 406
 407                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 408                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 409                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 410                         if (s2 == d) {
 411                                 M_INTMOVE(s1, REG_ITMP1);
 412                                 M_LSUB(s2, REG_ITMP1);
 413                                 M_INTMOVE(REG_ITMP1, d);
 414                         } else {
 415                                 M_INTMOVE(s1, d);
 416                                 M_LSUB(s2, d);
 417                         }
 418                         emit_store_dst(jd, iptr, d);
 419                         break;
 420
 421                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 422                                       /* sx.val.l = constant                             */
 423
 424                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 425                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 426                         M_INTMOVE(s1, d);
 427                         if (IS_IMM32(iptr->sx.val.l))
 428                                 M_LSUB_IMM(iptr->sx.val.l, d);
 429                         else {
 430                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 431                                 M_LSUB(REG_ITMP2, d);
 432                         }
 433                         emit_store_dst(jd, iptr, d);
 434                         break;
 435
 436                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 437
 438                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 439                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 440                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 441                         if (s2 == d)
 442                                 M_IMUL(s1, d);
 443                         else {
 444                                 M_INTMOVE(s1, d);
 445                                 M_IMUL(s2, d);
 446                         }
 447                         emit_store_dst(jd, iptr, d);
 448                         break;
 449
 450                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 451                                       /* sx.val.i = constant                             */
 452
 453                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 454                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 455                         if (iptr->sx.val.i == 2) {
 456                                 M_INTMOVE(s1, d);
 457                                 M_ISLL_IMM(1, d);
 458                         } else
 459                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 460                         emit_store_dst(jd, iptr, d);
 461                         break;
 462
 463                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 464
 465                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 466                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 467                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 468                         if (s2 == d)
 469                                 M_LMUL(s1, d);
 470                         else {
 471                                 M_INTMOVE(s1, d);
 472                                 M_LMUL(s2, d);
 473                         }
 474                         emit_store_dst(jd, iptr, d);
 475                         break;
 476
 477                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 478                                       /* sx.val.l = constant                             */
 479
 480                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 481                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 482                         if (IS_IMM32(iptr->sx.val.l))
 483                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 484                         else {
 485                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 486                                 M_INTMOVE(s1, d);
 487                                 M_LMUL(REG_ITMP2, d);
 488                         }
 489                         emit_store_dst(jd, iptr, d);
 490                         break;
 491
 492                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 493
 494                         s1 = emit_load_s1(jd, iptr, RAX);
 495                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 496                         d = codegen_reg_of_dst(jd, iptr, RAX);
 497
 498                         M_INTMOVE(s1, RAX);
 499                         M_INTMOVE(s2, REG_ITMP3);
 500                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 501
 502                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 503
 504                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 505                         M_BNE(4 + 6);
 506                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 507                         M_BEQ(1 + 3);                                          /* 6 bytes */
 508
 509                         emit_cltd(cd);                                         /* 1 byte  */
 510                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 511
 512                         M_INTMOVE(RAX, d);
 513                         emit_store_dst(jd, iptr, d);
 514                         dst = VAROP(iptr->dst);
 515                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 516                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 517                         break;
 518
 519                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 520
 521                         s1 = emit_load_s1(jd, iptr, RAX);
 522                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 523                         d = codegen_reg_of_dst(jd, iptr, RDX);
 524
 525                         M_INTMOVE(s1, RAX);
 526                         M_INTMOVE(s2, REG_ITMP3);
 527                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 528
 529                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 530
 531                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 532                         M_BNE(3 + 4 + 6);
 533                         M_CLR(RDX);                                            /* 3 bytes */
 534                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 535                         M_BEQ(1 + 3);                                          /* 6 bytes */
 536
 537                         emit_cltd(cd);                                         /* 1 byte  */
 538                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 539
 540                         M_INTMOVE(RDX, d);
 541                         emit_store_dst(jd, iptr, d);
 542                         dst = VAROP(iptr->dst);
 543                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 544                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 545                         break;
 546
 547                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 548                                       /* sx.val.i = constant                             */
 549
 550                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 551                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 552                         M_INTMOVE(s1, REG_ITMP1);
 553                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 554                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 555                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 556                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 557                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 558                         emit_store_dst(jd, iptr, d);
 559                         break;
 560
 561                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 562                                       /* sx.val.i = constant                             */
 563
 564                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 565                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 566                         M_INTMOVE(s1, REG_ITMP1);
 567                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 568                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 569                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 570                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 571                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 572                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 573                         emit_store_dst(jd, iptr, d);
 574                         break;
 575
 576
 577                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 578
 579                         s1 = emit_load_s1(jd, iptr, RAX);
 580                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 581                         d = codegen_reg_of_dst(jd, iptr, RAX);
 582
 583                         M_INTMOVE(s1, RAX);
 584                         M_INTMOVE(s2, REG_ITMP3);
 585                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 586
 587                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 588
 589                         /* check as described in jvm spec */
 590                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 591                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 592                         M_BNE(4 + 6);
 593                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 594                         M_BEQ(2 + 3);                                          /* 6 bytes */
 595
 596                         emit_cqto(cd);                                         /* 2 bytes */
 597                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 598
 599                         M_INTMOVE(RAX, d);
 600                         emit_store_dst(jd, iptr, d);
 601                         dst = VAROP(iptr->dst);
 602                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 603                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 604                         break;
 605
 606                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 607
 608                         s1 = emit_load_s1(jd, iptr, RAX);
 609                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 610                         d = codegen_reg_of_dst(jd, iptr, RDX);
 611
 612                         M_INTMOVE(s1, RAX);
 613                         M_INTMOVE(s2, REG_ITMP3);
 614                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 615
 616                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 617
 618                         /* check as described in jvm spec */
 619                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 620                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 621                         M_BNE(3 + 4 + 6);
 622                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 623                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 624                         M_BEQ(2 + 3);                                          /* 6 bytes */
 625
 626                         emit_cqto(cd);                                         /* 2 bytes */
 627                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 628
 629                         M_INTMOVE(RDX, d);
 630                         emit_store_dst(jd, iptr, d);
 631                         dst = VAROP(iptr->dst);
 632                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 633                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 634                         break;
 635
 636                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 637                                       /* sx.val.i = constant                             */
 638
 639                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 640                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 641                         M_INTMOVE(s1, REG_ITMP1);
 642                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 643                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 644                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 645                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 646                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 647                         emit_store_dst(jd, iptr, d);
 648                         break;
 649
 650                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 651                                       /* sx.val.l = constant                             */
 652
 653                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 654                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 655                         M_INTMOVE(s1, REG_ITMP1);
 656                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 657                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 658                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 659                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 660                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 661                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 662                         emit_store_dst(jd, iptr, d);
 663                         break;
 664
 665                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 666
 667                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 668                         emit_ishift(jd, SHIFT_SHL, iptr);
 669                         break;
 670
 671                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 672                                       /* sx.val.i = constant                             */
 673
 674                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 675                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 676                         M_INTMOVE(s1, d);
 677                         M_ISLL_IMM(iptr->sx.val.i, d);
 678                         emit_store_dst(jd, iptr, d);
 679                         break;
 680
 681                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 682
 683                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 684                         emit_ishift(jd, SHIFT_SAR, iptr);
 685                         break;
 686
 687                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 688                                       /* sx.val.i = constant                             */
 689
 690                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 691                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 692                         M_INTMOVE(s1, d);
 693                         M_ISRA_IMM(iptr->sx.val.i, d);
 694                         emit_store_dst(jd, iptr, d);
 695                         break;
 696
 697                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 698
 699                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 700                         emit_ishift(jd, SHIFT_SHR, iptr);
 701                         break;
 702
 703                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 704                                       /* sx.val.i = constant                             */
 705
 706                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 707                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 708                         M_INTMOVE(s1, d);
 709                         M_ISRL_IMM(iptr->sx.val.i, d);
 710                         emit_store_dst(jd, iptr, d);
 711                         break;
 712
 713                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 714
 715                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 716                         emit_lshift(jd, SHIFT_SHL, iptr);
 717                         break;
 718
 719         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 720                                           /* sx.val.i = constant                             */
 721
 722                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 723                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 724                         M_INTMOVE(s1, d);
 725                         M_LSLL_IMM(iptr->sx.val.i, d);
 726                         emit_store_dst(jd, iptr, d);
 727                         break;
 728
 729                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 730
 731                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 732                         emit_lshift(jd, SHIFT_SAR, iptr);
 733                         break;
 734
 735                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 736                                       /* sx.val.i = constant                             */
 737
 738                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 739                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 740                         M_INTMOVE(s1, d);
 741                         M_LSRA_IMM(iptr->sx.val.i, d);
 742                         emit_store_dst(jd, iptr, d);
 743                         break;
 744
 745                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 746
 747                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 748                         emit_lshift(jd, SHIFT_SHR, iptr);
 749                         break;
 750
 751                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 752                                       /* sx.val.l = constant                             */
 753
 754                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 755                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 756                         M_INTMOVE(s1, d);
 757                         M_LSRL_IMM(iptr->sx.val.i, d);
 758                         emit_store_dst(jd, iptr, d);
 759                         break;
 760
 761                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
 762
 763                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 764                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 765                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 766                         if (s2 == d)
 767                                 M_IAND(s1, d);
 768                         else {
 769                                 M_INTMOVE(s1, d);
 770                                 M_IAND(s2, d);
 771                         }
 772                         emit_store_dst(jd, iptr, d);
 773                         break;
 774
 775                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
 776                                       /* sx.val.i = constant                             */
 777
 778                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 779                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 780                         M_INTMOVE(s1, d);
 781                         M_IAND_IMM(iptr->sx.val.i, d);
 782                         emit_store_dst(jd, iptr, d);
 783                         break;
 784
 785                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
 786
 787                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 788                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 789                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 790                         if (s2 == d)
 791                                 M_LAND(s1, d);
 792                         else {
 793                                 M_INTMOVE(s1, d);
 794                                 M_LAND(s2, d);
 795                         }
 796                         emit_store_dst(jd, iptr, d);
 797                         break;
 798
 799                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
 800                                       /* sx.val.l = constant                             */
 801
 802                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 803                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 804                         M_INTMOVE(s1, d);
 805                         if (IS_IMM32(iptr->sx.val.l))
 806                                 M_LAND_IMM(iptr->sx.val.l, d);
 807                         else {
 808                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 809                                 M_LAND(REG_ITMP2, d);
 810                         }
 811                         emit_store_dst(jd, iptr, d);
 812                         break;
 813
 814                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
 815
 816                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 817                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 818                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 819                         if (s2 == d)
 820                                 M_IOR(s1, d);
 821                         else {
 822                                 M_INTMOVE(s1, d);
 823                                 M_IOR(s2, d);
 824                         }
 825                         emit_store_dst(jd, iptr, d);
 826                         break;
 827
 828                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
 829                                       /* sx.val.i = constant                             */
 830
 831                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 832                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 833                         M_INTMOVE(s1, d);
 834                         M_IOR_IMM(iptr->sx.val.i, d);
 835                         emit_store_dst(jd, iptr, d);
 836                         break;
 837
 838                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
 839
 840                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 841                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 842                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 843                         if (s2 == d)
 844                                 M_LOR(s1, d);
 845                         else {
 846                                 M_INTMOVE(s1, d);
 847                                 M_LOR(s2, d);
 848                         }
 849                         emit_store_dst(jd, iptr, d);
 850                         break;
 851
 852                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
 853                                       /* sx.val.l = constant                             */
 854
 855                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 856                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 857                         M_INTMOVE(s1, d);
 858                         if (IS_IMM32(iptr->sx.val.l))
 859                                 M_LOR_IMM(iptr->sx.val.l, d);
 860                         else {
 861                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 862                                 M_LOR(REG_ITMP2, d);
 863                         }
 864                         emit_store_dst(jd, iptr, d);
 865                         break;
 866
 867                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
 868
 869                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 870                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 871                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 872                         if (s2 == d)
 873                                 M_IXOR(s1, d);
 874                         else {
 875                                 M_INTMOVE(s1, d);
 876                                 M_IXOR(s2, d);
 877                         }
 878                         emit_store_dst(jd, iptr, d);
 879                         break;
 880
 881                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
 882                                       /* sx.val.i = constant                             */
 883
 884                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 885                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 886                         M_INTMOVE(s1, d);
 887                         M_IXOR_IMM(iptr->sx.val.i, d);
 888                         emit_store_dst(jd, iptr, d);
 889                         break;
 890
 891                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
 892
 893                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 894                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 895                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 896                         if (s2 == d)
 897                                 M_LXOR(s1, d);
 898                         else {
 899                                 M_INTMOVE(s1, d);
 900                                 M_LXOR(s2, d);
 901                         }
 902                         emit_store_dst(jd, iptr, d);
 903                         break;
 904
 905                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
 906                                       /* sx.val.l = constant                             */
 907
 908                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 909                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 910                         M_INTMOVE(s1, d);
 911                         if (IS_IMM32(iptr->sx.val.l))
 912                                 M_LXOR_IMM(iptr->sx.val.l, d);
 913                         else {
 914                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 915                                 M_LXOR(REG_ITMP2, d);
 916                         }
 917                         emit_store_dst(jd, iptr, d);
 918                         break;
 919
 920
 921                 /* floating operations ************************************************/
 922
 923                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
 924
 925                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 926                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 927                         disp = dseg_add_s4(cd, 0x80000000);
 928                         emit_fmove(cd, s1, d);
 929                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
 930                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
 931                         emit_store_dst(jd, iptr, d);
 932                         break;
 933
 934                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
 935
 936                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 937                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 938                         disp = dseg_add_s8(cd, 0x8000000000000000);
 939                         emit_fmove(cd, s1, d);
 940                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
 941                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
 942                         emit_store_dst(jd, iptr, d);
 943                         break;
 944
 945                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 946
 947                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 948                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 949                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 950                         if (s2 == d)
 951                                 M_FADD(s1, d);
 952                         else {
 953                                 emit_fmove(cd, s1, d);
 954                                 M_FADD(s2, d);
 955                         }
 956                         emit_store_dst(jd, iptr, d);
 957                         break;
 958
 959                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 960
 961                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 962                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 963                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 964                         if (s2 == d)
 965                                 M_DADD(s1, d);
 966                         else {
 967                                 emit_fmove(cd, s1, d);
 968                                 M_DADD(s2, d);
 969                         }
 970                         emit_store_dst(jd, iptr, d);
 971                         break;
 972
 973                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 974
 975                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 976                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 977                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 978                         if (s2 == d) {
 979                                 emit_fmove(cd, s2, REG_FTMP2);
 980                                 s2 = REG_FTMP2;
 981                         }
 982                         emit_fmove(cd, s1, d);
 983                         M_FSUB(s2, d);
 984                         emit_store_dst(jd, iptr, d);
 985                         break;
 986
 987                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 988
 989                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 990                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
 991                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
 992                         if (s2 == d) {
 993                                 emit_fmove(cd, s2, REG_FTMP2);
 994                                 s2 = REG_FTMP2;
 995                         }
 996                         emit_fmove(cd, s1, d);
 997                         M_DSUB(s2, d);
 998                         emit_store_dst(jd, iptr, d);
 999                         break;
1000
1001                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1002
1003                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1004                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1005                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1006                         if (s2 == d)
1007                                 M_FMUL(s1, d);
1008                         else {
1009                                 emit_fmove(cd, s1, d);
1010                                 M_FMUL(s2, d);
1011                         }
1012                         emit_store_dst(jd, iptr, d);
1013                         break;
1014
1015                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1016
1017                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1018                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1019                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1020                         if (s2 == d)
1021                                 M_DMUL(s1, d);
1022                         else {
1023                                 emit_fmove(cd, s1, d);
1024                                 M_DMUL(s2, d);
1025                         }
1026                         emit_store_dst(jd, iptr, d);
1027                         break;
1028
1029                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1030
1031                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1032                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1033                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1034                         if (s2 == d) {
1035                                 emit_fmove(cd, s2, REG_FTMP2);
1036                                 s2 = REG_FTMP2;
1037                         }
1038                         emit_fmove(cd, s1, d);
1039                         M_FDIV(s2, d);
1040                         emit_store_dst(jd, iptr, d);
1041                         break;
1042
1043                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1044
1045                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1046                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1047                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1048                         if (s2 == d) {
1049                                 emit_fmove(cd, s2, REG_FTMP2);
1050                                 s2 = REG_FTMP2;
1051                         }
1052                         emit_fmove(cd, s1, d);
1053                         M_DDIV(s2, d);
1054                         emit_store_dst(jd, iptr, d);
1055                         break;
1056
1057                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1058
1059                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1060                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1061                         M_CVTIF(s1, d);
1062                         emit_store_dst(jd, iptr, d);
1063                         break;
1064
1065                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1066
1067                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1068                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1069                         M_CVTID(s1, d);
1070                         emit_store_dst(jd, iptr, d);
1071                         break;
1072
1073                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1074
1075                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1076                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1077                         M_CVTLF(s1, d);
1078                         emit_store_dst(jd, iptr, d);
1079                         break;
1080
1081                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1082
1083                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1084                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1085                         M_CVTLD(s1, d);
1086                         emit_store_dst(jd, iptr, d);
1087                         break;
1088
1089                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1090
1091                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1092                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1093                         M_CVTFI(s1, d);
1094                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1095                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1096                                 ((REG_RESULT == d) ? 0 : 3);
1097                         M_BNE(disp);
1098                         emit_fmove(cd, s1, REG_FTMP1);
1099                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1100                         M_CALL(REG_ITMP2);
1101                         M_INTMOVE(REG_RESULT, d);
1102                         emit_store_dst(jd, iptr, d);
1103                         break;
1104
1105                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1106
1107                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1108                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1109                         M_CVTDI(s1, d);
1110                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1111                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1112                                 ((REG_RESULT == d) ? 0 : 3);
1113                         M_BNE(disp);
1114                         emit_fmove(cd, s1, REG_FTMP1);
1115                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1116                         M_CALL(REG_ITMP2);
1117                         M_INTMOVE(REG_RESULT, d);
1118                         emit_store_dst(jd, iptr, d);
1119                         break;
1120
1121                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1122
1123                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1124                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1125                         M_CVTFL(s1, d);
1126                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1127                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1128                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1129                                 ((REG_RESULT == d) ? 0 : 3);
1130                         M_BNE(disp);
1131                         emit_fmove(cd, s1, REG_FTMP1);
1132                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1133                         M_CALL(REG_ITMP2);
1134                         M_INTMOVE(REG_RESULT, d);
1135                         emit_store_dst(jd, iptr, d);
1136                         break;
1137
1138                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1139
1140                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1141                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1142                         M_CVTDL(s1, d);
1143                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1144                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1145                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1146                                 ((REG_RESULT == d) ? 0 : 3);
1147                         M_BNE(disp);
1148                         emit_fmove(cd, s1, REG_FTMP1);
1149                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1150                         M_CALL(REG_ITMP2);
1151                         M_INTMOVE(REG_RESULT, d);
1152                         emit_store_dst(jd, iptr, d);
1153                         break;
1154
1155                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1156
1157                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1158                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1159                         M_CVTFD(s1, d);
1160                         emit_store_dst(jd, iptr, d);
1161                         break;
1162
1163                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1164
1165                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1166                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1167                         M_CVTDF(s1, d);
1168                         emit_store_dst(jd, iptr, d);
1169                         break;
1170
1171                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1172                                           /* == => 0, < => 1, > => -1 */
1173
1174                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1175                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1176                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1177                         M_CLR(d);
1178                         M_MOV_IMM(1, REG_ITMP1);
1179                         M_MOV_IMM(-1, REG_ITMP2);
1180                         emit_ucomiss_reg_reg(cd, s1, s2);
1181                         M_CMOVULT(REG_ITMP1, d);
1182                         M_CMOVUGT(REG_ITMP2, d);
1183                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1184                         emit_store_dst(jd, iptr, d);
1185                         break;
1186
1187                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1188                                           /* == => 0, < => 1, > => -1 */
1189
1190                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1191                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1192                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1193                         M_CLR(d);
1194                         M_MOV_IMM(1, REG_ITMP1);
1195                         M_MOV_IMM(-1, REG_ITMP2);
1196                         emit_ucomiss_reg_reg(cd, s1, s2);
1197                         M_CMOVULT(REG_ITMP1, d);
1198                         M_CMOVUGT(REG_ITMP2, d);
1199                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1200                         emit_store_dst(jd, iptr, d);
1201                         break;
1202
1203                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1204                                           /* == => 0, < => 1, > => -1 */
1205
1206                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1207                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1208                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1209                         M_CLR(d);
1210                         M_MOV_IMM(1, REG_ITMP1);
1211                         M_MOV_IMM(-1, REG_ITMP2);
1212                         emit_ucomisd_reg_reg(cd, s1, s2);
1213                         M_CMOVULT(REG_ITMP1, d);
1214                         M_CMOVUGT(REG_ITMP2, d);
1215                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1216                         emit_store_dst(jd, iptr, d);
1217                         break;
1218
1219                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1220                                           /* == => 0, < => 1, > => -1 */
1221
1222                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1223                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1224                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1225                         M_CLR(d);
1226                         M_MOV_IMM(1, REG_ITMP1);
1227                         M_MOV_IMM(-1, REG_ITMP2);
1228                         emit_ucomisd_reg_reg(cd, s1, s2);
1229                         M_CMOVULT(REG_ITMP1, d);
1230                         M_CMOVUGT(REG_ITMP2, d);
1231                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1232                         emit_store_dst(jd, iptr, d);
1233                         break;
1234
1235
1236                 /* memory operations **************************************************/
1237
1238                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1239
1240                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1241                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1242                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1243                         /* implicit null-pointer check */
1244                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1245                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1246                         emit_store_dst(jd, iptr, d);
1247                         break;
1248
1249                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1250
1251                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1252                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1253                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1254                         /* implicit null-pointer check */
1255                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1256                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1257                         emit_store_dst(jd, iptr, d);
1258                         break;
1259
1260                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1261
1262                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1263                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1264                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1265                         /* implicit null-pointer check */
1266                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1267                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1268                         emit_store_dst(jd, iptr, d);
1269                         break;
1270
1271                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1272
1273                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1274                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1275                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1276                         /* implicit null-pointer check */
1277                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1278                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1279                         emit_store_dst(jd, iptr, d);
1280                         break;
1281
1282                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1283
1284                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1285                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1286                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1287                         /* implicit null-pointer check */
1288                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1289                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1290                         emit_store_dst(jd, iptr, d);
1291                         break;
1292
1293                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1294
1295                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1296                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1297                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1298                         /* implicit null-pointer check */
1299                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1300                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1301                         emit_store_dst(jd, iptr, d);
1302                         break;
1303
1304                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1305
1306                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1307                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1308                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1309                         /* implicit null-pointer check */
1310                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1311                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1312                         emit_store_dst(jd, iptr, d);
1313                         break;
1314
1315                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1316
1317                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1318                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1319                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1320                         /* implicit null-pointer check */
1321                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1322                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1323                         emit_store_dst(jd, iptr, d);
1324                         break;
1325
1326
1327                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1328
1329                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1330                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1331                         /* implicit null-pointer check */
1332                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1333                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1334                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1335                         break;
1336
1337                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1340                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1341                         /* implicit null-pointer check */
1342                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1343                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1344                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1345                         break;
1346
1347                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1348
1349                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1350                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1351                         /* implicit null-pointer check */
1352                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1353                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1354                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1355                         break;
1356
1357                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1358
1359                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1360                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1361                         /* implicit null-pointer check */
1362                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1363                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1364                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1365                         break;
1366
1367                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1368
1369                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1370                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1371                         /* implicit null-pointer check */
1372                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1373                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1374                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1375                         break;
1376
1377                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1378
1379                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1380                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1381                         /* implicit null-pointer check */
1382                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1383                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1384                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1385                         break;
1386
1387                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1388
1389                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1390                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1391                         /* implicit null-pointer check */
1392                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1393                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1394                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1395                         break;
1396
1397                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1398
1399                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1400                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1401                         /* implicit null-pointer check */
1402                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1403                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1404
1405                         M_MOV(s1, REG_A0);
1406                         M_MOV(s3, REG_A1);
1407                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1408                         M_CALL(REG_ITMP1);
1409                         emit_arraystore_check(cd, iptr);
1410
1411                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1412                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1413                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1414                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1415                         break;
1416
1417
1418                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1419
1420                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1421                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1422                         /* implicit null-pointer check */
1423                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1424                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1425                         break;
1426
1427                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1428
1429                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1430                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1431                         /* implicit null-pointer check */
1432                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1433                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1434                         break;
1435
1436                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1437
1438                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1439                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1440                         /* implicit null-pointer check */
1441                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1442                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1443                         break;
1444
1445                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1446
1447                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1448                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1449                         /* implicit null-pointer check */
1450                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1451                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1452                         break;
1453
1454                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1455
1456                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1457                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1458                         /* implicit null-pointer check */
1459                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1460
1461                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1462                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1463                         }
1464                         else {
1465                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1466                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1467                         }
1468                         break;
1469
1470                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1471
1472                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1473                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1474                         /* implicit null-pointer check */
1475                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1476                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1477                         break;
1478
1479                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1480                                           /* val = value (in current instruction)     */
1481                                           /* following NOP)                           */
1482
1483                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1484                                 uf        = iptr->sx.s23.s3.uf;
1485                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1486                                 disp      = dseg_add_unique_address(cd, uf);
1487
1488 /*                              PROFILE_CYCLE_STOP; */
1489
1490                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1491
1492 /*                              PROFILE_CYCLE_START; */
1493                         }
1494                         else {
1495                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1496                                 fieldtype = fi->type;
1497                                 disp      = dseg_add_address(cd, fi->value);
1498
1499                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1500                                         //PROFILE_CYCLE_STOP;
1501
1502                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1503                                                                                   fi->clazz, 0);
1504
1505                                         //PROFILE_CYCLE_START;
1506                                 }
1507                         }
1508
1509                         /* This approach is much faster than moving the field
1510                            address inline into a register. */
1511
1512                         M_ALD(REG_ITMP1, RIP, disp);
1513
1514                         switch (fieldtype) {
1515                         case TYPE_INT:
1516                         case TYPE_FLT:
1517                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1518                                 break;
1519                         case TYPE_LNG:
1520                         case TYPE_ADR:
1521                         case TYPE_DBL:
1522                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1523                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1524                                 else {
1525                                         M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1526                                         M_LST(REG_ITMP2, REG_ITMP1, 0);
1527                                 }
1528                                 break;
1529                         }
1530                         break;
1531
1532                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1533
1534                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1535
1536                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1537                                 uf        = iptr->sx.s23.s3.uf;
1538                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1539                                 disp      = 0;
1540
1541 /*                              PROFILE_CYCLE_STOP; */
1542
1543                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1544
1545 /*                              PROFILE_CYCLE_START; */
1546                         }
1547                         else {
1548                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1549                                 fieldtype = fi->type;
1550                                 disp      = fi->offset;
1551                         }
1552
1553                         /* implicit null-pointer check */
1554                         switch (fieldtype) {
1555                         case TYPE_INT:
1556                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1557                                 M_ILD32(d, s1, disp);
1558                                 break;
1559                         case TYPE_LNG:
1560                         case TYPE_ADR:
1561                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1562                                 M_LLD32(d, s1, disp);
1563                                 break;
1564                         case TYPE_FLT:
1565                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1566                                 M_FLD32(d, s1, disp);
1567                                 break;
1568                         case TYPE_DBL:
1569                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1570                                 M_DLD32(d, s1, disp);
1571                                 break;
1572                         }
1573                         emit_store_dst(jd, iptr, d);
1574                         break;
1575
1576                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1577
1578                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1579                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1580
1581                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1582                                 uf        = iptr->sx.s23.s3.uf;
1583                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1584                                 disp      = 0;
1585
1586 /*                              PROFILE_CYCLE_STOP; */
1587
1588                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1589
1590 /*                              PROFILE_CYCLE_START; */
1591                         }
1592                         else {
1593                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1594                                 fieldtype = fi->type;
1595                                 disp      = fi->offset;
1596                         }
1597
1598                         /* implicit null-pointer check */
1599                         switch (fieldtype) {
1600                         case TYPE_INT:
1601                                 M_IST32(s2, s1, disp);
1602                                 break;
1603                         case TYPE_LNG:
1604                         case TYPE_ADR:
1605                                 M_LST32(s2, s1, disp);
1606                                 break;
1607                         case TYPE_FLT:
1608                                 M_FST32(s2, s1, disp);
1609                                 break;
1610                         case TYPE_DBL:
1611                                 M_DST32(s2, s1, disp);
1612                                 break;
1613                         }
1614                         break;
1615
1616                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1617                                           /* val = value (in current instruction)     */
1618                                           /* following NOP)                           */
1619
1620                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1621
1622                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1623                                 uf        = iptr->sx.s23.s3.uf;
1624                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1625                                 disp      = 0;
1626
1627 /*                              PROFILE_CYCLE_STOP; */
1628
1629                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1630
1631 /*                              PROFILE_CYCLE_START; */
1632                         }
1633                         else {
1634                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1635                                 fieldtype = fi->type;
1636                                 disp      = fi->offset;
1637                         }
1638
1639                         /* implicit null-pointer check */
1640                         switch (fieldtype) {
1641                         case TYPE_INT:
1642                         case TYPE_FLT:
1643                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
1644                                 break;
1645                         case TYPE_LNG:
1646                         case TYPE_ADR:
1647                         case TYPE_DBL:
1648                                 /* XXX why no check for IS_IMM32? -- probably because of the patcher */
1649                                 M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1650                                 if (disp)  /* resolved, disp can never be 0 */
1651                                         M_LST(REG_ITMP2, s1, disp);
1652                                 else       /* unresolved */
1653                                         M_LST32(REG_ITMP2, s1, disp);
1654                                 break;
1655                         }
1656                         break;
1657
1658
1659                 /* branch operations **************************************************/
1660
1661                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
1662
1663                         M_CALL_IMM(0);                            /* passing exception pc */
1664                         M_POP(REG_ITMP2_XPC);
1665
1666                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
1667                         M_JMP(REG_ITMP3);
1668                         break;
1669
1670                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
1671                 case ICMD_IF_LNE:
1672                 case ICMD_IF_LLT:
1673                 case ICMD_IF_LGE:
1674                 case ICMD_IF_LGT:
1675                 case ICMD_IF_LLE:
1676
1677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1678                         if (IS_IMM32(iptr->sx.val.l))
1679                                 M_LCMP_IMM(iptr->sx.val.l, s1);
1680                         else {
1681                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1682                                 M_LCMP(REG_ITMP2, s1);
1683                         }
1684                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
1685                         break;
1686
1687                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
1688                 case ICMD_IF_LCMPNE:
1689                 case ICMD_IF_LCMPLT:
1690                 case ICMD_IF_LCMPGE:
1691                 case ICMD_IF_LCMPGT:
1692                 case ICMD_IF_LCMPLE:
1693
1694                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1695                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1696                         M_LCMP(s2, s1);
1697                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
1698                         break;
1699
1700                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
1701                         {
1702                                 s4 i, l;
1703                                 branch_target_t *table;
1704
1705                                 table = iptr->dst.table;
1706
1707                                 l = iptr->sx.s23.s2.tablelow;
1708                                 i = iptr->sx.s23.s3.tablehigh;
1709
1710                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1711                                 M_INTMOVE(s1, REG_ITMP1);
1712
1713                                 if (l != 0)
1714                                         M_ISUB_IMM(l, REG_ITMP1);
1715
1716                                 /* number of targets */
1717                                 i = i - l + 1;
1718
1719                 /* range check */
1720
1721                                 M_ICMP_IMM(i - 1, REG_ITMP1);
1722                                 emit_bugt(cd, table[0].block);
1723
1724                                 /* build jump table top down and use address of lowest entry */
1725
1726                                 table += i;
1727
1728                                 while (--i >= 0) {
1729                                         dseg_add_target(cd, table->block);
1730                                         --table;
1731                                 }
1732
1733                                 /* length of dataseg after last dseg_add_target is used
1734                                    by load */
1735
1736                                 M_MOV_IMM(0, REG_ITMP2);
1737                                 dseg_adddata(cd);
1738                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
1739                                 M_JMP(REG_ITMP1);
1740                         }
1741                         break;
1742
1743                 case ICMD_BUILTIN:
1744                         bte = iptr->sx.s23.s3.bte;
1745                         if (bte->stub == NULL) {
1746                                 M_MOV_IMM(bte->fp, REG_ITMP1);
1747                         }
1748                         else {
1749                                 M_MOV_IMM(bte->stub, REG_ITMP1);
1750                         }
1751                         M_CALL(REG_ITMP1);
1752                         break;
1753
1754                 case ICMD_INVOKESPECIAL:
1755                         emit_nullpointer_check(cd, iptr, REG_A0);
1756                         /* fall through */
1757
1758                 case ICMD_INVOKESTATIC:
1759                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1760                                 um = iptr->sx.s23.s3.um;
1761                                 disp = dseg_add_unique_address(cd, um);
1762
1763                                 patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
1764                                                                           um, disp);
1765                         }
1766                         else {
1767                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1768                                 disp = dseg_add_functionptr(cd, lm->stubroutine);
1769                         }
1770
1771                         M_ALD(REG_ITMP2, RIP, disp);
1772                         M_CALL(REG_ITMP2);
1773                         break;
1774
1775                 case ICMD_INVOKEVIRTUAL:
1776                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1777                                 um = iptr->sx.s23.s3.um;
1778                                 patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
1779
1780                                 s1 = 0;
1781                         }
1782                         else {
1783                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1784                                 s1 = OFFSET(vftbl_t, table[0]) +
1785                                         sizeof(methodptr) * lm->vftblindex;
1786                         }
1787
1788                         /* implicit null-pointer check */
1789                         M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
1790                         M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
1791                         M_CALL(REG_ITMP3);
1792                         break;
1793
1794                 case ICMD_INVOKEINTERFACE:
1795                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1796                                 um = iptr->sx.s23.s3.um;
1797                                 patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
1798
1799                                 s1 = 0;
1800                                 s2 = 0;
1801                         }
1802                         else {
1803                                 lm = iptr->sx.s23.s3.fmiref->p.method;
1804                                 s1 = OFFSET(vftbl_t, interfacetable[0]) -
1805                                         sizeof(methodptr) * lm->clazz->index;
1806
1807                                 s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
1808                         }
1809
1810                         /* implicit null-pointer check */
1811                         M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
1812                         M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
1813                         M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
1814                         M_CALL(REG_ITMP3);
1815                         break;
1816
1817                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
1818
1819                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
1820                                 /* object type cast-check */
1821
1822                                 classinfo *super;
1823                                 s4         superindex;
1824
1825                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1826                                         super      = NULL;
1827                                         superindex = 0;
1828                                 }
1829                                 else {
1830                                         super      = iptr->sx.s23.s3.c.cls;
1831                                         superindex = super->index;
1832                                 }
1833
1834                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1835
1836                                 /* if class is not resolved, check which code to call */
1837
1838                                 if (super == NULL) {
1839                                         M_TEST(s1);
1840                                         emit_label_beq(cd, BRANCH_LABEL_1);
1841
1842                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
1843                                                                                   iptr->sx.s23.s3.c.ref, 0);
1844
1845                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
1846                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
1847                                         emit_label_beq(cd, BRANCH_LABEL_2);
1848                                 }
1849
1850                                 /* interface checkcast code */
1851
1852                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
1853                                         if (super != NULL) {
1854                                                 M_TEST(s1);
1855                                                 emit_label_beq(cd, BRANCH_LABEL_3);
1856                                         }
1857
1858                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
1859
1860                                         if (super == NULL) {
1861                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
1862                                                                                           iptr->sx.s23.s3.c.ref,
1863                                                                                           0);
1864                                         }
1865
1866                                         M_ILD32(REG_ITMP3,
1867                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
1868                                         M_ICMP_IMM32(superindex, REG_ITMP3);
1869                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
1870
1871                                         M_ALD32(REG_ITMP3, REG_ITMP2,
1872                                                         OFFSET(vftbl_t, interfacetable[0]) -
1873                                                         superindex * sizeof(methodptr*));
1874                                         M_TEST(REG_ITMP3);
1875                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
1876
1877                                         if (super == NULL)
1878                                                 emit_label_br(cd, BRANCH_LABEL_4);
1879                                         else
1880                                                 emit_label(cd, BRANCH_LABEL_3);
1881                                 }
1882
1883                                 /* class checkcast code */
1884
1885                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
1886                                         if (super == NULL) {
1887                                                 emit_label(cd, BRANCH_LABEL_2);
1888
1889                                                 constant_classref *cr = iptr->sx.s23.s3.c.ref;
1890                                                 disp = dseg_add_unique_address(cd, cr);
1891
1892                                                 patcher_add_patch_ref(jd,
1893                                                                                           PATCHER_resolve_classref_to_vftbl,
1894                                                                                           cr, disp);
1895                                         }
1896                                         else {
1897                                                 M_TEST(s1);
1898                                                 emit_label_beq(cd, BRANCH_LABEL_5);
1899
1900                                                 disp = dseg_add_address(cd, super->vftbl);
1901                                         }
1902
1903                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
1904                                         M_ALD(REG_ITMP3, RIP, disp);
1905
1906                                         if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
1907                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
1908                                                 M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
1909                                                 emit_label_beq(cd, BRANCH_LABEL_6);  /* good */
1910
1911                                                 if (super == NULL) {
1912                                                         M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
1913                                                         emit_label_bne(cd, BRANCH_LABEL_10);  /* throw */
1914                                                 }
1915
1916                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
1917                                                 M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
1918                                                 emit_label_bgt(cd, BRANCH_LABEL_9);  /* throw */
1919
1920                                                 M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
1921                                                 M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
1922                                                 emit_label_beq(cd, BRANCH_LABEL_7);  /* good */
1923
1924                                                 emit_label(cd, BRANCH_LABEL_9);
1925                                                 if (super == NULL)
1926                                                         emit_label(cd, BRANCH_LABEL_10);
1927
1928                                                 /* reload s1, might have been destroyed */
1929                                                 emit_load_s1(jd, iptr, REG_ITMP1);
1930                                                 M_ALD_MEM(s1, TRAP_ClassCastException);
1931
1932                                                 emit_label(cd, BRANCH_LABEL_7);
1933                                                 emit_label(cd, BRANCH_LABEL_6);
1934                                                 /* reload s1, might have been destroyed */
1935                                                 emit_load_s1(jd, iptr, REG_ITMP1);
1936                                         }
1937                                         else {
1938                                                 M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
1939                                                 emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
1940                                         }
1941
1942                                         if (super != NULL)
1943                                                 emit_label(cd, BRANCH_LABEL_5);
1944                                 }
1945
1946                                 if (super == NULL) {
1947                                         emit_label(cd, BRANCH_LABEL_1);
1948                                         emit_label(cd, BRANCH_LABEL_4);
1949                                 }
1950
1951                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1952                         }
1953                         else {
1954                                 /* array type cast-check */
1955
1956                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1957                                 M_INTMOVE(s1, REG_A0);
1958
1959                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1960                                         constant_classref *cr = iptr->sx.s23.s3.c.ref;
1961                                         disp = dseg_add_unique_address(cd, cr);
1962
1963                                         patcher_add_patch_ref(jd,
1964                                                                                   PATCHER_resolve_classref_to_classinfo,
1965                                                                                   cr, disp);
1966                                 }
1967                                 else {
1968                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
1969                                 }
1970
1971                                 M_ALD(REG_A1, RIP, disp);
1972                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
1973                                 M_CALL(REG_ITMP1);
1974
1975                                 /* s1 may have been destroyed over the function call */
1976                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1977                                 M_TEST(REG_RESULT);
1978                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
1979
1980                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1981                         }
1982
1983                         M_INTMOVE(s1, d);
1984                         emit_store_dst(jd, iptr, d);
1985                         break;
1986
1987                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
1988
1989                         {
1990                         classinfo *super;
1991                         s4         superindex;
1992
1993                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1994                                 super      = NULL;
1995                                 superindex = 0;
1996                         }
1997                         else {
1998                                 super      = iptr->sx.s23.s3.c.cls;
1999                                 superindex = super->index;
2000                         }
2001
2002                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2003                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2004
2005                         if (s1 == d) {
2006                                 M_INTMOVE(s1, REG_ITMP1);
2007                                 s1 = REG_ITMP1;
2008                         }
2009
2010                         M_CLR(d);
2011
2012                         /* if class is not resolved, check which code to call */
2013
2014                         if (super == NULL) {
2015                                 M_TEST(s1);
2016                                 emit_label_beq(cd, BRANCH_LABEL_1);
2017
2018                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2019                                                                           iptr->sx.s23.s3.c.ref, 0);
2020
2021                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2022                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2023                                 emit_label_beq(cd, BRANCH_LABEL_2);
2024                         }
2025
2026                         /* interface instanceof code */
2027
2028                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2029                                 if (super != NULL) {
2030                                         M_TEST(s1);
2031                                         emit_label_beq(cd, BRANCH_LABEL_3);
2032                                 }
2033
2034                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2035
2036                                 if (super == NULL) {
2037                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2038                                                                                   iptr->sx.s23.s3.c.ref, 0);
2039                                 }
2040
2041                                 M_ILD32(REG_ITMP3,
2042                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2043                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2044
2045                                 int a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2046
2047                                 M_BLE(a);
2048                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2049                                                 OFFSET(vftbl_t, interfacetable[0]) -
2050                                                 superindex * sizeof(methodptr*));
2051                                 M_TEST(REG_ITMP1);
2052                                 M_SETNE(d);
2053
2054                                 if (super == NULL)
2055                                         emit_label_br(cd, BRANCH_LABEL_4);
2056                                 else
2057                                         emit_label(cd, BRANCH_LABEL_3);
2058                         }
2059
2060                         /* class instanceof code */
2061
2062                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2063                                 if (super == NULL) {
2064                                         emit_label(cd, BRANCH_LABEL_2);
2065
2066                                         constant_classref *cr = iptr->sx.s23.s3.c.ref;
2067                                         disp = dseg_add_unique_address(cd, cr);
2068
2069                                         patcher_add_patch_ref(jd,
2070                                                                                   PATCHER_resolve_classref_to_vftbl,
2071                                                                                   cr, disp);
2072                                 }
2073                                 else {
2074                                         M_TEST(s1);
2075                                         emit_label_beq(cd, BRANCH_LABEL_5);
2076
2077                                         disp = dseg_add_address(cd, super->vftbl);
2078                                 }
2079
2080                                 M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2081                                 M_ALD(REG_ITMP3, RIP, disp);
2082
2083                                 if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2084                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2085                                         M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
2086                                         emit_label_bne(cd, BRANCH_LABEL_8); /* jump over INC/SETE */
2087                                         if (d == REG_ITMP2) {
2088                                                 M_SETE(d);
2089                                                 M_BSEXT(d, d);
2090                                         } else
2091                                                 M_LINC(d);
2092                                         emit_label_br(cd, BRANCH_LABEL_6);  /* true */
2093                                         emit_label(cd, BRANCH_LABEL_8);
2094
2095                                         if (super == NULL) {
2096                                                 M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2097                                                 emit_label_bne(cd, BRANCH_LABEL_10);  /* false */
2098                                         }
2099
2100                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
2101                                         M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
2102                                         emit_label_bgt(cd, BRANCH_LABEL_9);  /* false */
2103
2104                                         M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2105                                         M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
2106                                         M_SETE(d);
2107                                         if (d == REG_ITMP2) {
2108                                                 M_BSEXT(d, d);
2109
2110                                                 emit_label_br(cd, BRANCH_LABEL_7); /* jump over M_CLR */
2111                                         }
2112
2113                                         emit_label(cd, BRANCH_LABEL_9);
2114                                         if (super == NULL)
2115                                                 emit_label(cd, BRANCH_LABEL_10);
2116                                         if (d == REG_ITMP2) {
2117                                                 M_CLR(d);
2118
2119                                                 emit_label(cd, BRANCH_LABEL_7);
2120                                         }
2121                                         emit_label(cd, BRANCH_LABEL_6);
2122                                 }
2123                                 else {
2124                                         M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
2125                                         M_SETE(d);
2126                                         if (d == REG_ITMP2)
2127                                                 M_BSEXT(d, d);
2128                                 }
2129
2130                                 if (super != NULL)
2131                                         emit_label(cd, BRANCH_LABEL_5);
2132                         }
2133
2134                         if (super == NULL) {
2135                                 emit_label(cd, BRANCH_LABEL_1);
2136                                 emit_label(cd, BRANCH_LABEL_4);
2137                         }
2138
2139                         emit_store_dst(jd, iptr, d);
2140                         }
2141                         break;
2142
2143                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2144
2145                         /* check for negative sizes and copy sizes to stack if necessary  */
2146
2147                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2148
2149                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2150
2151                                 /* copy SAVEDVAR sizes to stack */
2152                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2153
2154                                 /* Already Preallocated? */
2155                                 if (!(var->flags & PREALLOC)) {
2156                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2157                                         M_LST(s2, REG_SP, s1 * 8);
2158                                 }
2159                         }
2160
2161                         /* a0 = dimension count */
2162
2163                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2164
2165                         /* is a patcher function set? */
2166
2167                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2168                                 constant_classref *cr = iptr->sx.s23.s3.c.ref;
2169                                 disp = dseg_add_unique_address(cd, cr);
2170
2171                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2172                                                                           cr, disp);
2173                         }
2174                         else {
2175                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2176                         }
2177
2178                         /* a1 = classinfo */
2179
2180                         M_ALD(REG_A1, RIP, disp);
2181
2182                         /* a2 = pointer to dimensions = stack pointer */
2183
2184                         M_MOV(REG_SP, REG_A2);
2185
2186                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2187                         M_CALL(REG_ITMP1);
2188
2189                         /* check for exception before result assignment */
2190
2191                         emit_exception_check(cd, iptr);
2192
2193                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2194                         M_INTMOVE(REG_RESULT, s1);
2195                         emit_store_dst(jd, iptr, s1);
2196                         break;
2197
2198                 default:
2199                         vm_abort("Unknown ICMD %d during code generation", iptr->opc);
2200         } /* switch */
2201 }
2202
2203
2204 /* codegen_emit_stub_native ****************************************************
2205
2206    Emits a stub routine which calls a native method.
2207
2208 *******************************************************************************/
2209
2210 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2211 {
2212         methodinfo  *m;
2213         codeinfo    *code;
2214         codegendata *cd;
2215         methoddesc  *md;
2216         int          i, j;
2217         int          s1, s2;
2218         int          disp;
2219
2220         /* Sanity check. */
2221
2222         assert(f != NULL);
2223
2224         /* Get required compiler data. */
2225
2226         m    = jd->m;
2227         code = jd->code;
2228         cd   = jd->cd;
2229
2230         /* initialize variables */
2231
2232         md = m->parseddesc;
2233
2234         /* calculate stack frame size */
2235
2236         cd->stackframesize =
2237                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2238                 sizeof(localref_table) / SIZEOF_VOID_P +
2239                 md->paramcount +
2240                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2241                 nmd->memuse;
2242
2243         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2244
2245         /* create method header */
2246
2247         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2248         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2249         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2250         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2251         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2252
2253 #if defined(ENABLE_PROFILING)
2254         /* generate native method profiling code */
2255
2256         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2257                 /* count frequency */
2258
2259                 M_MOV_IMM(code, REG_ITMP3);
2260                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2261         }
2262 #endif
2263
2264         /* generate stub code */
2265
2266         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2267
2268 #if defined(ENABLE_GC_CACAO)
2269         /* Save callee saved integer registers in stackframeinfo (GC may
2270            need to recover them during a collection). */
2271
2272         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2273                 OFFSET(stackframeinfo_t, intregs);
2274
2275         for (i = 0; i < INT_SAV_CNT; i++)
2276                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2277 #endif
2278
2279         /* save integer and float argument registers */
2280
2281         for (i = 0; i < md->paramcount; i++) {
2282                 if (!md->params[i].inmemory) {
2283                         s1 = md->params[i].regoff;
2284
2285                         switch (md->paramtypes[i].type) {
2286                         case TYPE_INT:
2287                         case TYPE_LNG:
2288                         case TYPE_ADR:
2289                                 M_LST(s1, REG_SP, i * 8);
2290                                 break;
2291                         case TYPE_FLT:
2292                         case TYPE_DBL:
2293                                 M_DST(s1, REG_SP, i * 8);
2294                                 break;
2295                         }
2296                 }
2297         }
2298
2299         /* create dynamic stack info */
2300
2301         M_MOV(REG_SP, REG_A0);
2302         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2303         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2304         M_CALL(REG_ITMP1);
2305
2306         /* remember class argument */
2307
2308         if (m->flags & ACC_STATIC)
2309                 M_MOV(REG_RESULT, REG_ITMP2);
2310
2311         /* restore integer and float argument registers */
2312
2313         for (i = 0; i < md->paramcount; i++) {
2314                 if (!md->params[i].inmemory) {
2315                         s1 = md->params[i].regoff;
2316
2317                         switch (md->paramtypes[i].type) {
2318                         case TYPE_INT:
2319                         case TYPE_LNG:
2320                         case TYPE_ADR:
2321                                 M_LLD(s1, REG_SP, i * 8);
2322                                 break;
2323                         case TYPE_FLT:
2324                         case TYPE_DBL:
2325                                 M_DLD(s1, REG_SP, i * 8);
2326                                 break;
2327                         }
2328                 }
2329         }
2330
2331         /* Copy or spill arguments to new locations. */
2332
2333         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
2334                 s2 = nmd->params[j].regoff;
2335
2336                 switch (md->paramtypes[i].type) {
2337                 case TYPE_INT:
2338                 case TYPE_LNG:
2339                 case TYPE_ADR:
2340                         if (!md->params[i].inmemory) {
2341                                 s1 = md->params[i].regoff;
2342
2343                                 if (!nmd->params[j].inmemory)
2344                                         M_INTMOVE(s1, s2);
2345                                 else
2346                                         M_LST(s1, REG_SP, s2);
2347                         }
2348                         else {
2349                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2350                                 M_LLD(REG_ITMP1, REG_SP, s1);
2351                                 M_LST(REG_ITMP1, REG_SP, s2);
2352                         }
2353                         break;
2354                 case TYPE_FLT:
2355                         /* We only copy spilled float arguments, as the float
2356                            argument registers keep unchanged. */
2357
2358                         if (md->params[i].inmemory) {
2359                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2360
2361                                 M_FLD(REG_FTMP1, REG_SP, s1);
2362                                 M_FST(REG_FTMP1, REG_SP, s2);
2363                         }
2364                         break;
2365                 case TYPE_DBL:
2366                         if (md->params[i].inmemory) {
2367                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2368                                 M_DLD(REG_FTMP1, REG_SP, s1);
2369                                 M_DST(REG_FTMP1, REG_SP, s2);
2370                         }
2371                         break;
2372                 }
2373         }
2374
2375         /* Handle native Java methods. */
2376
2377         if (m->flags & ACC_NATIVE) {
2378                 /* put class into second argument register */
2379
2380                 if (m->flags & ACC_STATIC)
2381                         M_MOV(REG_ITMP2, REG_A1);
2382
2383                 /* put env into first argument register */
2384
2385                 M_MOV_IMM(VM_get_jnienv(), REG_A0);
2386         }
2387
2388         /* Call the native function. */
2389
2390         disp = dseg_add_functionptr(cd, f);
2391         M_ALD(REG_ITMP1, RIP, disp);
2392         M_CALL(REG_ITMP1);
2393
2394         /* save return value */
2395
2396         switch (md->returntype.type) {
2397         case TYPE_INT:
2398         case TYPE_LNG:
2399         case TYPE_ADR:
2400                 switch (md->returntype.primitivetype) {
2401                 case PRIMITIVETYPE_BOOLEAN:
2402                         M_BZEXT(REG_RESULT, REG_RESULT);
2403                         break;
2404                 case PRIMITIVETYPE_BYTE:
2405                         M_BSEXT(REG_RESULT, REG_RESULT);
2406                         break;
2407                 case PRIMITIVETYPE_CHAR:
2408                         M_CZEXT(REG_RESULT, REG_RESULT);
2409                         break;
2410                 case PRIMITIVETYPE_SHORT:
2411                         M_SSEXT(REG_RESULT, REG_RESULT);
2412                         break;
2413                 }
2414                 M_LST(REG_RESULT, REG_SP, 0 * 8);
2415                 break;
2416         case TYPE_FLT:
2417         case TYPE_DBL:
2418                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
2419                 break;
2420         case TYPE_VOID:
2421                 break;
2422         }
2423
2424         /* remove native stackframe info */
2425
2426         M_MOV(REG_SP, REG_A0);
2427         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2428         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
2429         M_CALL(REG_ITMP1);
2430         M_MOV(REG_RESULT, REG_ITMP3);
2431
2432         /* restore return value */
2433
2434         switch (md->returntype.type) {
2435         case TYPE_INT:
2436         case TYPE_LNG:
2437         case TYPE_ADR:
2438                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
2439                 break;
2440         case TYPE_FLT:
2441         case TYPE_DBL:
2442                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
2443                 break;
2444         case TYPE_VOID:
2445                 break;
2446         }
2447
2448 #if defined(ENABLE_GC_CACAO)
2449         /* Restore callee saved integer registers from stackframeinfo (GC
2450            might have modified them during a collection). */
2451
2452         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2453                 OFFSET(stackframeinfo_t, intregs);
2454
2455         for (i = 0; i < INT_SAV_CNT; i++)
2456                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2457 #endif
2458
2459         /* remove stackframe */
2460
2461         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2462
2463         /* test for exception */
2464
2465         M_TEST(REG_ITMP3);
2466         M_BNE(1);
2467         M_RET;
2468
2469         /* handle exception */
2470
2471         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
2472         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
2473         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
2474
2475         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
2476         M_JMP(REG_ITMP3);
2477 }
2478
2479
2480 /*
2481  * These are local overrides for various environment variables in Emacs.
2482  * Please do not remove this and leave it at the end of the file, where
2483  * Emacs will automagically detect them.
2484  * ---------------------------------------------------------------------
2485  * Local variables:
2486  * mode: c
2487  * indent-tabs-mode: t
2488  * c-basic-offset: 4
2489  * tab-width: 4
2490  * End:
2491  * vim:noexpandtab:sw=4:ts=4:
2492  */