mono/mini/mini-amd64.c

   1 /*
   2  * mini-amd64.c: AMD64 backend for the Mono code generator
   3  *
   4  * Based on mini-x86.c.
   5  *
   6  * Authors:
   7  *   Paolo Molaro (lupus@ximian.com)
   8  *   Dietmar Maurer (dietmar@ximian.com)
   9  *   Patrik Torstensson
  10  *   Zoltan Varga (vargaz@gmail.com)
  11  *
  12  * (C) 2003 Ximian, Inc.
  13  * Copyright 2003-2011 Novell, Inc (http://www.novell.com)
  14  * Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  15  */
  16 #include "mini.h"
  17 #include <string.h>
  18 #include <math.h>
  19 #ifdef HAVE_UNISTD_H
  20 #include <unistd.h>
  21 #endif
  22
  23 #include <mono/metadata/appdomain.h>
  24 #include <mono/metadata/debug-helpers.h>
  25 #include <mono/metadata/threads.h>
  26 #include <mono/metadata/profiler-private.h>
  27 #include <mono/metadata/mono-debug.h>
  28 #include <mono/metadata/gc-internal.h>
  29 #include <mono/utils/mono-math.h>
  30 #include <mono/utils/mono-mmap.h>
  31 #include <mono/utils/mono-memory-model.h>
  32 #include <mono/utils/mono-tls.h>
  33
  34 #include "trace.h"
  35 #include "ir-emit.h"
  36 #include "mini-amd64.h"
  37 #include "cpu-amd64.h"
  38 #include "debugger-agent.h"
  39 #include "mini-gc.h"
  40
  41 static gint lmf_tls_offset = -1;
  42 static gint lmf_addr_tls_offset = -1;
  43 static gint appdomain_tls_offset = -1;
  44
  45 #ifdef MONO_XEN_OPT
  46 static gboolean optimize_for_xen = TRUE;
  47 #else
  48 #define optimize_for_xen 0
  49 #endif
  50
  51 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
  52
  53 #define IS_IMM32(val) ((((guint64)val) >> 32) == 0)
  54
  55 #define IS_REX(inst) (((inst) >= 0x40) && ((inst) <= 0x4f))
  56
  57 #ifdef HOST_WIN32
  58 /* Under windows, the calling convention is never stdcall */
  59 #define CALLCONV_IS_STDCALL(call_conv) (FALSE)
  60 #else
  61 #define CALLCONV_IS_STDCALL(call_conv) ((call_conv) == MONO_CALL_STDCALL)
  62 #endif
  63
  64 /* This mutex protects architecture specific caches */
  65 #define mono_mini_arch_lock() EnterCriticalSection (&mini_arch_mutex)
  66 #define mono_mini_arch_unlock() LeaveCriticalSection (&mini_arch_mutex)
  67 static CRITICAL_SECTION mini_arch_mutex;
  68
  69 MonoBreakpointInfo
  70 mono_breakpoint_info [MONO_BREAKPOINT_ARRAY_SIZE];
  71
  72 /* Structure used by the sequence points in AOTed code */
  73 typedef struct {
  74         gpointer ss_trigger_page;
  75         gpointer bp_trigger_page;
  76         gpointer bp_addrs [MONO_ZERO_LEN_ARRAY];
  77 } SeqPointInfo;
  78
  79 /*
  80  * The code generated for sequence points reads from this location, which is
  81  * made read-only when single stepping is enabled.
  82  */
  83 static gpointer ss_trigger_page;
  84
  85 /* Enabled breakpoints read from this trigger page */
  86 static gpointer bp_trigger_page;
  87
  88 /* The size of the breakpoint sequence */
  89 static int breakpoint_size;
  90
  91 /* The size of the breakpoint instruction causing the actual fault */
  92 static int breakpoint_fault_size;
  93
  94 /* The size of the single step instruction causing the actual fault */
  95 static int single_step_fault_size;
  96
  97 #ifdef HOST_WIN32
  98 /* On Win64 always reserve first 32 bytes for first four arguments */
  99 #define ARGS_OFFSET 48
 100 #else
 101 #define ARGS_OFFSET 16
 102 #endif
 103 #define GP_SCRATCH_REG AMD64_R11
 104
 105 /*
 106  * AMD64 register usage:
 107  * - callee saved registers are used for global register allocation
 108  * - %r11 is used for materializing 64 bit constants in opcodes
 109  * - the rest is used for local allocation
 110  */
 111
 112 /*
 113  * Floating point comparison results:
 114  *                  ZF PF CF
 115  * A > B            0  0  0
 116  * A < B            0  0  1
 117  * A = B            1  0  0
 118  * A > B            0  0  0
 119  * UNORDERED        1  1  1
 120  */
 121
 122 const char*
 123 mono_arch_regname (int reg)
 124 {
 125         switch (reg) {
 126         case AMD64_RAX: return "%rax";
 127         case AMD64_RBX: return "%rbx";
 128         case AMD64_RCX: return "%rcx";
 129         case AMD64_RDX: return "%rdx";
 130         case AMD64_RSP: return "%rsp";
 131         case AMD64_RBP: return "%rbp";
 132         case AMD64_RDI: return "%rdi";
 133         case AMD64_RSI: return "%rsi";
 134         case AMD64_R8: return "%r8";
 135         case AMD64_R9: return "%r9";
 136         case AMD64_R10: return "%r10";
 137         case AMD64_R11: return "%r11";
 138         case AMD64_R12: return "%r12";
 139         case AMD64_R13: return "%r13";
 140         case AMD64_R14: return "%r14";
 141         case AMD64_R15: return "%r15";
 142         }
 143         return "unknown";
 144 }
 145
 146 static const char * packed_xmmregs [] = {
 147         "p:xmm0", "p:xmm1", "p:xmm2", "p:xmm3", "p:xmm4", "p:xmm5", "p:xmm6", "p:xmm7", "p:xmm8",
 148         "p:xmm9", "p:xmm10", "p:xmm11", "p:xmm12", "p:xmm13", "p:xmm14", "p:xmm15"
 149 };
 150
 151 static const char * single_xmmregs [] = {
 152         "s:xmm0", "s:xmm1", "s:xmm2", "s:xmm3", "s:xmm4", "s:xmm5", "s:xmm6", "s:xmm7", "s:xmm8",
 153         "s:xmm9", "s:xmm10", "s:xmm11", "s:xmm12", "s:xmm13", "s:xmm14", "s:xmm15"
 154 };
 155
 156 const char*
 157 mono_arch_fregname (int reg)
 158 {
 159         if (reg < AMD64_XMM_NREG)
 160                 return single_xmmregs [reg];
 161         else
 162                 return "unknown";
 163 }
 164
 165 const char *
 166 mono_arch_xregname (int reg)
 167 {
 168         if (reg < AMD64_XMM_NREG)
 169                 return packed_xmmregs [reg];
 170         else
 171                 return "unknown";
 172 }
 173
 174 static gboolean
 175 debug_omit_fp (void)
 176 {
 177 #if 0
 178         return mono_debug_count ();
 179 #else
 180         return TRUE;
 181 #endif
 182 }
 183
 184 static inline gboolean
 185 amd64_is_near_call (guint8 *code)
 186 {
 187         /* Skip REX */
 188         if ((code [0] >= 0x40) && (code [0] <= 0x4f))
 189                 code += 1;
 190
 191         return code [0] == 0xe8;
 192 }
 193
 194 #ifdef __native_client_codegen__
 195
 196 /* Keep track of instruction "depth", that is, the level of sub-instruction */
 197 /* for any given instruction.  For instance, amd64_call_reg resolves to     */
 198 /* amd64_call_reg_internal, which uses amd64_alu_* macros, etc.             */
 199 /* We only want to force bundle alignment for the top level instruction,    */
 200 /* so NaCl pseudo-instructions can be implemented with sub instructions.    */
 201 static MonoNativeTlsKey nacl_instruction_depth;
 202
 203 static MonoNativeTlsKey nacl_rex_tag;
 204 static MonoNativeTlsKey nacl_legacy_prefix_tag;
 205
 206 void
 207 amd64_nacl_clear_legacy_prefix_tag ()
 208 {
 209         mono_native_tls_set_value (nacl_legacy_prefix_tag, NULL);
 210 }
 211
 212 void
 213 amd64_nacl_tag_legacy_prefix (guint8* code)
 214 {
 215         if (mono_native_tls_get_value (nacl_legacy_prefix_tag) == NULL)
 216                 mono_native_tls_set_value (nacl_legacy_prefix_tag, code);
 217 }
 218
 219 void
 220 amd64_nacl_tag_rex (guint8* code)
 221 {
 222         mono_native_tls_set_value (nacl_rex_tag, code);
 223 }
 224
 225 guint8*
 226 amd64_nacl_get_legacy_prefix_tag ()
 227 {
 228         return (guint8*)mono_native_tls_get_value (nacl_legacy_prefix_tag);
 229 }
 230
 231 guint8*
 232 amd64_nacl_get_rex_tag ()
 233 {
 234         return (guint8*)mono_native_tls_get_value (nacl_rex_tag);
 235 }
 236
 237 /* Increment the instruction "depth" described above */
 238 void
 239 amd64_nacl_instruction_pre ()
 240 {
 241         intptr_t depth = (intptr_t) mono_native_tls_get_value (nacl_instruction_depth);
 242         depth++;
 243         mono_native_tls_set_value (nacl_instruction_depth, (gpointer)depth);
 244 }
 245
 246 /* amd64_nacl_instruction_post: Decrement instruction "depth", force bundle */
 247 /* alignment if depth == 0 (top level instruction)                          */
 248 /* IN: start, end    pointers to instruction beginning and end              */
 249 /* OUT: start, end   pointers to beginning and end after possible alignment */
 250 /* GLOBALS: nacl_instruction_depth     defined above                        */
 251 void
 252 amd64_nacl_instruction_post (guint8 **start, guint8 **end)
 253 {
 254         intptr_t depth = (intptr_t) mono_native_tls_get_value (nacl_instruction_depth);
 255         depth--;
 256         mono_native_tls_set_value (nacl_instruction_depth, (void*)depth);
 257
 258         g_assert ( depth >= 0 );
 259         if (depth == 0) {
 260                 uintptr_t space_in_block;
 261                 uintptr_t instlen;
 262                 guint8 *prefix = amd64_nacl_get_legacy_prefix_tag ();
 263                 /* if legacy prefix is present, and if it was emitted before */
 264                 /* the start of the instruction sequence, adjust the start   */
 265                 if (prefix != NULL && prefix < *start) {
 266                         g_assert (*start - prefix <= 3);/* only 3 are allowed */
 267                         *start = prefix;
 268                 }
 269                 space_in_block = kNaClAlignment - ((uintptr_t)(*start) & kNaClAlignmentMask);
 270                 instlen = (uintptr_t)(*end - *start);
 271                 /* Only check for instructions which are less than        */
 272                 /* kNaClAlignment. The only instructions that should ever */
 273                 /* be that long are call sequences, which are already     */
 274                 /* padded out to align the return to the next bundle.     */
 275                 if (instlen > space_in_block && instlen < kNaClAlignment) {
 276                         const size_t MAX_NACL_INST_LENGTH = kNaClAlignment;
 277                         guint8 copy_of_instruction[MAX_NACL_INST_LENGTH];
 278                         const size_t length = (size_t)((*end)-(*start));
 279                         g_assert (length < MAX_NACL_INST_LENGTH);
 280
 281                         memcpy (copy_of_instruction, *start, length);
 282                         *start = mono_arch_nacl_pad (*start, space_in_block);
 283                         memcpy (*start, copy_of_instruction, length);
 284                         *end = *start + length;
 285                 }
 286                 amd64_nacl_clear_legacy_prefix_tag ();
 287                 amd64_nacl_tag_rex (NULL);
 288         }
 289 }
 290
 291 /* amd64_nacl_membase_handler: ensure all access to memory of the form      */
 292 /*   OFFSET(%rXX) is sandboxed.  For allowable base registers %rip, %rbp,   */
 293 /*   %rsp, and %r15, emit the membase as usual.  For all other registers,   */
 294 /*   make sure the upper 32-bits are cleared, and use that register in the  */
 295 /*   index field of a new address of this form: OFFSET(%r15,%eXX,1)         */
 296 /* IN:      code                                                            */
 297 /*             pointer to current instruction stream (in the                */
 298 /*             middle of an instruction, after opcode is emitted)           */
 299 /*          basereg/offset/dreg                                             */
 300 /*             operands of normal membase address                           */
 301 /* OUT:     code                                                            */
 302 /*             pointer to the end of the membase/memindex emit              */
 303 /* GLOBALS: nacl_rex_tag                                                    */
 304 /*             position in instruction stream that rex prefix was emitted   */
 305 /*          nacl_legacy_prefix_tag                                          */
 306 /*             (possibly NULL) position in instruction of legacy x86 prefix */
 307 void
 308 amd64_nacl_membase_handler (guint8** code, gint8 basereg, gint32 offset, gint8 dreg)
 309 {
 310         gint8 true_basereg = basereg;
 311
 312         /* Cache these values, they might change  */
 313         /* as new instructions are emitted below. */
 314         guint8* rex_tag = amd64_nacl_get_rex_tag ();
 315         guint8* legacy_prefix_tag = amd64_nacl_get_legacy_prefix_tag ();
 316
 317         /* 'basereg' is given masked to 0x7 at this point, so check */
 318         /* the rex prefix to see if this is an extended register.   */
 319         if ((rex_tag != NULL) && IS_REX(*rex_tag) && (*rex_tag & AMD64_REX_B)) {
 320                 true_basereg |= 0x8;
 321         }
 322
 323 #define X86_LEA_OPCODE (0x8D)
 324
 325         if (!amd64_is_valid_nacl_base (true_basereg) && (*(*code-1) != X86_LEA_OPCODE)) {
 326                 guint8* old_instruction_start;
 327
 328                 /* This will hold the 'mov %eXX, %eXX' that clears the upper */
 329                 /* 32-bits of the old base register (new index register)     */
 330                 guint8 buf[32];
 331                 guint8* buf_ptr = buf;
 332                 size_t insert_len;
 333
 334                 g_assert (rex_tag != NULL);
 335
 336                 if (IS_REX(*rex_tag)) {
 337                         /* The old rex.B should be the new rex.X */
 338                         if (*rex_tag & AMD64_REX_B) {
 339                                 *rex_tag |= AMD64_REX_X;
 340                         }
 341                         /* Since our new base is %r15 set rex.B */
 342                         *rex_tag |= AMD64_REX_B;
 343                 } else {
 344                         /* Shift the instruction by one byte  */
 345                         /* so we can insert a rex prefix      */
 346                         memmove (rex_tag + 1, rex_tag, (size_t)(*code - rex_tag));
 347                         *code += 1;
 348                         /* New rex prefix only needs rex.B for %r15 base */
 349                         *rex_tag = AMD64_REX(AMD64_REX_B);
 350                 }
 351
 352                 if (legacy_prefix_tag) {
 353                         old_instruction_start = legacy_prefix_tag;
 354                 } else {
 355                         old_instruction_start = rex_tag;
 356                 }
 357
 358                 /* Clears the upper 32-bits of the previous base register */
 359                 amd64_mov_reg_reg_size (buf_ptr, true_basereg, true_basereg, 4);
 360                 insert_len = buf_ptr - buf;
 361
 362                 /* Move the old instruction forward to make */
 363                 /* room for 'mov' stored in 'buf_ptr'       */
 364                 memmove (old_instruction_start + insert_len, old_instruction_start, (size_t)(*code - old_instruction_start));
 365                 *code += insert_len;
 366                 memcpy (old_instruction_start, buf, insert_len);
 367
 368                 /* Sandboxed replacement for the normal membase_emit */
 369                 x86_memindex_emit (*code, dreg, AMD64_R15, offset, basereg, 0);
 370
 371         } else {
 372                 /* Normal default behavior, emit membase memory location */
 373                 x86_membase_emit_body (*code, dreg, basereg, offset);
 374         }
 375 }
 376
 377
 378 static inline unsigned char*
 379 amd64_skip_nops (unsigned char* code)
 380 {
 381         guint8 in_nop;
 382         do {
 383                 in_nop = 0;
 384                 if (   code[0] == 0x90) {
 385                         in_nop = 1;
 386                         code += 1;
 387                 }
 388                 if (   code[0] == 0x66 && code[1] == 0x90) {
 389                         in_nop = 1;
 390                         code += 2;
 391                 }
 392                 if (code[0] == 0x0f && code[1] == 0x1f
 393                  && code[2] == 0x00) {
 394                         in_nop = 1;
 395                         code += 3;
 396                 }
 397                 if (code[0] == 0x0f && code[1] == 0x1f
 398                  && code[2] == 0x40 && code[3] == 0x00) {
 399                         in_nop = 1;
 400                         code += 4;
 401                 }
 402                 if (code[0] == 0x0f && code[1] == 0x1f
 403                  && code[2] == 0x44 && code[3] == 0x00
 404                  && code[4] == 0x00) {
 405                         in_nop = 1;
 406                         code += 5;
 407                 }
 408                 if (code[0] == 0x66 && code[1] == 0x0f
 409                  && code[2] == 0x1f && code[3] == 0x44
 410                  && code[4] == 0x00 && code[5] == 0x00) {
 411                         in_nop = 1;
 412                         code += 6;
 413                 }
 414                 if (code[0] == 0x0f && code[1] == 0x1f
 415                  && code[2] == 0x80 && code[3] == 0x00
 416                  && code[4] == 0x00 && code[5] == 0x00
 417                  && code[6] == 0x00) {
 418                         in_nop = 1;
 419                         code += 7;
 420                 }
 421                 if (code[0] == 0x0f && code[1] == 0x1f
 422                  && code[2] == 0x84 && code[3] == 0x00
 423                  && code[4] == 0x00 && code[5] == 0x00
 424                  && code[6] == 0x00 && code[7] == 0x00) {
 425                         in_nop = 1;
 426                         code += 8;
 427                 }
 428         } while ( in_nop );
 429         return code;
 430 }
 431
 432 guint8*
 433 mono_arch_nacl_skip_nops (guint8* code)
 434 {
 435   return amd64_skip_nops(code);
 436 }
 437
 438 #endif /*__native_client_codegen__*/
 439
 440 static inline void
 441 amd64_patch (unsigned char* code, gpointer target)
 442 {
 443         guint8 rex = 0;
 444
 445 #ifdef __native_client_codegen__
 446         code = amd64_skip_nops (code);
 447 #endif
 448 #if defined(__native_client_codegen__) && defined(__native_client__)
 449         if (nacl_is_code_address (code)) {
 450                 /* For tail calls, code is patched after being installed */
 451                 /* but not through the normal "patch callsite" method.   */
 452                 unsigned char buf[kNaClAlignment];
 453                 unsigned char *aligned_code = (uintptr_t)code & ~kNaClAlignmentMask;
 454                 int ret;
 455                 memcpy (buf, aligned_code, kNaClAlignment);
 456                 /* Patch a temp buffer of bundle size, */
 457                 /* then install to actual location.    */
 458                 amd64_patch (buf + ((uintptr_t)code - (uintptr_t)aligned_code), target);
 459                 ret = nacl_dyncode_modify (aligned_code, buf, kNaClAlignment);
 460                 g_assert (ret == 0);
 461                 return;
 462         }
 463         target = nacl_modify_patch_target (target);
 464 #endif
 465
 466         /* Skip REX */
 467         if ((code [0] >= 0x40) && (code [0] <= 0x4f)) {
 468                 rex = code [0];
 469                 code += 1;
 470         }
 471
 472         if ((code [0] & 0xf8) == 0xb8) {
 473                 /* amd64_set_reg_template */
 474                 *(guint64*)(code + 1) = (guint64)target;
 475         }
 476         else if ((code [0] == 0x8b) && rex && x86_modrm_mod (code [1]) == 0 && x86_modrm_rm (code [1]) == 5) {
 477                 /* mov 0(%rip), %dreg */
 478                 *(guint32*)(code + 2) = (guint32)(guint64)target - 7;
 479         }
 480         else if ((code [0] == 0xff) && (code [1] == 0x15)) {
 481                 /* call *<OFFSET>(%rip) */
 482                 *(guint32*)(code + 2) = ((guint32)(guint64)target) - 7;
 483         }
 484         else if (code [0] == 0xe8) {
 485                 /* call <DISP> */
 486                 gint64 disp = (guint8*)target - (guint8*)code;
 487                 g_assert (amd64_is_imm32 (disp));
 488                 x86_patch (code, (unsigned char*)target);
 489         }
 490         else
 491                 x86_patch (code, (unsigned char*)target);
 492 }
 493
 494 void
 495 mono_amd64_patch (unsigned char* code, gpointer target)
 496 {
 497         amd64_patch (code, target);
 498 }
 499
 500 typedef enum {
 501         ArgInIReg,
 502         ArgInFloatSSEReg,
 503         ArgInDoubleSSEReg,
 504         ArgOnStack,
 505         ArgValuetypeInReg,
 506         ArgValuetypeAddrInIReg,
 507         ArgNone /* only in pair_storage */
 508 } ArgStorage;
 509
 510 typedef struct {
 511         gint16 offset;
 512         gint8  reg;
 513         ArgStorage storage;
 514
 515         /* Only if storage == ArgValuetypeInReg */
 516         ArgStorage pair_storage [2];
 517         gint8 pair_regs [2];
 518         int nregs;
 519 } ArgInfo;
 520
 521 typedef struct {
 522         int nargs;
 523         guint32 stack_usage;
 524         guint32 reg_usage;
 525         guint32 freg_usage;
 526         gboolean need_stack_align;
 527         gboolean vtype_retaddr;
 528         /* The index of the vret arg in the argument list */
 529         int vret_arg_index;
 530         ArgInfo ret;
 531         ArgInfo sig_cookie;
 532         ArgInfo args [1];
 533 } CallInfo;
 534
 535 #define DEBUG(a) if (cfg->verbose_level > 1) a
 536
 537 #ifdef HOST_WIN32
 538 #define PARAM_REGS 4
 539
 540 static AMD64_Reg_No param_regs [] = { AMD64_RCX, AMD64_RDX, AMD64_R8, AMD64_R9 };
 541
 542 static AMD64_Reg_No return_regs [] = { AMD64_RAX, AMD64_RDX };
 543 #else
 544 #define PARAM_REGS 6
 545
 546 static AMD64_Reg_No param_regs [] = { AMD64_RDI, AMD64_RSI, AMD64_RDX, AMD64_RCX, AMD64_R8, AMD64_R9 };
 547
 548  static AMD64_Reg_No return_regs [] = { AMD64_RAX, AMD64_RDX };
 549 #endif
 550
 551 static void inline
 552 add_general (guint32 *gr, guint32 *stack_size, ArgInfo *ainfo)
 553 {
 554     ainfo->offset = *stack_size;
 555
 556     if (*gr >= PARAM_REGS) {
 557                 ainfo->storage = ArgOnStack;
 558                 /* Since the same stack slot size is used for all arg */
 559                 /*  types, it needs to be big enough to hold them all */
 560                 (*stack_size) += sizeof(mgreg_t);
 561     }
 562     else {
 563                 ainfo->storage = ArgInIReg;
 564                 ainfo->reg = param_regs [*gr];
 565                 (*gr) ++;
 566     }
 567 }
 568
 569 #ifdef HOST_WIN32
 570 #define FLOAT_PARAM_REGS 4
 571 #else
 572 #define FLOAT_PARAM_REGS 8
 573 #endif
 574
 575 static void inline
 576 add_float (guint32 *gr, guint32 *stack_size, ArgInfo *ainfo, gboolean is_double)
 577 {
 578     ainfo->offset = *stack_size;
 579
 580     if (*gr >= FLOAT_PARAM_REGS) {
 581                 ainfo->storage = ArgOnStack;
 582                 /* Since the same stack slot size is used for both float */
 583                 /*  types, it needs to be big enough to hold them both */
 584                 (*stack_size) += sizeof(mgreg_t);
 585     }
 586     else {
 587                 /* A double register */
 588                 if (is_double)
 589                         ainfo->storage = ArgInDoubleSSEReg;
 590                 else
 591                         ainfo->storage = ArgInFloatSSEReg;
 592                 ainfo->reg = *gr;
 593                 (*gr) += 1;
 594     }
 595 }
 596
 597 typedef enum ArgumentClass {
 598         ARG_CLASS_NO_CLASS,
 599         ARG_CLASS_MEMORY,
 600         ARG_CLASS_INTEGER,
 601         ARG_CLASS_SSE
 602 } ArgumentClass;
 603
 604 static ArgumentClass
 605 merge_argument_class_from_type (MonoType *type, ArgumentClass class1)
 606 {
 607         ArgumentClass class2 = ARG_CLASS_NO_CLASS;
 608         MonoType *ptype;
 609
 610         ptype = mini_type_get_underlying_type (NULL, type);
 611         switch (ptype->type) {
 612         case MONO_TYPE_BOOLEAN:
 613         case MONO_TYPE_CHAR:
 614         case MONO_TYPE_I1:
 615         case MONO_TYPE_U1:
 616         case MONO_TYPE_I2:
 617         case MONO_TYPE_U2:
 618         case MONO_TYPE_I4:
 619         case MONO_TYPE_U4:
 620         case MONO_TYPE_I:
 621         case MONO_TYPE_U:
 622         case MONO_TYPE_STRING:
 623         case MONO_TYPE_OBJECT:
 624         case MONO_TYPE_CLASS:
 625         case MONO_TYPE_SZARRAY:
 626         case MONO_TYPE_PTR:
 627         case MONO_TYPE_FNPTR:
 628         case MONO_TYPE_ARRAY:
 629         case MONO_TYPE_I8:
 630         case MONO_TYPE_U8:
 631                 class2 = ARG_CLASS_INTEGER;
 632                 break;
 633         case MONO_TYPE_R4:
 634         case MONO_TYPE_R8:
 635 #ifdef HOST_WIN32
 636                 class2 = ARG_CLASS_INTEGER;
 637 #else
 638                 class2 = ARG_CLASS_SSE;
 639 #endif
 640                 break;
 641
 642         case MONO_TYPE_TYPEDBYREF:
 643                 g_assert_not_reached ();
 644
 645         case MONO_TYPE_GENERICINST:
 646                 if (!mono_type_generic_inst_is_valuetype (ptype)) {
 647                         class2 = ARG_CLASS_INTEGER;
 648                         break;
 649                 }
 650                 /* fall through */
 651         case MONO_TYPE_VALUETYPE: {
 652                 MonoMarshalType *info = mono_marshal_load_type_info (ptype->data.klass);
 653                 int i;
 654
 655                 for (i = 0; i < info->num_fields; ++i) {
 656                         class2 = class1;
 657                         class2 = merge_argument_class_from_type (info->fields [i].field->type, class2);
 658                 }
 659                 break;
 660         }
 661         default:
 662                 g_assert_not_reached ();
 663         }
 664
 665         /* Merge */
 666         if (class1 == class2)
 667                 ;
 668         else if (class1 == ARG_CLASS_NO_CLASS)
 669                 class1 = class2;
 670         else if ((class1 == ARG_CLASS_MEMORY) || (class2 == ARG_CLASS_MEMORY))
 671                 class1 = ARG_CLASS_MEMORY;
 672         else if ((class1 == ARG_CLASS_INTEGER) || (class2 == ARG_CLASS_INTEGER))
 673                 class1 = ARG_CLASS_INTEGER;
 674         else
 675                 class1 = ARG_CLASS_SSE;
 676
 677         return class1;
 678 }
 679 #ifdef __native_client_codegen__
 680 const guint kNaClAlignment = kNaClAlignmentAMD64;
 681 const guint kNaClAlignmentMask = kNaClAlignmentMaskAMD64;
 682
 683 /* Default alignment for Native Client is 32-byte. */
 684 gint8 nacl_align_byte = -32; /* signed version of 0xe0 */
 685
 686 /* mono_arch_nacl_pad: Add pad bytes of alignment instructions at code,  */
 687 /* Check that alignment doesn't cross an alignment boundary.             */
 688 guint8*
 689 mono_arch_nacl_pad(guint8 *code, int pad)
 690 {
 691         const int kMaxPadding = 8; /* see amd64-codegen.h:amd64_padding_size() */
 692
 693         if (pad == 0) return code;
 694         /* assertion: alignment cannot cross a block boundary */
 695         g_assert (((uintptr_t)code & (~kNaClAlignmentMask)) ==
 696                  (((uintptr_t)code + pad - 1) & (~kNaClAlignmentMask)));
 697         while (pad >= kMaxPadding) {
 698                 amd64_padding (code, kMaxPadding);
 699                 pad -= kMaxPadding;
 700         }
 701         if (pad != 0) amd64_padding (code, pad);
 702         return code;
 703 }
 704 #endif
 705
 706 static void
 707 add_valuetype (MonoGenericSharingContext *gsctx, MonoMethodSignature *sig, ArgInfo *ainfo, MonoType *type,
 708                            gboolean is_return,
 709                            guint32 *gr, guint32 *fr, guint32 *stack_size)
 710 {
 711         guint32 size, quad, nquads, i;
 712         /* Keep track of the size used in each quad so we can */
 713         /* use the right size when copying args/return vars.  */
 714         guint32 quadsize [2] = {8, 8};
 715         ArgumentClass args [2];
 716         MonoMarshalType *info = NULL;
 717         MonoClass *klass;
 718         MonoGenericSharingContext tmp_gsctx;
 719         gboolean pass_on_stack = FALSE;
 720
 721         /*
 722          * The gsctx currently contains no data, it is only used for checking whenever
 723          * open types are allowed, some callers like mono_arch_get_argument_info ()
 724          * don't pass it to us, so work around that.
 725          */
 726         if (!gsctx)
 727                 gsctx = &tmp_gsctx;
 728
 729         klass = mono_class_from_mono_type (type);
 730         size = mini_type_stack_size_full (gsctx, &klass->byval_arg, NULL, sig->pinvoke);
 731 #ifndef HOST_WIN32
 732         if (!sig->pinvoke && !disable_vtypes_in_regs && ((is_return && (size == 8)) || (!is_return && (size <= 16)))) {
 733                 /* We pass and return vtypes of size 8 in a register */
 734         } else if (!sig->pinvoke || (size == 0) || (size > 16)) {
 735                 pass_on_stack = TRUE;
 736         }
 737 #else
 738         if (!sig->pinvoke) {
 739                 pass_on_stack = TRUE;
 740         }
 741 #endif
 742
 743         /* If this struct can't be split up naturally into 8-byte */
 744         /* chunks (registers), pass it on the stack.              */
 745         if (sig->pinvoke && !pass_on_stack) {
 746                 guint32 align;
 747                 guint32 field_size;
 748
 749                 info = mono_marshal_load_type_info (klass);
 750                 g_assert(info);
 751                 for (i = 0; i < info->num_fields; ++i) {
 752                         field_size = mono_marshal_type_size (info->fields [i].field->type,
 753                                                            info->fields [i].mspec,
 754                                                            &align, TRUE, klass->unicode);
 755                         if ((info->fields [i].offset < 8) && (info->fields [i].offset + field_size) > 8) {
 756                                 pass_on_stack = TRUE;
 757                                 break;
 758                         }
 759                 }
 760         }
 761
 762         if (pass_on_stack) {
 763                 /* Allways pass in memory */
 764                 ainfo->offset = *stack_size;
 765                 *stack_size += ALIGN_TO (size, 8);
 766                 ainfo->storage = ArgOnStack;
 767
 768                 return;
 769         }
 770
 771         /* FIXME: Handle structs smaller than 8 bytes */
 772         //if ((size % 8) != 0)
 773         //      NOT_IMPLEMENTED;
 774
 775         if (size > 8)
 776                 nquads = 2;
 777         else
 778                 nquads = 1;
 779
 780         if (!sig->pinvoke) {
 781                 /* Always pass in 1 or 2 integer registers */
 782                 args [0] = ARG_CLASS_INTEGER;
 783                 args [1] = ARG_CLASS_INTEGER;
 784                 /* Only the simplest cases are supported */
 785                 if (is_return && nquads != 1) {
 786                         args [0] = ARG_CLASS_MEMORY;
 787                         args [1] = ARG_CLASS_MEMORY;
 788                 }
 789         } else {
 790                 /*
 791                  * Implement the algorithm from section 3.2.3 of the X86_64 ABI.
 792                  * The X87 and SSEUP stuff is left out since there are no such types in
 793                  * the CLR.
 794                  */
 795                 info = mono_marshal_load_type_info (klass);
 796                 g_assert (info);
 797
 798 #ifndef HOST_WIN32
 799                 if (info->native_size > 16) {
 800                         ainfo->offset = *stack_size;
 801                         *stack_size += ALIGN_TO (info->native_size, 8);
 802                         ainfo->storage = ArgOnStack;
 803
 804                         return;
 805                 }
 806 #else
 807                 switch (info->native_size) {
 808                 case 1: case 2: case 4: case 8:
 809                         break;
 810                 default:
 811                         if (is_return) {
 812                                 ainfo->storage = ArgOnStack;
 813                                 ainfo->offset = *stack_size;
 814                                 *stack_size += ALIGN_TO (info->native_size, 8);
 815                         }
 816                         else {
 817                                 ainfo->storage = ArgValuetypeAddrInIReg;
 818
 819                                 if (*gr < PARAM_REGS) {
 820                                         ainfo->pair_storage [0] = ArgInIReg;
 821                                         ainfo->pair_regs [0] = param_regs [*gr];
 822                                         (*gr) ++;
 823                                 }
 824                                 else {
 825                                         ainfo->pair_storage [0] = ArgOnStack;
 826                                         ainfo->offset = *stack_size;
 827                                         *stack_size += 8;
 828                                 }
 829                         }
 830
 831                         return;
 832                 }
 833 #endif
 834
 835                 args [0] = ARG_CLASS_NO_CLASS;
 836                 args [1] = ARG_CLASS_NO_CLASS;
 837                 for (quad = 0; quad < nquads; ++quad) {
 838                         int size;
 839                         guint32 align;
 840                         ArgumentClass class1;
 841
 842                         if (info->num_fields == 0)
 843                                 class1 = ARG_CLASS_MEMORY;
 844                         else
 845                                 class1 = ARG_CLASS_NO_CLASS;
 846                         for (i = 0; i < info->num_fields; ++i) {
 847                                 size = mono_marshal_type_size (info->fields [i].field->type,
 848                                                                                            info->fields [i].mspec,
 849                                                                                            &align, TRUE, klass->unicode);
 850                                 if ((info->fields [i].offset < 8) && (info->fields [i].offset + size) > 8) {
 851                                         /* Unaligned field */
 852                                         NOT_IMPLEMENTED;
 853                                 }
 854
 855                                 /* Skip fields in other quad */
 856                                 if ((quad == 0) && (info->fields [i].offset >= 8))
 857                                         continue;
 858                                 if ((quad == 1) && (info->fields [i].offset < 8))
 859                                         continue;
 860
 861                                 /* How far into this quad this data extends.*/
 862                                 /* (8 is size of quad) */
 863                                 quadsize [quad] = info->fields [i].offset + size - (quad * 8);
 864
 865                                 class1 = merge_argument_class_from_type (info->fields [i].field->type, class1);
 866                         }
 867                         g_assert (class1 != ARG_CLASS_NO_CLASS);
 868                         args [quad] = class1;
 869                 }
 870         }
 871
 872         /* Post merger cleanup */
 873         if ((args [0] == ARG_CLASS_MEMORY) || (args [1] == ARG_CLASS_MEMORY))
 874                 args [0] = args [1] = ARG_CLASS_MEMORY;
 875
 876         /* Allocate registers */
 877         {
 878                 int orig_gr = *gr;
 879                 int orig_fr = *fr;
 880
 881                 ainfo->storage = ArgValuetypeInReg;
 882                 ainfo->pair_storage [0] = ainfo->pair_storage [1] = ArgNone;
 883                 ainfo->nregs = nquads;
 884                 for (quad = 0; quad < nquads; ++quad) {
 885                         switch (args [quad]) {
 886                         case ARG_CLASS_INTEGER:
 887                                 if (*gr >= PARAM_REGS)
 888                                         args [quad] = ARG_CLASS_MEMORY;
 889                                 else {
 890                                         ainfo->pair_storage [quad] = ArgInIReg;
 891                                         if (is_return)
 892                                                 ainfo->pair_regs [quad] = return_regs [*gr];
 893                                         else
 894                                                 ainfo->pair_regs [quad] = param_regs [*gr];
 895                                         (*gr) ++;
 896                                 }
 897                                 break;
 898                         case ARG_CLASS_SSE:
 899                                 if (*fr >= FLOAT_PARAM_REGS)
 900                                         args [quad] = ARG_CLASS_MEMORY;
 901                                 else {
 902                                         if (quadsize[quad] <= 4)
 903                                                 ainfo->pair_storage [quad] = ArgInFloatSSEReg;
 904                                         else ainfo->pair_storage [quad] = ArgInDoubleSSEReg;
 905                                         ainfo->pair_regs [quad] = *fr;
 906                                         (*fr) ++;
 907                                 }
 908                                 break;
 909                         case ARG_CLASS_MEMORY:
 910                                 break;
 911                         default:
 912                                 g_assert_not_reached ();
 913                         }
 914                 }
 915
 916                 if ((args [0] == ARG_CLASS_MEMORY) || (args [1] == ARG_CLASS_MEMORY)) {
 917                         /* Revert possible register assignments */
 918                         *gr = orig_gr;
 919                         *fr = orig_fr;
 920
 921                         ainfo->offset = *stack_size;
 922                         if (sig->pinvoke)
 923                                 *stack_size += ALIGN_TO (info->native_size, 8);
 924                         else
 925                                 *stack_size += nquads * sizeof(mgreg_t);
 926                         ainfo->storage = ArgOnStack;
 927                 }
 928         }
 929 }
 930
 931 /*
 932  * get_call_info:
 933  *
 934  *  Obtain information about a call according to the calling convention.
 935  * For AMD64, see the "System V ABI, x86-64 Architecture Processor Supplement
 936  * Draft Version 0.23" document for more information.
 937  */
 938 static CallInfo*
 939 get_call_info (MonoGenericSharingContext *gsctx, MonoMemPool *mp, MonoMethodSignature *sig)
 940 {
 941         guint32 i, gr, fr, pstart;
 942         MonoType *ret_type;
 943         int n = sig->hasthis + sig->param_count;
 944         guint32 stack_size = 0;
 945         CallInfo *cinfo;
 946         gboolean is_pinvoke = sig->pinvoke;
 947
 948         if (mp)
 949                 cinfo = mono_mempool_alloc0 (mp, sizeof (CallInfo) + (sizeof (ArgInfo) * n));
 950         else
 951                 cinfo = g_malloc0 (sizeof (CallInfo) + (sizeof (ArgInfo) * n));
 952
 953         cinfo->nargs = n;
 954
 955         gr = 0;
 956         fr = 0;
 957
 958         /* return value */
 959         {
 960                 ret_type = mini_type_get_underlying_type (gsctx, sig->ret);
 961                 switch (ret_type->type) {
 962                 case MONO_TYPE_BOOLEAN:
 963                 case MONO_TYPE_I1:
 964                 case MONO_TYPE_U1:
 965                 case MONO_TYPE_I2:
 966                 case MONO_TYPE_U2:
 967                 case MONO_TYPE_CHAR:
 968                 case MONO_TYPE_I4:
 969                 case MONO_TYPE_U4:
 970                 case MONO_TYPE_I:
 971                 case MONO_TYPE_U:
 972                 case MONO_TYPE_PTR:
 973                 case MONO_TYPE_FNPTR:
 974                 case MONO_TYPE_CLASS:
 975                 case MONO_TYPE_OBJECT:
 976                 case MONO_TYPE_SZARRAY:
 977                 case MONO_TYPE_ARRAY:
 978                 case MONO_TYPE_STRING:
 979                         cinfo->ret.storage = ArgInIReg;
 980                         cinfo->ret.reg = AMD64_RAX;
 981                         break;
 982                 case MONO_TYPE_U8:
 983                 case MONO_TYPE_I8:
 984                         cinfo->ret.storage = ArgInIReg;
 985                         cinfo->ret.reg = AMD64_RAX;
 986                         break;
 987                 case MONO_TYPE_R4:
 988                         cinfo->ret.storage = ArgInFloatSSEReg;
 989                         cinfo->ret.reg = AMD64_XMM0;
 990                         break;
 991                 case MONO_TYPE_R8:
 992                         cinfo->ret.storage = ArgInDoubleSSEReg;
 993                         cinfo->ret.reg = AMD64_XMM0;
 994                         break;
 995                 case MONO_TYPE_GENERICINST:
 996                         if (!mono_type_generic_inst_is_valuetype (ret_type)) {
 997                                 cinfo->ret.storage = ArgInIReg;
 998                                 cinfo->ret.reg = AMD64_RAX;
 999                                 break;
1000                         }
1001                         /* fall through */
1002 #if defined( __native_client_codegen__ )
1003                 case MONO_TYPE_TYPEDBYREF:
1004 #endif
1005                 case MONO_TYPE_VALUETYPE: {
1006                         guint32 tmp_gr = 0, tmp_fr = 0, tmp_stacksize = 0;
1007
1008                         add_valuetype (gsctx, sig, &cinfo->ret, sig->ret, TRUE, &tmp_gr, &tmp_fr, &tmp_stacksize);
1009                         if (cinfo->ret.storage == ArgOnStack) {
1010                                 cinfo->vtype_retaddr = TRUE;
1011                                 /* The caller passes the address where the value is stored */
1012                         }
1013                         break;
1014                 }
1015 #if !defined( __native_client_codegen__ )
1016                 case MONO_TYPE_TYPEDBYREF:
1017                         /* Same as a valuetype with size 24 */
1018                         cinfo->vtype_retaddr = TRUE;
1019                         break;
1020 #endif
1021                 case MONO_TYPE_VOID:
1022                         break;
1023                 default:
1024                         g_error ("Can't handle as return value 0x%x", sig->ret->type);
1025                 }
1026         }
1027
1028         pstart = 0;
1029         /*
1030          * To simplify get_this_arg_reg () and LLVM integration, emit the vret arg after
1031          * the first argument, allowing 'this' to be always passed in the first arg reg.
1032          * Also do this if the first argument is a reference type, since virtual calls
1033          * are sometimes made using calli without sig->hasthis set, like in the delegate
1034          * invoke wrappers.
1035          */
1036         if (cinfo->vtype_retaddr && !is_pinvoke && (sig->hasthis || (sig->param_count > 0 && MONO_TYPE_IS_REFERENCE (mini_type_get_underlying_type (gsctx, sig->params [0]))))) {
1037                 if (sig->hasthis) {
1038                         add_general (&gr, &stack_size, cinfo->args + 0);
1039                 } else {
1040                         add_general (&gr, &stack_size, &cinfo->args [sig->hasthis + 0]);
1041                         pstart = 1;
1042                 }
1043                 add_general (&gr, &stack_size, &cinfo->ret);
1044                 cinfo->vret_arg_index = 1;
1045         } else {
1046                 /* this */
1047                 if (sig->hasthis)
1048                         add_general (&gr, &stack_size, cinfo->args + 0);
1049
1050                 if (cinfo->vtype_retaddr)
1051                         add_general (&gr, &stack_size, &cinfo->ret);
1052         }
1053
1054         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n == 0)) {
1055                 gr = PARAM_REGS;
1056                 fr = FLOAT_PARAM_REGS;
1057
1058                 /* Emit the signature cookie just before the implicit arguments */
1059                 add_general (&gr, &stack_size, &cinfo->sig_cookie);
1060         }
1061
1062         for (i = pstart; i < sig->param_count; ++i) {
1063                 ArgInfo *ainfo = &cinfo->args [sig->hasthis + i];
1064                 MonoType *ptype;
1065
1066 #ifdef HOST_WIN32
1067                 /* The float param registers and other param registers must be the same index on Windows x64.*/
1068                 if (gr > fr)
1069                         fr = gr;
1070                 else if (fr > gr)
1071                         gr = fr;
1072 #endif
1073
1074                 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
1075                         /* We allways pass the sig cookie on the stack for simplicity */
1076                         /*
1077                          * Prevent implicit arguments + the sig cookie from being passed
1078                          * in registers.
1079                          */
1080                         gr = PARAM_REGS;
1081                         fr = FLOAT_PARAM_REGS;
1082
1083                         /* Emit the signature cookie just before the implicit arguments */
1084                         add_general (&gr, &stack_size, &cinfo->sig_cookie);
1085                 }
1086
1087                 ptype = mini_type_get_underlying_type (gsctx, sig->params [i]);
1088                 switch (ptype->type) {
1089                 case MONO_TYPE_BOOLEAN:
1090                 case MONO_TYPE_I1:
1091                 case MONO_TYPE_U1:
1092                         add_general (&gr, &stack_size, ainfo);
1093                         break;
1094                 case MONO_TYPE_I2:
1095                 case MONO_TYPE_U2:
1096                 case MONO_TYPE_CHAR:
1097                         add_general (&gr, &stack_size, ainfo);
1098                         break;
1099                 case MONO_TYPE_I4:
1100                 case MONO_TYPE_U4:
1101                         add_general (&gr, &stack_size, ainfo);
1102                         break;
1103                 case MONO_TYPE_I:
1104                 case MONO_TYPE_U:
1105                 case MONO_TYPE_PTR:
1106                 case MONO_TYPE_FNPTR:
1107                 case MONO_TYPE_CLASS:
1108                 case MONO_TYPE_OBJECT:
1109                 case MONO_TYPE_STRING:
1110                 case MONO_TYPE_SZARRAY:
1111                 case MONO_TYPE_ARRAY:
1112                         add_general (&gr, &stack_size, ainfo);
1113                         break;
1114                 case MONO_TYPE_GENERICINST:
1115                         if (!mono_type_generic_inst_is_valuetype (ptype)) {
1116                                 add_general (&gr, &stack_size, ainfo);
1117                                 break;
1118                         }
1119                         /* fall through */
1120                 case MONO_TYPE_VALUETYPE:
1121                         add_valuetype (gsctx, sig, ainfo, sig->params [i], FALSE, &gr, &fr, &stack_size);
1122                         break;
1123                 case MONO_TYPE_TYPEDBYREF:
1124 #if defined( HOST_WIN32 ) || defined( __native_client_codegen__ )
1125                         add_valuetype (gsctx, sig, ainfo, sig->params [i], FALSE, &gr, &fr, &stack_size);
1126 #else
1127                         stack_size += sizeof (MonoTypedRef);
1128                         ainfo->storage = ArgOnStack;
1129 #endif
1130                         break;
1131                 case MONO_TYPE_U8:
1132                 case MONO_TYPE_I8:
1133                         add_general (&gr, &stack_size, ainfo);
1134                         break;
1135                 case MONO_TYPE_R4:
1136                         add_float (&fr, &stack_size, ainfo, FALSE);
1137                         break;
1138                 case MONO_TYPE_R8:
1139                         add_float (&fr, &stack_size, ainfo, TRUE);
1140                         break;
1141                 default:
1142                         g_assert_not_reached ();
1143                 }
1144         }
1145
1146         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n > 0) && (sig->sentinelpos == sig->param_count)) {
1147                 gr = PARAM_REGS;
1148                 fr = FLOAT_PARAM_REGS;
1149
1150                 /* Emit the signature cookie just before the implicit arguments */
1151                 add_general (&gr, &stack_size, &cinfo->sig_cookie);
1152         }
1153
1154 #ifdef HOST_WIN32
1155         // There always is 32 bytes reserved on the stack when calling on Winx64
1156         stack_size += 0x20;
1157 #endif
1158
1159 #ifndef MONO_AMD64_NO_PUSHES
1160         if (stack_size & 0x8) {
1161                 /* The AMD64 ABI requires each stack frame to be 16 byte aligned */
1162                 cinfo->need_stack_align = TRUE;
1163                 stack_size += 8;
1164         }
1165 #endif
1166
1167         cinfo->stack_usage = stack_size;
1168         cinfo->reg_usage = gr;
1169         cinfo->freg_usage = fr;
1170         return cinfo;
1171 }
1172
1173 /*
1174  * mono_arch_get_argument_info:
1175  * @csig:  a method signature
1176  * @param_count: the number of parameters to consider
1177  * @arg_info: an array to store the result infos
1178  *
1179  * Gathers information on parameters such as size, alignment and
1180  * padding. arg_info should be large enought to hold param_count + 1 entries.
1181  *
1182  * Returns the size of the argument area on the stack.
1183  */
1184 int
1185 mono_arch_get_argument_info (MonoGenericSharingContext *gsctx, MonoMethodSignature *csig, int param_count, MonoJitArgumentInfo *arg_info)
1186 {
1187         int k;
1188         CallInfo *cinfo = get_call_info (NULL, NULL, csig);
1189         guint32 args_size = cinfo->stack_usage;
1190
1191         /* The arguments are saved to a stack area in mono_arch_instrument_prolog */
1192         if (csig->hasthis) {
1193                 arg_info [0].offset = 0;
1194         }
1195
1196         for (k = 0; k < param_count; k++) {
1197                 arg_info [k + 1].offset = ((k + csig->hasthis) * 8);
1198                 /* FIXME: */
1199                 arg_info [k + 1].size = 0;
1200         }
1201
1202         g_free (cinfo);
1203
1204         return args_size;
1205 }
1206
1207 gboolean
1208 mono_amd64_tail_call_supported (MonoMethodSignature *caller_sig, MonoMethodSignature *callee_sig)
1209 {
1210         CallInfo *c1, *c2;
1211         gboolean res;
1212
1213         c1 = get_call_info (NULL, NULL, caller_sig);
1214         c2 = get_call_info (NULL, NULL, callee_sig);
1215         res = c1->stack_usage >= c2->stack_usage;
1216         if (callee_sig->ret && MONO_TYPE_ISSTRUCT (callee_sig->ret) && c2->ret.storage != ArgValuetypeInReg)
1217                 /* An address on the callee's stack is passed as the first argument */
1218                 res = FALSE;
1219
1220         g_free (c1);
1221         g_free (c2);
1222
1223         return res;
1224 }
1225
1226 static int
1227 cpuid (int id, int* p_eax, int* p_ebx, int* p_ecx, int* p_edx)
1228 {
1229 #if defined(MONO_CROSS_COMPILE)
1230         return 0;
1231 #else
1232 #ifndef _MSC_VER
1233         __asm__ __volatile__ ("cpuid"
1234                 : "=a" (*p_eax), "=b" (*p_ebx), "=c" (*p_ecx), "=d" (*p_edx)
1235                 : "a" (id));
1236 #else
1237         int info[4];
1238         __cpuid(info, id);
1239         *p_eax = info[0];
1240         *p_ebx = info[1];
1241         *p_ecx = info[2];
1242         *p_edx = info[3];
1243 #endif
1244         return 1;
1245 #endif
1246 }
1247
1248 /*
1249  * Initialize the cpu to execute managed code.
1250  */
1251 void
1252 mono_arch_cpu_init (void)
1253 {
1254 #ifndef _MSC_VER
1255         guint16 fpcw;
1256
1257         /* spec compliance requires running with double precision */
1258         __asm__  __volatile__ ("fnstcw %0\n": "=m" (fpcw));
1259         fpcw &= ~X86_FPCW_PRECC_MASK;
1260         fpcw |= X86_FPCW_PREC_DOUBLE;
1261         __asm__  __volatile__ ("fldcw %0\n": : "m" (fpcw));
1262         __asm__  __volatile__ ("fnstcw %0\n": "=m" (fpcw));
1263 #else
1264         /* TODO: This is crashing on Win64 right now.
1265         * _control87 (_PC_53, MCW_PC);
1266         */
1267 #endif
1268 }
1269
1270 /*
1271  * Initialize architecture specific code.
1272  */
1273 void
1274 mono_arch_init (void)
1275 {
1276         int flags;
1277
1278         InitializeCriticalSection (&mini_arch_mutex);
1279 #if defined(__native_client_codegen__)
1280         mono_native_tls_alloc (&nacl_instruction_depth, NULL);
1281         mono_native_tls_set_value (nacl_instruction_depth, (gpointer)0);
1282         mono_native_tls_alloc (&nacl_rex_tag, NULL);
1283         mono_native_tls_alloc (&nacl_legacy_prefix_tag, NULL);
1284 #endif
1285
1286 #ifdef MONO_ARCH_NOMAP32BIT
1287         flags = MONO_MMAP_READ;
1288         /* amd64_mov_reg_imm () + amd64_mov_reg_membase () */
1289         breakpoint_size = 13;
1290         breakpoint_fault_size = 3;
1291 #else
1292         flags = MONO_MMAP_READ|MONO_MMAP_32BIT;
1293         /* amd64_mov_reg_mem () */
1294         breakpoint_size = 8;
1295         breakpoint_fault_size = 8;
1296 #endif
1297
1298         /* amd64_alu_membase_imm_size (code, X86_CMP, AMD64_R11, 0, 0, 4); */
1299         single_step_fault_size = 4;
1300
1301         ss_trigger_page = mono_valloc (NULL, mono_pagesize (), flags);
1302         bp_trigger_page = mono_valloc (NULL, mono_pagesize (), flags);
1303         mono_mprotect (bp_trigger_page, mono_pagesize (), 0);
1304
1305         mono_aot_register_jit_icall ("mono_amd64_throw_exception", mono_amd64_throw_exception);
1306         mono_aot_register_jit_icall ("mono_amd64_throw_corlib_exception", mono_amd64_throw_corlib_exception);
1307         mono_aot_register_jit_icall ("mono_amd64_get_original_ip", mono_amd64_get_original_ip);
1308 }
1309
1310 /*
1311  * Cleanup architecture specific code.
1312  */
1313 void
1314 mono_arch_cleanup (void)
1315 {
1316         DeleteCriticalSection (&mini_arch_mutex);
1317 #if defined(__native_client_codegen__)
1318         mono_native_tls_free (nacl_instruction_depth);
1319         mono_native_tls_free (nacl_rex_tag);
1320         mono_native_tls_free (nacl_legacy_prefix_tag);
1321 #endif
1322 }
1323
1324 /*
1325  * This function returns the optimizations supported on this cpu.
1326  */
1327 guint32
1328 mono_arch_cpu_optimizations (guint32 *exclude_mask)
1329 {
1330         int eax, ebx, ecx, edx;
1331         guint32 opts = 0;
1332
1333         *exclude_mask = 0;
1334         /* Feature Flags function, flags returned in EDX. */
1335         if (cpuid (1, &eax, &ebx, &ecx, &edx)) {
1336                 if (edx & (1 << 15)) {
1337                         opts |= MONO_OPT_CMOV;
1338                         if (edx & 1)
1339                                 opts |= MONO_OPT_FCMOV;
1340                         else
1341                                 *exclude_mask |= MONO_OPT_FCMOV;
1342                 } else
1343                         *exclude_mask |= MONO_OPT_CMOV;
1344         }
1345
1346         return opts;
1347 }
1348
1349 /*
1350  * This function test for all SSE functions supported.
1351  *
1352  * Returns a bitmask corresponding to all supported versions.
1353  *
1354  */
1355 guint32
1356 mono_arch_cpu_enumerate_simd_versions (void)
1357 {
1358         int eax, ebx, ecx, edx;
1359         guint32 sse_opts = 0;
1360
1361         if (cpuid (1, &eax, &ebx, &ecx, &edx)) {
1362                 if (edx & (1 << 25))
1363                         sse_opts |= SIMD_VERSION_SSE1;
1364                 if (edx & (1 << 26))
1365                         sse_opts |= SIMD_VERSION_SSE2;
1366                 if (ecx & (1 << 0))
1367                         sse_opts |= SIMD_VERSION_SSE3;
1368                 if (ecx & (1 << 9))
1369                         sse_opts |= SIMD_VERSION_SSSE3;
1370                 if (ecx & (1 << 19))
1371                         sse_opts |= SIMD_VERSION_SSE41;
1372                 if (ecx & (1 << 20))
1373                         sse_opts |= SIMD_VERSION_SSE42;
1374         }
1375
1376         /* Yes, all this needs to be done to check for sse4a.
1377            See: "Amd: CPUID Specification"
1378          */
1379         if (cpuid (0x80000000, &eax, &ebx, &ecx, &edx)) {
1380                 /* eax greater or equal than 0x80000001, ebx = 'htuA', ecx = DMAc', edx = 'itne'*/
1381                 if ((((unsigned int) eax) >= 0x80000001) && (ebx == 0x68747541) && (ecx == 0x444D4163) && (edx == 0x69746E65)) {
1382                         cpuid (0x80000001, &eax, &ebx, &ecx, &edx);
1383                         if (ecx & (1 << 6))
1384                                 sse_opts |= SIMD_VERSION_SSE4a;
1385                 }
1386         }
1387
1388         return sse_opts;
1389 }
1390
1391 #ifndef DISABLE_JIT
1392
1393 GList *
1394 mono_arch_get_allocatable_int_vars (MonoCompile *cfg)
1395 {
1396         GList *vars = NULL;
1397         int i;
1398
1399         for (i = 0; i < cfg->num_varinfo; i++) {
1400                 MonoInst *ins = cfg->varinfo [i];
1401                 MonoMethodVar *vmv = MONO_VARINFO (cfg, i);
1402
1403                 /* unused vars */
1404                 if (vmv->range.first_use.abs_pos >= vmv->range.last_use.abs_pos)
1405                         continue;
1406
1407                 if ((ins->flags & (MONO_INST_IS_DEAD|MONO_INST_VOLATILE|MONO_INST_INDIRECT)) ||
1408                     (ins->opcode != OP_LOCAL && ins->opcode != OP_ARG))
1409                         continue;
1410
1411                 if (mono_is_regsize_var (ins->inst_vtype)) {
1412                         g_assert (MONO_VARINFO (cfg, i)->reg == -1);
1413                         g_assert (i == vmv->idx);
1414                         vars = g_list_prepend (vars, vmv);
1415                 }
1416         }
1417
1418         vars = mono_varlist_sort (cfg, vars, 0);
1419
1420         return vars;
1421 }
1422
1423 /**
1424  * mono_arch_compute_omit_fp:
1425  *
1426  *   Determine whenever the frame pointer can be eliminated.
1427  */
1428 static void
1429 mono_arch_compute_omit_fp (MonoCompile *cfg)
1430 {
1431         MonoMethodSignature *sig;
1432         MonoMethodHeader *header;
1433         int i, locals_size;
1434         CallInfo *cinfo;
1435
1436         if (cfg->arch.omit_fp_computed)
1437                 return;
1438
1439         header = cfg->header;
1440
1441         sig = mono_method_signature (cfg->method);
1442
1443         if (!cfg->arch.cinfo)
1444                 cfg->arch.cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig);
1445         cinfo = cfg->arch.cinfo;
1446
1447         /*
1448          * FIXME: Remove some of the restrictions.
1449          */
1450         cfg->arch.omit_fp = TRUE;
1451         cfg->arch.omit_fp_computed = TRUE;
1452
1453 #ifdef __native_client_codegen__
1454         /* NaCl modules may not change the value of RBP, so it cannot be */
1455         /* used as a normal register, but it can be used as a frame pointer*/
1456         cfg->disable_omit_fp = TRUE;
1457         cfg->arch.omit_fp = FALSE;
1458 #endif
1459
1460         if (cfg->disable_omit_fp)
1461                 cfg->arch.omit_fp = FALSE;
1462
1463         if (!debug_omit_fp ())
1464                 cfg->arch.omit_fp = FALSE;
1465         /*
1466         if (cfg->method->save_lmf)
1467                 cfg->arch.omit_fp = FALSE;
1468         */
1469         if (cfg->flags & MONO_CFG_HAS_ALLOCA)
1470                 cfg->arch.omit_fp = FALSE;
1471         if (header->num_clauses)
1472                 cfg->arch.omit_fp = FALSE;
1473         if (cfg->param_area)
1474                 cfg->arch.omit_fp = FALSE;
1475         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG))
1476                 cfg->arch.omit_fp = FALSE;
1477         if ((mono_jit_trace_calls != NULL && mono_trace_eval (cfg->method)) ||
1478                 (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE))
1479                 cfg->arch.omit_fp = FALSE;
1480         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
1481                 ArgInfo *ainfo = &cinfo->args [i];
1482
1483                 if (ainfo->storage == ArgOnStack) {
1484                         /*
1485                          * The stack offset can only be determined when the frame
1486                          * size is known.
1487                          */
1488                         cfg->arch.omit_fp = FALSE;
1489                 }
1490         }
1491
1492         locals_size = 0;
1493         for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
1494                 MonoInst *ins = cfg->varinfo [i];
1495                 int ialign;
1496
1497                 locals_size += mono_type_size (ins->inst_vtype, &ialign);
1498         }
1499 }
1500
1501 GList *
1502 mono_arch_get_global_int_regs (MonoCompile *cfg)
1503 {
1504         GList *regs = NULL;
1505
1506         mono_arch_compute_omit_fp (cfg);
1507
1508         if (cfg->globalra) {
1509                 if (cfg->arch.omit_fp)
1510                         regs = g_list_prepend (regs, (gpointer)AMD64_RBP);
1511
1512                 regs = g_list_prepend (regs, (gpointer)AMD64_RBX);
1513                 regs = g_list_prepend (regs, (gpointer)AMD64_R12);
1514                 regs = g_list_prepend (regs, (gpointer)AMD64_R13);
1515                 regs = g_list_prepend (regs, (gpointer)AMD64_R14);
1516 #ifndef __native_client_codegen__
1517                 regs = g_list_prepend (regs, (gpointer)AMD64_R15);
1518 #endif
1519
1520                 regs = g_list_prepend (regs, (gpointer)AMD64_R10);
1521                 regs = g_list_prepend (regs, (gpointer)AMD64_R9);
1522                 regs = g_list_prepend (regs, (gpointer)AMD64_R8);
1523                 regs = g_list_prepend (regs, (gpointer)AMD64_RDI);
1524                 regs = g_list_prepend (regs, (gpointer)AMD64_RSI);
1525                 regs = g_list_prepend (regs, (gpointer)AMD64_RDX);
1526                 regs = g_list_prepend (regs, (gpointer)AMD64_RCX);
1527                 regs = g_list_prepend (regs, (gpointer)AMD64_RAX);
1528         } else {
1529                 if (cfg->arch.omit_fp)
1530                         regs = g_list_prepend (regs, (gpointer)AMD64_RBP);
1531
1532                 /* We use the callee saved registers for global allocation */
1533                 regs = g_list_prepend (regs, (gpointer)AMD64_RBX);
1534                 regs = g_list_prepend (regs, (gpointer)AMD64_R12);
1535                 regs = g_list_prepend (regs, (gpointer)AMD64_R13);
1536                 regs = g_list_prepend (regs, (gpointer)AMD64_R14);
1537 #ifndef __native_client_codegen__
1538                 regs = g_list_prepend (regs, (gpointer)AMD64_R15);
1539 #endif
1540 #ifdef HOST_WIN32
1541                 regs = g_list_prepend (regs, (gpointer)AMD64_RDI);
1542                 regs = g_list_prepend (regs, (gpointer)AMD64_RSI);
1543 #endif
1544         }
1545
1546         return regs;
1547 }
1548
1549 GList*
1550 mono_arch_get_global_fp_regs (MonoCompile *cfg)
1551 {
1552         GList *regs = NULL;
1553         int i;
1554
1555         /* All XMM registers */
1556         for (i = 0; i < 16; ++i)
1557                 regs = g_list_prepend (regs, GINT_TO_POINTER (i));
1558
1559         return regs;
1560 }
1561
1562 GList*
1563 mono_arch_get_iregs_clobbered_by_call (MonoCallInst *call)
1564 {
1565         static GList *r = NULL;
1566
1567         if (r == NULL) {
1568                 GList *regs = NULL;
1569
1570                 regs = g_list_prepend (regs, (gpointer)AMD64_RBP);
1571                 regs = g_list_prepend (regs, (gpointer)AMD64_RBX);
1572                 regs = g_list_prepend (regs, (gpointer)AMD64_R12);
1573                 regs = g_list_prepend (regs, (gpointer)AMD64_R13);
1574                 regs = g_list_prepend (regs, (gpointer)AMD64_R14);
1575 #ifndef __native_client_codegen__
1576                 regs = g_list_prepend (regs, (gpointer)AMD64_R15);
1577 #endif
1578
1579                 regs = g_list_prepend (regs, (gpointer)AMD64_R10);
1580                 regs = g_list_prepend (regs, (gpointer)AMD64_R9);
1581                 regs = g_list_prepend (regs, (gpointer)AMD64_R8);
1582                 regs = g_list_prepend (regs, (gpointer)AMD64_RDI);
1583                 regs = g_list_prepend (regs, (gpointer)AMD64_RSI);
1584                 regs = g_list_prepend (regs, (gpointer)AMD64_RDX);
1585                 regs = g_list_prepend (regs, (gpointer)AMD64_RCX);
1586                 regs = g_list_prepend (regs, (gpointer)AMD64_RAX);
1587
1588                 InterlockedCompareExchangePointer ((gpointer*)&r, regs, NULL);
1589         }
1590
1591         return r;
1592 }
1593
1594 GList*
1595 mono_arch_get_fregs_clobbered_by_call (MonoCallInst *call)
1596 {
1597         int i;
1598         static GList *r = NULL;
1599
1600         if (r == NULL) {
1601                 GList *regs = NULL;
1602
1603                 for (i = 0; i < AMD64_XMM_NREG; ++i)
1604                         regs = g_list_prepend (regs, GINT_TO_POINTER (MONO_MAX_IREGS + i));
1605
1606                 InterlockedCompareExchangePointer ((gpointer*)&r, regs, NULL);
1607         }
1608
1609         return r;
1610 }
1611
1612 /*
1613  * mono_arch_regalloc_cost:
1614  *
1615  *  Return the cost, in number of memory references, of the action of
1616  * allocating the variable VMV into a register during global register
1617  * allocation.
1618  */
1619 guint32
1620 mono_arch_regalloc_cost (MonoCompile *cfg, MonoMethodVar *vmv)
1621 {
1622         MonoInst *ins = cfg->varinfo [vmv->idx];
1623
1624         if (cfg->method->save_lmf)
1625                 /* The register is already saved */
1626                 /* substract 1 for the invisible store in the prolog */
1627                 return (ins->opcode == OP_ARG) ? 0 : 1;
1628         else
1629                 /* push+pop */
1630                 return (ins->opcode == OP_ARG) ? 1 : 2;
1631 }
1632
1633 /*
1634  * mono_arch_fill_argument_info:
1635  *
1636  *   Populate cfg->args, cfg->ret and cfg->vret_addr with information about the arguments
1637  * of the method.
1638  */
1639 void
1640 mono_arch_fill_argument_info (MonoCompile *cfg)
1641 {
1642         MonoMethodSignature *sig;
1643         MonoMethodHeader *header;
1644         MonoInst *ins;
1645         int i;
1646         CallInfo *cinfo;
1647
1648         header = cfg->header;
1649
1650         sig = mono_method_signature (cfg->method);
1651
1652         cinfo = cfg->arch.cinfo;
1653
1654         /*
1655          * Contrary to mono_arch_allocate_vars (), the information should describe
1656          * where the arguments are at the beginning of the method, not where they can be
1657          * accessed during the execution of the method. The later makes no sense for the
1658          * global register allocator, since a variable can be in more than one location.
1659          */
1660         if (sig->ret->type != MONO_TYPE_VOID) {
1661                 switch (cinfo->ret.storage) {
1662                 case ArgInIReg:
1663                 case ArgInFloatSSEReg:
1664                 case ArgInDoubleSSEReg:
1665                         if ((MONO_TYPE_ISSTRUCT (sig->ret) && !mono_class_from_mono_type (sig->ret)->enumtype) || ((sig->ret->type == MONO_TYPE_TYPEDBYREF) && cinfo->vtype_retaddr)) {
1666                                 cfg->vret_addr->opcode = OP_REGVAR;
1667                                 cfg->vret_addr->inst_c0 = cinfo->ret.reg;
1668                         }
1669                         else {
1670                                 cfg->ret->opcode = OP_REGVAR;
1671                                 cfg->ret->inst_c0 = cinfo->ret.reg;
1672                         }
1673                         break;
1674                 case ArgValuetypeInReg:
1675                         cfg->ret->opcode = OP_REGOFFSET;
1676                         cfg->ret->inst_basereg = -1;
1677                         cfg->ret->inst_offset = -1;
1678                         break;
1679                 default:
1680                         g_assert_not_reached ();
1681                 }
1682         }
1683
1684         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
1685                 ArgInfo *ainfo = &cinfo->args [i];
1686                 MonoType *arg_type;
1687
1688                 ins = cfg->args [i];
1689
1690                 if (sig->hasthis && (i == 0))
1691                         arg_type = &mono_defaults.object_class->byval_arg;
1692                 else
1693                         arg_type = sig->params [i - sig->hasthis];
1694
1695                 switch (ainfo->storage) {
1696                 case ArgInIReg:
1697                 case ArgInFloatSSEReg:
1698                 case ArgInDoubleSSEReg:
1699                         ins->opcode = OP_REGVAR;
1700                         ins->inst_c0 = ainfo->reg;
1701                         break;
1702                 case ArgOnStack:
1703                         ins->opcode = OP_REGOFFSET;
1704                         ins->inst_basereg = -1;
1705                         ins->inst_offset = -1;
1706                         break;
1707                 case ArgValuetypeInReg:
1708                         /* Dummy */
1709                         ins->opcode = OP_NOP;
1710                         break;
1711                 default:
1712                         g_assert_not_reached ();
1713                 }
1714         }
1715 }
1716
1717 void
1718 mono_arch_allocate_vars (MonoCompile *cfg)
1719 {
1720         MonoMethodSignature *sig;
1721         MonoMethodHeader *header;
1722         MonoInst *ins;
1723         int i, offset;
1724         guint32 locals_stack_size, locals_stack_align;
1725         gint32 *offsets;
1726         CallInfo *cinfo;
1727
1728         header = cfg->header;
1729
1730         sig = mono_method_signature (cfg->method);
1731
1732         cinfo = cfg->arch.cinfo;
1733
1734         mono_arch_compute_omit_fp (cfg);
1735
1736         /*
1737          * We use the ABI calling conventions for managed code as well.
1738          * Exception: valuetypes are only sometimes passed or returned in registers.
1739          */
1740
1741         /*
1742          * The stack looks like this:
1743          * <incoming arguments passed on the stack>
1744          * <return value>
1745          * <lmf/caller saved registers>
1746          * <locals>
1747          * <spill area>
1748          * <localloc area>  -> grows dynamically
1749          * <params area>
1750          */
1751
1752         if (cfg->arch.omit_fp) {
1753                 cfg->flags |= MONO_CFG_HAS_SPILLUP;
1754                 cfg->frame_reg = AMD64_RSP;
1755                 offset = 0;
1756         } else {
1757                 /* Locals are allocated backwards from %fp */
1758                 cfg->frame_reg = AMD64_RBP;
1759                 offset = 0;
1760         }
1761
1762         if (cfg->method->save_lmf) {
1763                 /* The LMF var is allocated normally */
1764         } else {
1765                 if (cfg->arch.omit_fp)
1766                         cfg->arch.reg_save_area_offset = offset;
1767                 /* Reserve space for callee saved registers */
1768                 for (i = 0; i < AMD64_NREG; ++i)
1769                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
1770                                 offset += sizeof(mgreg_t);
1771                         }
1772         }
1773
1774         if (sig->ret->type != MONO_TYPE_VOID) {
1775                 switch (cinfo->ret.storage) {
1776                 case ArgInIReg:
1777                 case ArgInFloatSSEReg:
1778                 case ArgInDoubleSSEReg:
1779                         if ((MONO_TYPE_ISSTRUCT (sig->ret) && !mono_class_from_mono_type (sig->ret)->enumtype) || ((sig->ret->type == MONO_TYPE_TYPEDBYREF) && cinfo->vtype_retaddr)) {
1780                                 if (cfg->globalra) {
1781                                         cfg->vret_addr->opcode = OP_REGVAR;
1782                                         cfg->vret_addr->inst_c0 = cinfo->ret.reg;
1783                                 } else {
1784                                         /* The register is volatile */
1785                                         cfg->vret_addr->opcode = OP_REGOFFSET;
1786                                         cfg->vret_addr->inst_basereg = cfg->frame_reg;
1787                                         if (cfg->arch.omit_fp) {
1788                                                 cfg->vret_addr->inst_offset = offset;
1789                                                 offset += 8;
1790                                         } else {
1791                                                 offset += 8;
1792                                                 cfg->vret_addr->inst_offset = -offset;
1793                                         }
1794                                         if (G_UNLIKELY (cfg->verbose_level > 1)) {
1795                                                 printf ("vret_addr =");
1796                                                 mono_print_ins (cfg->vret_addr);
1797                                         }
1798                                 }
1799                         }
1800                         else {
1801                                 cfg->ret->opcode = OP_REGVAR;
1802                                 cfg->ret->inst_c0 = cinfo->ret.reg;
1803                         }
1804                         break;
1805                 case ArgValuetypeInReg:
1806                         /* Allocate a local to hold the result, the epilog will copy it to the correct place */
1807                         cfg->ret->opcode = OP_REGOFFSET;
1808                         cfg->ret->inst_basereg = cfg->frame_reg;
1809                         if (cfg->arch.omit_fp) {
1810                                 cfg->ret->inst_offset = offset;
1811                                 offset += cinfo->ret.pair_storage [1] == ArgNone ? 8 : 16;
1812                         } else {
1813                                 offset += cinfo->ret.pair_storage [1] == ArgNone ? 8 : 16;
1814                                 cfg->ret->inst_offset = - offset;
1815                         }
1816                         break;
1817                 default:
1818                         g_assert_not_reached ();
1819                 }
1820                 if (!cfg->globalra)
1821                         cfg->ret->dreg = cfg->ret->inst_c0;
1822         }
1823
1824         /* Allocate locals */
1825         if (!cfg->globalra) {
1826                 offsets = mono_allocate_stack_slots (cfg, cfg->arch.omit_fp ? FALSE: TRUE, &locals_stack_size, &locals_stack_align);
1827                 if (locals_stack_size > MONO_ARCH_MAX_FRAME_SIZE) {
1828                         char *mname = mono_method_full_name (cfg->method, TRUE);
1829                         cfg->exception_type = MONO_EXCEPTION_INVALID_PROGRAM;
1830                         cfg->exception_message = g_strdup_printf ("Method %s stack is too big.", mname);
1831                         g_free (mname);
1832                         return;
1833                 }
1834
1835                 if (locals_stack_align) {
1836                         offset += (locals_stack_align - 1);
1837                         offset &= ~(locals_stack_align - 1);
1838                 }
1839                 if (cfg->arch.omit_fp) {
1840                         cfg->locals_min_stack_offset = offset;
1841                         cfg->locals_max_stack_offset = offset + locals_stack_size;
1842                 } else {
1843                         cfg->locals_min_stack_offset = - (offset + locals_stack_size);
1844                         cfg->locals_max_stack_offset = - offset;
1845                 }
1846
1847                 for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
1848                         if (offsets [i] != -1) {
1849                                 MonoInst *ins = cfg->varinfo [i];
1850                                 ins->opcode = OP_REGOFFSET;
1851                                 ins->inst_basereg = cfg->frame_reg;
1852                                 if (cfg->arch.omit_fp)
1853                                         ins->inst_offset = (offset + offsets [i]);
1854                                 else
1855                                         ins->inst_offset = - (offset + offsets [i]);
1856                                 //printf ("allocated local %d to ", i); mono_print_tree_nl (ins);
1857                         }
1858                 }
1859                 offset += locals_stack_size;
1860         }
1861
1862         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG)) {
1863                 g_assert (!cfg->arch.omit_fp);
1864                 g_assert (cinfo->sig_cookie.storage == ArgOnStack);
1865                 cfg->sig_cookie = cinfo->sig_cookie.offset + ARGS_OFFSET;
1866         }
1867
1868         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
1869                 ins = cfg->args [i];
1870                 if (ins->opcode != OP_REGVAR) {
1871                         ArgInfo *ainfo = &cinfo->args [i];
1872                         gboolean inreg = TRUE;
1873                         MonoType *arg_type;
1874
1875                         if (sig->hasthis && (i == 0))
1876                                 arg_type = &mono_defaults.object_class->byval_arg;
1877                         else
1878                                 arg_type = sig->params [i - sig->hasthis];
1879
1880                         if (cfg->globalra) {
1881                                 /* The new allocator needs info about the original locations of the arguments */
1882                                 switch (ainfo->storage) {
1883                                 case ArgInIReg:
1884                                 case ArgInFloatSSEReg:
1885                                 case ArgInDoubleSSEReg:
1886                                         ins->opcode = OP_REGVAR;
1887                                         ins->inst_c0 = ainfo->reg;
1888                                         break;
1889                                 case ArgOnStack:
1890                                         g_assert (!cfg->arch.omit_fp);
1891                                         ins->opcode = OP_REGOFFSET;
1892                                         ins->inst_basereg = cfg->frame_reg;
1893                                         ins->inst_offset = ainfo->offset + ARGS_OFFSET;
1894                                         break;
1895                                 case ArgValuetypeInReg:
1896                                         ins->opcode = OP_REGOFFSET;
1897                                         ins->inst_basereg = cfg->frame_reg;
1898                                         /* These arguments are saved to the stack in the prolog */
1899                                         offset = ALIGN_TO (offset, sizeof(mgreg_t));
1900                                         if (cfg->arch.omit_fp) {
1901                                                 ins->inst_offset = offset;
1902                                                 offset += (ainfo->storage == ArgValuetypeInReg) ? ainfo->nregs * sizeof (mgreg_t) : sizeof (mgreg_t);
1903                                         } else {
1904                                                 offset += (ainfo->storage == ArgValuetypeInReg) ? ainfo->nregs * sizeof (mgreg_t) : sizeof (mgreg_t);
1905                                                 ins->inst_offset = - offset;
1906                                         }
1907                                         break;
1908                                 default:
1909                                         g_assert_not_reached ();
1910                                 }
1911
1912                                 continue;
1913                         }
1914
1915                         /* FIXME: Allocate volatile arguments to registers */
1916                         if (ins->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT))
1917                                 inreg = FALSE;
1918
1919                         /*
1920                          * Under AMD64, all registers used to pass arguments to functions
1921                          * are volatile across calls.
1922                          * FIXME: Optimize this.
1923                          */
1924                         if ((ainfo->storage == ArgInIReg) || (ainfo->storage == ArgInFloatSSEReg) || (ainfo->storage == ArgInDoubleSSEReg) || (ainfo->storage == ArgValuetypeInReg))
1925                                 inreg = FALSE;
1926
1927                         ins->opcode = OP_REGOFFSET;
1928
1929                         switch (ainfo->storage) {
1930                         case ArgInIReg:
1931                         case ArgInFloatSSEReg:
1932                         case ArgInDoubleSSEReg:
1933                                 if (inreg) {
1934                                         ins->opcode = OP_REGVAR;
1935                                         ins->dreg = ainfo->reg;
1936                                 }
1937                                 break;
1938                         case ArgOnStack:
1939                                 g_assert (!cfg->arch.omit_fp);
1940                                 ins->opcode = OP_REGOFFSET;
1941                                 ins->inst_basereg = cfg->frame_reg;
1942                                 ins->inst_offset = ainfo->offset + ARGS_OFFSET;
1943                                 break;
1944                         case ArgValuetypeInReg:
1945                                 break;
1946                         case ArgValuetypeAddrInIReg: {
1947                                 MonoInst *indir;
1948                                 g_assert (!cfg->arch.omit_fp);
1949
1950                                 MONO_INST_NEW (cfg, indir, 0);
1951                                 indir->opcode = OP_REGOFFSET;
1952                                 if (ainfo->pair_storage [0] == ArgInIReg) {
1953                                         indir->inst_basereg = cfg->frame_reg;
1954                                         offset = ALIGN_TO (offset, sizeof (gpointer));
1955                                         offset += (sizeof (gpointer));
1956                                         indir->inst_offset = - offset;
1957                                 }
1958                                 else {
1959                                         indir->inst_basereg = cfg->frame_reg;
1960                                         indir->inst_offset = ainfo->offset + ARGS_OFFSET;
1961                                 }
1962
1963                                 ins->opcode = OP_VTARG_ADDR;
1964                                 ins->inst_left = indir;
1965
1966                                 break;
1967                         }
1968                         default:
1969                                 NOT_IMPLEMENTED;
1970                         }
1971
1972                         if (!inreg && (ainfo->storage != ArgOnStack) && (ainfo->storage != ArgValuetypeAddrInIReg)) {
1973                                 ins->opcode = OP_REGOFFSET;
1974                                 ins->inst_basereg = cfg->frame_reg;
1975                                 /* These arguments are saved to the stack in the prolog */
1976                                 offset = ALIGN_TO (offset, sizeof(mgreg_t));
1977                                 if (cfg->arch.omit_fp) {
1978                                         ins->inst_offset = offset;
1979                                         offset += (ainfo->storage == ArgValuetypeInReg) ? ainfo->nregs * sizeof (mgreg_t) : sizeof (mgreg_t);
1980                                         // Arguments are yet supported by the stack map creation code
1981                                         //cfg->locals_max_stack_offset = MAX (cfg->locals_max_stack_offset, offset);
1982                                 } else {
1983                                         offset += (ainfo->storage == ArgValuetypeInReg) ? ainfo->nregs * sizeof (mgreg_t) : sizeof (mgreg_t);
1984                                         ins->inst_offset = - offset;
1985                                         //cfg->locals_min_stack_offset = MIN (cfg->locals_min_stack_offset, offset);
1986                                 }
1987                         }
1988                 }
1989         }
1990
1991         cfg->stack_offset = offset;
1992 }
1993
1994 void
1995 mono_arch_create_vars (MonoCompile *cfg)
1996 {
1997         MonoMethodSignature *sig;
1998         CallInfo *cinfo;
1999
2000         sig = mono_method_signature (cfg->method);
2001
2002         if (!cfg->arch.cinfo)
2003                 cfg->arch.cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig);
2004         cinfo = cfg->arch.cinfo;
2005
2006         if (cinfo->ret.storage == ArgValuetypeInReg)
2007                 cfg->ret_var_is_local = TRUE;
2008
2009         if ((cinfo->ret.storage != ArgValuetypeInReg) && MONO_TYPE_ISSTRUCT (sig->ret)) {
2010                 cfg->vret_addr = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_ARG);
2011                 if (G_UNLIKELY (cfg->verbose_level > 1)) {
2012                         printf ("vret_addr = ");
2013                         mono_print_ins (cfg->vret_addr);
2014                 }
2015         }
2016
2017         if (cfg->gen_seq_points) {
2018                 MonoInst *ins;
2019
2020                 if (cfg->compile_aot) {
2021                         MonoInst *ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
2022                         ins->flags |= MONO_INST_VOLATILE;
2023                         cfg->arch.seq_point_info_var = ins;
2024                 }
2025
2026             ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
2027                 ins->flags |= MONO_INST_VOLATILE;
2028                 cfg->arch.ss_trigger_page_var = ins;
2029         }
2030
2031 #ifdef MONO_AMD64_NO_PUSHES
2032         /*
2033          * When this is set, we pass arguments on the stack by moves, and by allocating
2034          * a bigger stack frame, instead of pushes.
2035          * Pushes complicate exception handling because the arguments on the stack have
2036          * to be popped each time a frame is unwound. They also make fp elimination
2037          * impossible.
2038          * FIXME: This doesn't work inside filter/finally clauses, since those execute
2039          * on a new frame which doesn't include a param area.
2040          */
2041         cfg->arch.no_pushes = TRUE;
2042 #endif
2043
2044         if (cfg->method->save_lmf) {
2045                 MonoInst *lmf_var = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
2046                 lmf_var->flags |= MONO_INST_VOLATILE;
2047                 lmf_var->flags |= MONO_INST_LMF;
2048                 cfg->arch.lmf_var = lmf_var;
2049         }
2050
2051 #ifndef MONO_AMD64_NO_PUSHES
2052         cfg->arch_eh_jit_info = 1;
2053 #endif
2054 }
2055
2056 static void
2057 add_outarg_reg (MonoCompile *cfg, MonoCallInst *call, ArgStorage storage, int reg, MonoInst *tree)
2058 {
2059         MonoInst *ins;
2060
2061         switch (storage) {
2062         case ArgInIReg:
2063                 MONO_INST_NEW (cfg, ins, OP_MOVE);
2064                 ins->dreg = mono_alloc_ireg_copy (cfg, tree->dreg);
2065                 ins->sreg1 = tree->dreg;
2066                 MONO_ADD_INS (cfg->cbb, ins);
2067                 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, FALSE);
2068                 break;
2069         case ArgInFloatSSEReg:
2070                 MONO_INST_NEW (cfg, ins, OP_AMD64_SET_XMMREG_R4);
2071                 ins->dreg = mono_alloc_freg (cfg);
2072                 ins->sreg1 = tree->dreg;
2073                 MONO_ADD_INS (cfg->cbb, ins);
2074
2075                 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2076                 break;
2077         case ArgInDoubleSSEReg:
2078                 MONO_INST_NEW (cfg, ins, OP_FMOVE);
2079                 ins->dreg = mono_alloc_freg (cfg);
2080                 ins->sreg1 = tree->dreg;
2081                 MONO_ADD_INS (cfg->cbb, ins);
2082
2083                 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2084
2085                 break;
2086         default:
2087                 g_assert_not_reached ();
2088         }
2089 }
2090
2091 static int
2092 arg_storage_to_load_membase (ArgStorage storage)
2093 {
2094         switch (storage) {
2095         case ArgInIReg:
2096 #if defined(__mono_ilp32__)
2097                 return OP_LOADI8_MEMBASE;
2098 #else
2099                 return OP_LOAD_MEMBASE;
2100 #endif
2101         case ArgInDoubleSSEReg:
2102                 return OP_LOADR8_MEMBASE;
2103         case ArgInFloatSSEReg:
2104                 return OP_LOADR4_MEMBASE;
2105         default:
2106                 g_assert_not_reached ();
2107         }
2108
2109         return -1;
2110 }
2111
2112 static void
2113 emit_sig_cookie (MonoCompile *cfg, MonoCallInst *call, CallInfo *cinfo)
2114 {
2115         MonoInst *arg;
2116         MonoMethodSignature *tmp_sig;
2117         int sig_reg;
2118
2119         if (call->tail_call)
2120                 NOT_IMPLEMENTED;
2121
2122         g_assert (cinfo->sig_cookie.storage == ArgOnStack);
2123
2124         /*
2125          * mono_ArgIterator_Setup assumes the signature cookie is
2126          * passed first and all the arguments which were before it are
2127          * passed on the stack after the signature. So compensate by
2128          * passing a different signature.
2129          */
2130         tmp_sig = mono_metadata_signature_dup_full (cfg->method->klass->image, call->signature);
2131         tmp_sig->param_count -= call->signature->sentinelpos;
2132         tmp_sig->sentinelpos = 0;
2133         memcpy (tmp_sig->params, call->signature->params + call->signature->sentinelpos, tmp_sig->param_count * sizeof (MonoType*));
2134
2135         sig_reg = mono_alloc_ireg (cfg);
2136         MONO_EMIT_NEW_SIGNATURECONST (cfg, sig_reg, tmp_sig);
2137
2138         if (cfg->arch.no_pushes) {
2139                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, AMD64_RSP, cinfo->sig_cookie.offset, sig_reg);
2140         } else {
2141                 MONO_INST_NEW (cfg, arg, OP_X86_PUSH);
2142                 arg->sreg1 = sig_reg;
2143                 MONO_ADD_INS (cfg->cbb, arg);
2144         }
2145 }
2146
2147 static inline LLVMArgStorage
2148 arg_storage_to_llvm_arg_storage (MonoCompile *cfg, ArgStorage storage)
2149 {
2150         switch (storage) {
2151         case ArgInIReg:
2152                 return LLVMArgInIReg;
2153         case ArgNone:
2154                 return LLVMArgNone;
2155         default:
2156                 g_assert_not_reached ();
2157                 return LLVMArgNone;
2158         }
2159 }
2160
2161 #ifdef ENABLE_LLVM
2162 LLVMCallInfo*
2163 mono_arch_get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
2164 {
2165         int i, n;
2166         CallInfo *cinfo;
2167         ArgInfo *ainfo;
2168         int j;
2169         LLVMCallInfo *linfo;
2170         MonoType *t;
2171
2172         n = sig->param_count + sig->hasthis;
2173
2174         cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig);
2175
2176         linfo = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
2177
2178         /*
2179          * LLVM always uses the native ABI while we use our own ABI, the
2180          * only difference is the handling of vtypes:
2181          * - we only pass/receive them in registers in some cases, and only
2182          *   in 1 or 2 integer registers.
2183          */
2184         if (cinfo->ret.storage == ArgValuetypeInReg) {
2185                 if (sig->pinvoke) {
2186                         cfg->exception_message = g_strdup ("pinvoke + vtypes");
2187                         cfg->disable_llvm = TRUE;
2188                         return linfo;
2189                 }
2190
2191                 linfo->ret.storage = LLVMArgVtypeInReg;
2192                 for (j = 0; j < 2; ++j)
2193                         linfo->ret.pair_storage [j] = arg_storage_to_llvm_arg_storage (cfg, cinfo->ret.pair_storage [j]);
2194         }
2195
2196         if (MONO_TYPE_ISSTRUCT (sig->ret) && cinfo->ret.storage == ArgInIReg) {
2197                 /* Vtype returned using a hidden argument */
2198                 linfo->ret.storage = LLVMArgVtypeRetAddr;
2199                 linfo->vret_arg_index = cinfo->vret_arg_index;
2200         }
2201
2202         for (i = 0; i < n; ++i) {
2203                 ainfo = cinfo->args + i;
2204
2205                 if (i >= sig->hasthis)
2206                         t = sig->params [i - sig->hasthis];
2207                 else
2208                         t = &mono_defaults.int_class->byval_arg;
2209
2210                 linfo->args [i].storage = LLVMArgNone;
2211
2212                 switch (ainfo->storage) {
2213                 case ArgInIReg:
2214                         linfo->args [i].storage = LLVMArgInIReg;
2215                         break;
2216                 case ArgInDoubleSSEReg:
2217                 case ArgInFloatSSEReg:
2218                         linfo->args [i].storage = LLVMArgInFPReg;
2219                         break;
2220                 case ArgOnStack:
2221                         if (MONO_TYPE_ISSTRUCT (t)) {
2222                                 linfo->args [i].storage = LLVMArgVtypeByVal;
2223                         } else {
2224                                 linfo->args [i].storage = LLVMArgInIReg;
2225                                 if (!t->byref) {
2226                                         if (t->type == MONO_TYPE_R4)
2227                                                 linfo->args [i].storage = LLVMArgInFPReg;
2228                                         else if (t->type == MONO_TYPE_R8)
2229                                                 linfo->args [i].storage = LLVMArgInFPReg;
2230                                 }
2231                         }
2232                         break;
2233                 case ArgValuetypeInReg:
2234                         if (sig->pinvoke) {
2235                                 cfg->exception_message = g_strdup ("pinvoke + vtypes");
2236                                 cfg->disable_llvm = TRUE;
2237                                 return linfo;
2238                         }
2239
2240                         linfo->args [i].storage = LLVMArgVtypeInReg;
2241                         for (j = 0; j < 2; ++j)
2242                                 linfo->args [i].pair_storage [j] = arg_storage_to_llvm_arg_storage (cfg, ainfo->pair_storage [j]);
2243                         break;
2244                 default:
2245                         cfg->exception_message = g_strdup ("ainfo->storage");
2246                         cfg->disable_llvm = TRUE;
2247                         break;
2248                 }
2249         }
2250
2251         return linfo;
2252 }
2253 #endif
2254
2255 void
2256 mono_arch_emit_call (MonoCompile *cfg, MonoCallInst *call)
2257 {
2258         MonoInst *arg, *in;
2259         MonoMethodSignature *sig;
2260         int i, n, stack_size;
2261         CallInfo *cinfo;
2262         ArgInfo *ainfo;
2263
2264         stack_size = 0;
2265
2266         sig = call->signature;
2267         n = sig->param_count + sig->hasthis;
2268
2269         cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig);
2270
2271         if (COMPILE_LLVM (cfg)) {
2272                 /* We shouldn't be called in the llvm case */
2273                 cfg->disable_llvm = TRUE;
2274                 return;
2275         }
2276
2277         if (cinfo->need_stack_align) {
2278                 if (!cfg->arch.no_pushes)
2279                         MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 8);
2280         }
2281
2282         /*
2283          * Emit all arguments which are passed on the stack to prevent register
2284          * allocation problems.
2285          */
2286         if (cfg->arch.no_pushes) {
2287                 for (i = 0; i < n; ++i) {
2288                         MonoType *t;
2289                         ainfo = cinfo->args + i;
2290
2291                         in = call->args [i];
2292
2293                         if (sig->hasthis && i == 0)
2294                                 t = &mono_defaults.object_class->byval_arg;
2295                         else
2296                                 t = sig->params [i - sig->hasthis];
2297
2298                         if (ainfo->storage == ArgOnStack && !MONO_TYPE_ISSTRUCT (t) && !call->tail_call) {
2299                                 if (!t->byref) {
2300                                         if (t->type == MONO_TYPE_R4)
2301                                                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER4_MEMBASE_REG, AMD64_RSP, ainfo->offset, in->dreg);
2302                                         else if (t->type == MONO_TYPE_R8)
2303                                                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER8_MEMBASE_REG, AMD64_RSP, ainfo->offset, in->dreg);
2304                                         else
2305                                                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, AMD64_RSP, ainfo->offset, in->dreg);
2306                                 } else {
2307                                         MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, AMD64_RSP, ainfo->offset, in->dreg);
2308                                 }
2309                                 if (cfg->compute_gc_maps) {
2310                                         MonoInst *def;
2311
2312                                         EMIT_NEW_GC_PARAM_SLOT_LIVENESS_DEF (cfg, def, ainfo->offset, t);
2313                                 }
2314                         }
2315                 }
2316         }
2317
2318         /*
2319          * Emit all parameters passed in registers in non-reverse order for better readability
2320          * and to help the optimization in emit_prolog ().
2321          */
2322         for (i = 0; i < n; ++i) {
2323                 ainfo = cinfo->args + i;
2324
2325                 in = call->args [i];
2326
2327                 if (ainfo->storage == ArgInIReg)
2328                         add_outarg_reg (cfg, call, ainfo->storage, ainfo->reg, in);
2329         }
2330
2331         for (i = n - 1; i >= 0; --i) {
2332                 ainfo = cinfo->args + i;
2333
2334                 in = call->args [i];
2335
2336                 switch (ainfo->storage) {
2337                 case ArgInIReg:
2338                         /* Already done */
2339                         break;
2340                 case ArgInFloatSSEReg:
2341                 case ArgInDoubleSSEReg:
2342                         add_outarg_reg (cfg, call, ainfo->storage, ainfo->reg, in);
2343                         break;
2344                 case ArgOnStack:
2345                 case ArgValuetypeInReg:
2346                 case ArgValuetypeAddrInIReg:
2347                         if (ainfo->storage == ArgOnStack && call->tail_call) {
2348                                 MonoInst *call_inst = (MonoInst*)call;
2349                                 cfg->args [i]->flags |= MONO_INST_VOLATILE;
2350                                 EMIT_NEW_ARGSTORE (cfg, call_inst, i, in);
2351                         } else if ((i >= sig->hasthis) && (MONO_TYPE_ISSTRUCT(sig->params [i - sig->hasthis]))) {
2352                                 guint32 align;
2353                                 guint32 size;
2354
2355                                 if (sig->params [i - sig->hasthis]->type == MONO_TYPE_TYPEDBYREF) {
2356                                         size = sizeof (MonoTypedRef);
2357                                         align = sizeof (gpointer);
2358                                 }
2359                                 else {
2360                                         if (sig->pinvoke)
2361                                                 size = mono_type_native_stack_size (&in->klass->byval_arg, &align);
2362                                         else {
2363                                                 /*
2364                                                  * Other backends use mono_type_stack_size (), but that
2365                                                  * aligns the size to 8, which is larger than the size of
2366                                                  * the source, leading to reads of invalid memory if the
2367                                                  * source is at the end of address space.
2368                                                  */
2369                                                 size = mono_class_value_size (in->klass, &align);
2370                                         }
2371                                 }
2372                                 g_assert (in->klass);
2373
2374                                 if (ainfo->storage == ArgOnStack && size >= 10000) {
2375                                         /* Avoid asserts in emit_memcpy () */
2376                                         cfg->exception_type = MONO_EXCEPTION_INVALID_PROGRAM;
2377                                         cfg->exception_message = g_strdup_printf ("Passing an argument of size '%d'.", size);
2378                                         /* Continue normally */
2379                                 }
2380
2381                                 if (size > 0) {
2382                                         MONO_INST_NEW (cfg, arg, OP_OUTARG_VT);
2383                                         arg->sreg1 = in->dreg;
2384                                         arg->klass = in->klass;
2385                                         arg->backend.size = size;
2386                                         arg->inst_p0 = call;
2387                                         arg->inst_p1 = mono_mempool_alloc (cfg->mempool, sizeof (ArgInfo));
2388                                         memcpy (arg->inst_p1, ainfo, sizeof (ArgInfo));
2389
2390                                         MONO_ADD_INS (cfg->cbb, arg);
2391                                 }
2392                         } else {
2393                                 if (cfg->arch.no_pushes) {
2394                                         /* Already done */
2395                                 } else {
2396                                         MONO_INST_NEW (cfg, arg, OP_X86_PUSH);
2397                                         arg->sreg1 = in->dreg;
2398                                         if (!sig->params [i - sig->hasthis]->byref) {
2399                                                 if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R4) {
2400                                                         MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 8);
2401                                                         arg->opcode = OP_STORER4_MEMBASE_REG;
2402                                                         arg->inst_destbasereg = X86_ESP;
2403                                                         arg->inst_offset = 0;
2404                                                 } else if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R8) {
2405                                                         MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 8);
2406                                                         arg->opcode = OP_STORER8_MEMBASE_REG;
2407                                                         arg->inst_destbasereg = X86_ESP;
2408                                                         arg->inst_offset = 0;
2409                                                 }
2410                                         }
2411                                         MONO_ADD_INS (cfg->cbb, arg);
2412                                 }
2413                         }
2414                         break;
2415                 default:
2416                         g_assert_not_reached ();
2417                 }
2418
2419                 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos))
2420                         /* Emit the signature cookie just before the implicit arguments */
2421                         emit_sig_cookie (cfg, call, cinfo);
2422         }
2423
2424         /* Handle the case where there are no implicit arguments */
2425         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n == sig->sentinelpos))
2426                 emit_sig_cookie (cfg, call, cinfo);
2427
2428         if (sig->ret && MONO_TYPE_ISSTRUCT (sig->ret)) {
2429                 MonoInst *vtarg;
2430
2431                 if (cinfo->ret.storage == ArgValuetypeInReg) {
2432                         if (cinfo->ret.pair_storage [0] == ArgInIReg && cinfo->ret.pair_storage [1] == ArgNone) {
2433                                 /*
2434                                  * Tell the JIT to use a more efficient calling convention: call using
2435                                  * OP_CALL, compute the result location after the call, and save the
2436                                  * result there.
2437                                  */
2438                                 call->vret_in_reg = TRUE;
2439                                 /*
2440                                  * Nullify the instruction computing the vret addr to enable
2441                                  * future optimizations.
2442                                  */
2443                                 if (call->vret_var)
2444                                         NULLIFY_INS (call->vret_var);
2445                         } else {
2446                                 if (call->tail_call)
2447                                         NOT_IMPLEMENTED;
2448                                 /*
2449                                  * The valuetype is in RAX:RDX after the call, need to be copied to
2450                                  * the stack. Push the address here, so the call instruction can
2451                                  * access it.
2452                                  */
2453                                 if (!cfg->arch.vret_addr_loc) {
2454                                         cfg->arch.vret_addr_loc = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
2455                                         /* Prevent it from being register allocated or optimized away */
2456                                         ((MonoInst*)cfg->arch.vret_addr_loc)->flags |= MONO_INST_VOLATILE;
2457                                 }
2458
2459                                 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, ((MonoInst*)cfg->arch.vret_addr_loc)->dreg, call->vret_var->dreg);
2460                         }
2461                 }
2462                 else {
2463                         MONO_INST_NEW (cfg, vtarg, OP_MOVE);
2464                         vtarg->sreg1 = call->vret_var->dreg;
2465                         vtarg->dreg = mono_alloc_preg (cfg);
2466                         MONO_ADD_INS (cfg->cbb, vtarg);
2467
2468                         mono_call_inst_add_outarg_reg (cfg, call, vtarg->dreg, cinfo->ret.reg, FALSE);
2469                 }
2470         }
2471
2472 #ifdef HOST_WIN32
2473         if (call->inst.opcode != OP_JMP && OP_TAILCALL != call->inst.opcode) {
2474                 MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 0x20);
2475         }
2476 #endif
2477
2478         if (cfg->method->save_lmf) {
2479                 MONO_INST_NEW (cfg, arg, OP_AMD64_SAVE_SP_TO_LMF);
2480                 MONO_ADD_INS (cfg->cbb, arg);
2481         }
2482
2483         call->stack_usage = cinfo->stack_usage;
2484 }
2485
2486 void
2487 mono_arch_emit_outarg_vt (MonoCompile *cfg, MonoInst *ins, MonoInst *src)
2488 {
2489         MonoInst *arg;
2490         MonoCallInst *call = (MonoCallInst*)ins->inst_p0;
2491         ArgInfo *ainfo = (ArgInfo*)ins->inst_p1;
2492         int size = ins->backend.size;
2493
2494         if (ainfo->storage == ArgValuetypeInReg) {
2495                 MonoInst *load;
2496                 int part;
2497
2498                 for (part = 0; part < 2; ++part) {
2499                         if (ainfo->pair_storage [part] == ArgNone)
2500                                 continue;
2501
2502                         MONO_INST_NEW (cfg, load, arg_storage_to_load_membase (ainfo->pair_storage [part]));
2503                         load->inst_basereg = src->dreg;
2504                         load->inst_offset = part * sizeof(mgreg_t);
2505
2506                         switch (ainfo->pair_storage [part]) {
2507                         case ArgInIReg:
2508                                 load->dreg = mono_alloc_ireg (cfg);
2509                                 break;
2510                         case ArgInDoubleSSEReg:
2511                         case ArgInFloatSSEReg:
2512                                 load->dreg = mono_alloc_freg (cfg);
2513                                 break;
2514                         default:
2515                                 g_assert_not_reached ();
2516                         }
2517                         MONO_ADD_INS (cfg->cbb, load);
2518
2519                         add_outarg_reg (cfg, call, ainfo->pair_storage [part], ainfo->pair_regs [part], load);
2520                 }
2521         } else if (ainfo->storage == ArgValuetypeAddrInIReg) {
2522                 MonoInst *vtaddr, *load;
2523                 vtaddr = mono_compile_create_var (cfg, &ins->klass->byval_arg, OP_LOCAL);
2524
2525                 g_assert (!cfg->arch.no_pushes);
2526
2527                 MONO_INST_NEW (cfg, load, OP_LDADDR);
2528                 load->inst_p0 = vtaddr;
2529                 vtaddr->flags |= MONO_INST_INDIRECT;
2530                 load->type = STACK_MP;
2531                 load->klass = vtaddr->klass;
2532                 load->dreg = mono_alloc_ireg (cfg);
2533                 MONO_ADD_INS (cfg->cbb, load);
2534                 mini_emit_memcpy (cfg, load->dreg, 0, src->dreg, 0, size, 4);
2535
2536                 if (ainfo->pair_storage [0] == ArgInIReg) {
2537                         MONO_INST_NEW (cfg, arg, OP_X86_LEA_MEMBASE);
2538                         arg->dreg = mono_alloc_ireg (cfg);
2539                         arg->sreg1 = load->dreg;
2540                         arg->inst_imm = 0;
2541                         MONO_ADD_INS (cfg->cbb, arg);
2542                         mono_call_inst_add_outarg_reg (cfg, call, arg->dreg, ainfo->pair_regs [0], FALSE);
2543                 } else {
2544                         MONO_INST_NEW (cfg, arg, OP_X86_PUSH);
2545                         arg->sreg1 = load->dreg;
2546                         MONO_ADD_INS (cfg->cbb, arg);
2547                 }
2548         } else {
2549                 if (size == 8) {
2550                         if (cfg->arch.no_pushes) {
2551                                 int dreg = mono_alloc_ireg (cfg);
2552
2553                                 MONO_EMIT_NEW_LOAD_MEMBASE (cfg, dreg, src->dreg, 0);
2554                                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, AMD64_RSP, ainfo->offset, dreg);
2555                         } else {
2556                                 /* Can't use this for < 8 since it does an 8 byte memory load */
2557                                 MONO_INST_NEW (cfg, arg, OP_X86_PUSH_MEMBASE);
2558                                 arg->inst_basereg = src->dreg;
2559                                 arg->inst_offset = 0;
2560                                 MONO_ADD_INS (cfg->cbb, arg);
2561                         }
2562                 } else if (size <= 40) {
2563                         if (cfg->arch.no_pushes) {
2564                                 mini_emit_memcpy (cfg, AMD64_RSP, ainfo->offset, src->dreg, 0, size, 4);
2565                         } else {
2566                                 MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, ALIGN_TO (size, 8));
2567                                 mini_emit_memcpy (cfg, X86_ESP, 0, src->dreg, 0, size, 4);
2568                         }
2569                 } else {
2570                         if (cfg->arch.no_pushes) {
2571                                 // FIXME: Code growth
2572                                 mini_emit_memcpy (cfg, AMD64_RSP, ainfo->offset, src->dreg, 0, size, 4);
2573                         } else {
2574                                 MONO_INST_NEW (cfg, arg, OP_X86_PUSH_OBJ);
2575                                 arg->inst_basereg = src->dreg;
2576                                 arg->inst_offset = 0;
2577                                 arg->inst_imm = size;
2578                                 MONO_ADD_INS (cfg->cbb, arg);
2579                         }
2580                 }
2581
2582                 if (cfg->compute_gc_maps) {
2583                         MonoInst *def;
2584                         EMIT_NEW_GC_PARAM_SLOT_LIVENESS_DEF (cfg, def, ainfo->offset, &ins->klass->byval_arg);
2585                 }
2586         }
2587 }
2588
2589 void
2590 mono_arch_emit_setret (MonoCompile *cfg, MonoMethod *method, MonoInst *val)
2591 {
2592         MonoType *ret = mini_type_get_underlying_type (NULL, mono_method_signature (method)->ret);
2593
2594         if (ret->type == MONO_TYPE_R4) {
2595                 if (COMPILE_LLVM (cfg))
2596                         MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2597                 else
2598                         MONO_EMIT_NEW_UNALU (cfg, OP_AMD64_SET_XMMREG_R4, cfg->ret->dreg, val->dreg);
2599                 return;
2600         } else if (ret->type == MONO_TYPE_R8) {
2601                 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2602                 return;
2603         }
2604
2605         MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, cfg->ret->dreg, val->dreg);
2606 }
2607
2608 #endif /* DISABLE_JIT */
2609
2610 #define EMIT_COND_BRANCH(ins,cond,sign) \
2611         if (ins->inst_true_bb->native_offset) { \
2612                 x86_branch (code, cond, cfg->native_code + ins->inst_true_bb->native_offset, sign); \
2613         } else { \
2614                 mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_BB, ins->inst_true_bb); \
2615                 if ((cfg->opt & MONO_OPT_BRANCH) && \
2616             x86_is_imm8 (ins->inst_true_bb->max_offset - offset)) \
2617                         x86_branch8 (code, cond, 0, sign); \
2618                 else \
2619                         x86_branch32 (code, cond, 0, sign); \
2620 }
2621
2622 typedef struct {
2623         MonoMethodSignature *sig;
2624         CallInfo *cinfo;
2625 } ArchDynCallInfo;
2626
2627 typedef struct {
2628         mgreg_t regs [PARAM_REGS];
2629         mgreg_t res;
2630         guint8 *ret;
2631 } DynCallArgs;
2632
2633 static gboolean
2634 dyn_call_supported (MonoMethodSignature *sig, CallInfo *cinfo)
2635 {
2636         int i;
2637
2638 #ifdef HOST_WIN32
2639         return FALSE;
2640 #endif
2641
2642         switch (cinfo->ret.storage) {
2643         case ArgNone:
2644         case ArgInIReg:
2645                 break;
2646         case ArgValuetypeInReg: {
2647                 ArgInfo *ainfo = &cinfo->ret;
2648
2649                 if (ainfo->pair_storage [0] != ArgNone && ainfo->pair_storage [0] != ArgInIReg)
2650                         return FALSE;
2651                 if (ainfo->pair_storage [1] != ArgNone && ainfo->pair_storage [1] != ArgInIReg)
2652                         return FALSE;
2653                 break;
2654         }
2655         default:
2656                 return FALSE;
2657         }
2658
2659         for (i = 0; i < cinfo->nargs; ++i) {
2660                 ArgInfo *ainfo = &cinfo->args [i];
2661                 switch (ainfo->storage) {
2662                 case ArgInIReg:
2663                         break;
2664                 case ArgValuetypeInReg:
2665                         if (ainfo->pair_storage [0] != ArgNone && ainfo->pair_storage [0] != ArgInIReg)
2666                                 return FALSE;
2667                         if (ainfo->pair_storage [1] != ArgNone && ainfo->pair_storage [1] != ArgInIReg)
2668                                 return FALSE;
2669                         break;
2670                 default:
2671                         return FALSE;
2672                 }
2673         }
2674
2675         return TRUE;
2676 }
2677
2678 /*
2679  * mono_arch_dyn_call_prepare:
2680  *
2681  *   Return a pointer to an arch-specific structure which contains information
2682  * needed by mono_arch_get_dyn_call_args (). Return NULL if OP_DYN_CALL is not
2683  * supported for SIG.
2684  * This function is equivalent to ffi_prep_cif in libffi.
2685  */
2686 MonoDynCallInfo*
2687 mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
2688 {
2689         ArchDynCallInfo *info;
2690         CallInfo *cinfo;
2691
2692         cinfo = get_call_info (NULL, NULL, sig);
2693
2694         if (!dyn_call_supported (sig, cinfo)) {
2695                 g_free (cinfo);
2696                 return NULL;
2697         }
2698
2699         info = g_new0 (ArchDynCallInfo, 1);
2700         // FIXME: Preprocess the info to speed up get_dyn_call_args ().
2701         info->sig = sig;
2702         info->cinfo = cinfo;
2703
2704         return (MonoDynCallInfo*)info;
2705 }
2706
2707 /*
2708  * mono_arch_dyn_call_free:
2709  *
2710  *   Free a MonoDynCallInfo structure.
2711  */
2712 void
2713 mono_arch_dyn_call_free (MonoDynCallInfo *info)
2714 {
2715         ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
2716
2717         g_free (ainfo->cinfo);
2718         g_free (ainfo);
2719 }
2720
2721 #if !defined(__native_client__)
2722 #define PTR_TO_GREG(ptr) (mgreg_t)(ptr)
2723 #define GREG_TO_PTR(greg) (gpointer)(greg)
2724 #else
2725 /* Correctly handle casts to/from 32-bit pointers without compiler warnings */
2726 #define PTR_TO_GREG(ptr) (mgreg_t)(uintptr_t)(ptr)
2727 #define GREG_TO_PTR(greg) (gpointer)(guint32)(greg)
2728 #endif
2729
2730 /*
2731  * mono_arch_get_start_dyn_call:
2732  *
2733  *   Convert the arguments ARGS to a format which can be passed to OP_DYN_CALL, and
2734  * store the result into BUF.
2735  * ARGS should be an array of pointers pointing to the arguments.
2736  * RET should point to a memory buffer large enought to hold the result of the
2737  * call.
2738  * This function should be as fast as possible, any work which does not depend
2739  * on the actual values of the arguments should be done in
2740  * mono_arch_dyn_call_prepare ().
2741  * start_dyn_call + OP_DYN_CALL + finish_dyn_call is equivalent to ffi_call in
2742  * libffi.
2743  */
2744 void
2745 mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, guint8 *buf, int buf_len)
2746 {
2747         ArchDynCallInfo *dinfo = (ArchDynCallInfo*)info;
2748         DynCallArgs *p = (DynCallArgs*)buf;
2749         int arg_index, greg, i, pindex;
2750         MonoMethodSignature *sig = dinfo->sig;
2751
2752         g_assert (buf_len >= sizeof (DynCallArgs));
2753
2754         p->res = 0;
2755         p->ret = ret;
2756
2757         arg_index = 0;
2758         greg = 0;
2759         pindex = 0;
2760
2761         if (sig->hasthis || dinfo->cinfo->vret_arg_index == 1) {
2762                 p->regs [greg ++] = PTR_TO_GREG(*(args [arg_index ++]));
2763                 if (!sig->hasthis)
2764                         pindex = 1;
2765         }
2766
2767         if (dinfo->cinfo->vtype_retaddr)
2768                 p->regs [greg ++] = PTR_TO_GREG(ret);
2769
2770         for (i = pindex; i < sig->param_count; i++) {
2771                 MonoType *t = mono_type_get_underlying_type (sig->params [i]);
2772                 gpointer *arg = args [arg_index ++];
2773
2774                 if (t->byref) {
2775                         p->regs [greg ++] = PTR_TO_GREG(*(arg));
2776                         continue;
2777                 }
2778
2779                 switch (t->type) {
2780                 case MONO_TYPE_STRING:
2781                 case MONO_TYPE_CLASS:
2782                 case MONO_TYPE_ARRAY:
2783                 case MONO_TYPE_SZARRAY:
2784                 case MONO_TYPE_OBJECT:
2785                 case MONO_TYPE_PTR:
2786                 case MONO_TYPE_I:
2787                 case MONO_TYPE_U:
2788 #if !defined(__mono_ilp32__)
2789                 case MONO_TYPE_I8:
2790                 case MONO_TYPE_U8:
2791 #endif
2792                         g_assert (dinfo->cinfo->args [i + sig->hasthis].reg == param_regs [greg]);
2793                         p->regs [greg ++] = PTR_TO_GREG(*(arg));
2794                         break;
2795 #if defined(__mono_ilp32__)
2796                 case MONO_TYPE_I8:
2797                 case MONO_TYPE_U8:
2798                         g_assert (dinfo->cinfo->args [i + sig->hasthis].reg == param_regs [greg]);
2799                         p->regs [greg ++] = *(guint64*)(arg);
2800                         break;
2801 #endif
2802                 case MONO_TYPE_BOOLEAN:
2803                 case MONO_TYPE_U1:
2804                         p->regs [greg ++] = *(guint8*)(arg);
2805                         break;
2806                 case MONO_TYPE_I1:
2807                         p->regs [greg ++] = *(gint8*)(arg);
2808                         break;
2809                 case MONO_TYPE_I2:
2810                         p->regs [greg ++] = *(gint16*)(arg);
2811                         break;
2812                 case MONO_TYPE_U2:
2813                 case MONO_TYPE_CHAR:
2814                         p->regs [greg ++] = *(guint16*)(arg);
2815                         break;
2816                 case MONO_TYPE_I4:
2817                         p->regs [greg ++] = *(gint32*)(arg);
2818                         break;
2819                 case MONO_TYPE_U4:
2820                         p->regs [greg ++] = *(guint32*)(arg);
2821                         break;
2822                 case MONO_TYPE_GENERICINST:
2823                     if (MONO_TYPE_IS_REFERENCE (t)) {
2824                                 p->regs [greg ++] = PTR_TO_GREG(*(arg));
2825                                 break;
2826                         } else {
2827                                 /* Fall through */
2828                         }
2829                 case MONO_TYPE_VALUETYPE: {
2830                         ArgInfo *ainfo = &dinfo->cinfo->args [i + sig->hasthis];
2831
2832                         g_assert (ainfo->storage == ArgValuetypeInReg);
2833                         if (ainfo->pair_storage [0] != ArgNone) {
2834                                 g_assert (ainfo->pair_storage [0] == ArgInIReg);
2835                                 p->regs [greg ++] = ((mgreg_t*)(arg))[0];
2836                         }
2837                         if (ainfo->pair_storage [1] != ArgNone) {
2838                                 g_assert (ainfo->pair_storage [1] == ArgInIReg);
2839                                 p->regs [greg ++] = ((mgreg_t*)(arg))[1];
2840                         }
2841                         break;
2842                 }
2843                 default:
2844                         g_assert_not_reached ();
2845                 }
2846         }
2847
2848         g_assert (greg <= PARAM_REGS);
2849 }
2850
2851 /*
2852  * mono_arch_finish_dyn_call:
2853  *
2854  *   Store the result of a dyn call into the return value buffer passed to
2855  * start_dyn_call ().
2856  * This function should be as fast as possible, any work which does not depend
2857  * on the actual values of the arguments should be done in
2858  * mono_arch_dyn_call_prepare ().
2859  */
2860 void
2861 mono_arch_finish_dyn_call (MonoDynCallInfo *info, guint8 *buf)
2862 {
2863         ArchDynCallInfo *dinfo = (ArchDynCallInfo*)info;
2864         MonoMethodSignature *sig = dinfo->sig;
2865         guint8 *ret = ((DynCallArgs*)buf)->ret;
2866         mgreg_t res = ((DynCallArgs*)buf)->res;
2867
2868         switch (mono_type_get_underlying_type (sig->ret)->type) {
2869         case MONO_TYPE_VOID:
2870                 *(gpointer*)ret = NULL;
2871                 break;
2872         case MONO_TYPE_STRING:
2873         case MONO_TYPE_CLASS:
2874         case MONO_TYPE_ARRAY:
2875         case MONO_TYPE_SZARRAY:
2876         case MONO_TYPE_OBJECT:
2877         case MONO_TYPE_I:
2878         case MONO_TYPE_U:
2879         case MONO_TYPE_PTR:
2880                 *(gpointer*)ret = GREG_TO_PTR(res);
2881                 break;
2882         case MONO_TYPE_I1:
2883                 *(gint8*)ret = res;
2884                 break;
2885         case MONO_TYPE_U1:
2886         case MONO_TYPE_BOOLEAN:
2887                 *(guint8*)ret = res;
2888                 break;
2889         case MONO_TYPE_I2:
2890                 *(gint16*)ret = res;
2891                 break;
2892         case MONO_TYPE_U2:
2893         case MONO_TYPE_CHAR:
2894                 *(guint16*)ret = res;
2895                 break;
2896         case MONO_TYPE_I4:
2897                 *(gint32*)ret = res;
2898                 break;
2899         case MONO_TYPE_U4:
2900                 *(guint32*)ret = res;
2901                 break;
2902         case MONO_TYPE_I8:
2903                 *(gint64*)ret = res;
2904                 break;
2905         case MONO_TYPE_U8:
2906                 *(guint64*)ret = res;
2907                 break;
2908         case MONO_TYPE_GENERICINST:
2909                 if (MONO_TYPE_IS_REFERENCE (sig->ret)) {
2910                         *(gpointer*)ret = GREG_TO_PTR(res);
2911                         break;
2912                 } else {
2913                         /* Fall through */
2914                 }
2915         case MONO_TYPE_VALUETYPE:
2916                 if (dinfo->cinfo->vtype_retaddr) {
2917                         /* Nothing to do */
2918                 } else {
2919                         ArgInfo *ainfo = &dinfo->cinfo->ret;
2920
2921                         g_assert (ainfo->storage == ArgValuetypeInReg);
2922
2923                         if (ainfo->pair_storage [0] != ArgNone) {
2924                                 g_assert (ainfo->pair_storage [0] == ArgInIReg);
2925                                 ((mgreg_t*)ret)[0] = res;
2926                         }
2927
2928                         g_assert (ainfo->pair_storage [1] == ArgNone);
2929                 }
2930                 break;
2931         default:
2932                 g_assert_not_reached ();
2933         }
2934 }
2935
2936 /* emit an exception if condition is fail */
2937 #define EMIT_COND_SYSTEM_EXCEPTION(cond,signed,exc_name)            \
2938         do {                                                        \
2939                 MonoInst *tins = mono_branch_optimize_exception_target (cfg, bb, exc_name); \
2940                 if (tins == NULL) {                                                                             \
2941                         mono_add_patch_info (cfg, code - cfg->native_code,   \
2942                                         MONO_PATCH_INFO_EXC, exc_name);  \
2943                         x86_branch32 (code, cond, 0, signed);               \
2944                 } else {        \
2945                         EMIT_COND_BRANCH (tins, cond, signed);  \
2946                 }                       \
2947         } while (0);
2948
2949 #define EMIT_FPCOMPARE(code) do { \
2950         amd64_fcompp (code); \
2951         amd64_fnstsw (code); \
2952 } while (0);
2953
2954 #define EMIT_SSE2_FPFUNC(code, op, dreg, sreg1) do { \
2955     amd64_movsd_membase_reg (code, AMD64_RSP, -8, (sreg1)); \
2956         amd64_fld_membase (code, AMD64_RSP, -8, TRUE); \
2957         amd64_ ##op (code); \
2958         amd64_fst_membase (code, AMD64_RSP, -8, TRUE, TRUE); \
2959         amd64_movsd_reg_membase (code, (dreg), AMD64_RSP, -8); \
2960 } while (0);
2961
2962 static guint8*
2963 emit_call_body (MonoCompile *cfg, guint8 *code, guint32 patch_type, gconstpointer data)
2964 {
2965         gboolean no_patch = FALSE;
2966
2967         /*
2968          * FIXME: Add support for thunks
2969          */
2970         {
2971                 gboolean near_call = FALSE;
2972
2973                 /*
2974                  * Indirect calls are expensive so try to make a near call if possible.
2975                  * The caller memory is allocated by the code manager so it is
2976                  * guaranteed to be at a 32 bit offset.
2977                  */
2978
2979                 if (patch_type != MONO_PATCH_INFO_ABS) {
2980                         /* The target is in memory allocated using the code manager */
2981                         near_call = TRUE;
2982
2983                         if ((patch_type == MONO_PATCH_INFO_METHOD) || (patch_type == MONO_PATCH_INFO_METHOD_JUMP)) {
2984                                 if (((MonoMethod*)data)->klass->image->aot_module)
2985                                         /* The callee might be an AOT method */
2986                                         near_call = FALSE;
2987                                 if (((MonoMethod*)data)->dynamic)
2988                                         /* The target is in malloc-ed memory */
2989                                         near_call = FALSE;
2990                         }
2991
2992                         if (patch_type == MONO_PATCH_INFO_INTERNAL_METHOD) {
2993                                 /*
2994                                  * The call might go directly to a native function without
2995                                  * the wrapper.
2996                                  */
2997                                 MonoJitICallInfo *mi = mono_find_jit_icall_by_name (data);
2998                                 if (mi) {
2999                                         gconstpointer target = mono_icall_get_wrapper (mi);
3000                                         if ((((guint64)target) >> 32) != 0)
3001                                                 near_call = FALSE;
3002                                 }
3003                         }
3004                 }
3005                 else {
3006                         MonoJumpInfo *jinfo = NULL;
3007
3008                         if (cfg->abs_patches)
3009                                 jinfo = g_hash_table_lookup (cfg->abs_patches, data);
3010                         if (jinfo) {
3011                                 if (jinfo->type == MONO_PATCH_INFO_JIT_ICALL_ADDR) {
3012                                         if ((((guint64)data) >> 32) == 0)
3013                                                 near_call = TRUE;
3014                                         no_patch = TRUE;
3015                                 } else {
3016                                         /*
3017                                          * This is not really an optimization, but required because the
3018                                          * generic class init trampolines use R11 to pass the vtable.
3019                                          */
3020                                         near_call = TRUE;
3021                                 }
3022                         } else {
3023                                 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (data);
3024                                 if (info) {
3025                                         if (info->func == info->wrapper) {
3026                                                 /* No wrapper */
3027                                                 if ((((guint64)info->func) >> 32) == 0)
3028                                                         near_call = TRUE;
3029                                         }
3030                                         else {
3031                                                 /* See the comment in mono_codegen () */
3032                                                 if ((info->name [0] != 'v') || (strstr (info->name, "ves_array_new_va_") == NULL && strstr (info->name, "ves_array_element_address_") == NULL))
3033                                                         near_call = TRUE;
3034                                         }
3035                                 }
3036                                 else if ((((guint64)data) >> 32) == 0) {
3037                                         near_call = TRUE;
3038                                         no_patch = TRUE;
3039                                 }
3040                         }
3041                 }
3042
3043                 if (cfg->method->dynamic)
3044                         /* These methods are allocated using malloc */
3045                         near_call = FALSE;
3046
3047 #ifdef MONO_ARCH_NOMAP32BIT
3048                 near_call = FALSE;
3049 #endif
3050 #if defined(__native_client__)
3051                 /* Always use near_call == TRUE for Native Client */
3052                 near_call = TRUE;
3053 #endif
3054                 /* The 64bit XEN kernel does not honour the MAP_32BIT flag. (#522894) */
3055                 if (optimize_for_xen)
3056                         near_call = FALSE;
3057
3058                 if (cfg->compile_aot) {
3059                         near_call = TRUE;
3060                         no_patch = TRUE;
3061                 }
3062
3063                 if (near_call) {
3064                         /*
3065                          * Align the call displacement to an address divisible by 4 so it does
3066                          * not span cache lines. This is required for code patching to work on SMP
3067                          * systems.
3068                          */
3069                         if (!no_patch && ((guint32)(code + 1 - cfg->native_code) % 4) != 0) {
3070                                 guint32 pad_size = 4 - ((guint32)(code + 1 - cfg->native_code) % 4);
3071                                 amd64_padding (code, pad_size);
3072                         }
3073                         mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
3074                         amd64_call_code (code, 0);
3075                 }
3076                 else {
3077                         mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
3078                         amd64_set_reg_template (code, GP_SCRATCH_REG);
3079                         amd64_call_reg (code, GP_SCRATCH_REG);
3080                 }
3081         }
3082
3083         return code;
3084 }
3085
3086 static inline guint8*
3087 emit_call (MonoCompile *cfg, guint8 *code, guint32 patch_type, gconstpointer data, gboolean win64_adjust_stack)
3088 {
3089 #ifdef HOST_WIN32
3090         if (win64_adjust_stack)
3091                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 32);
3092 #endif
3093         code = emit_call_body (cfg, code, patch_type, data);
3094 #ifdef HOST_WIN32
3095         if (win64_adjust_stack)
3096                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 32);
3097 #endif
3098
3099         return code;
3100 }
3101
3102 static inline int
3103 store_membase_imm_to_store_membase_reg (int opcode)
3104 {
3105         switch (opcode) {
3106         case OP_STORE_MEMBASE_IMM:
3107                 return OP_STORE_MEMBASE_REG;
3108         case OP_STOREI4_MEMBASE_IMM:
3109                 return OP_STOREI4_MEMBASE_REG;
3110         case OP_STOREI8_MEMBASE_IMM:
3111                 return OP_STOREI8_MEMBASE_REG;
3112         }
3113
3114         return -1;
3115 }
3116
3117 #ifndef DISABLE_JIT
3118
3119 #define INST_IGNORES_CFLAGS(opcode) (!(((opcode) == OP_ADC) || ((opcode) == OP_ADC_IMM) || ((opcode) == OP_IADC) || ((opcode) == OP_IADC_IMM) || ((opcode) == OP_SBB) || ((opcode) == OP_SBB_IMM) || ((opcode) == OP_ISBB) || ((opcode) == OP_ISBB_IMM)))
3120
3121 /*
3122  * mono_arch_peephole_pass_1:
3123  *
3124  *   Perform peephole opts which should/can be performed before local regalloc
3125  */
3126 void
3127 mono_arch_peephole_pass_1 (MonoCompile *cfg, MonoBasicBlock *bb)
3128 {
3129         MonoInst *ins, *n;
3130
3131         MONO_BB_FOR_EACH_INS_SAFE (bb, n, ins) {
3132                 MonoInst *last_ins = ins->prev;
3133
3134                 switch (ins->opcode) {
3135                 case OP_ADD_IMM:
3136                 case OP_IADD_IMM:
3137                 case OP_LADD_IMM:
3138                         if ((ins->sreg1 < MONO_MAX_IREGS) && (ins->dreg >= MONO_MAX_IREGS) && (ins->inst_imm > 0)) {
3139                                 /*
3140                                  * X86_LEA is like ADD, but doesn't have the
3141                                  * sreg1==dreg restriction. inst_imm > 0 is needed since LEA sign-extends
3142                                  * its operand to 64 bit.
3143                                  */
3144                                 ins->opcode = OP_X86_LEA_MEMBASE;
3145                                 ins->inst_basereg = ins->sreg1;
3146                         }
3147                         break;
3148                 case OP_LXOR:
3149                 case OP_IXOR:
3150                         if ((ins->sreg1 == ins->sreg2) && (ins->sreg1 == ins->dreg)) {
3151                                 MonoInst *ins2;
3152
3153                                 /*
3154                                  * Replace STORE_MEMBASE_IMM 0 with STORE_MEMBASE_REG since
3155                                  * the latter has length 2-3 instead of 6 (reverse constant
3156                                  * propagation). These instruction sequences are very common
3157                                  * in the initlocals bblock.
3158                                  */
3159                                 for (ins2 = ins->next; ins2; ins2 = ins2->next) {
3160                                         if (((ins2->opcode == OP_STORE_MEMBASE_IMM) || (ins2->opcode == OP_STOREI4_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_IMM) || (ins2->opcode == OP_STORE_MEMBASE_IMM)) && (ins2->inst_imm == 0)) {
3161                                                 ins2->opcode = store_membase_imm_to_store_membase_reg (ins2->opcode);
3162                                                 ins2->sreg1 = ins->dreg;
3163                                         } else if ((ins2->opcode == OP_STOREI1_MEMBASE_IMM) || (ins2->opcode == OP_STOREI2_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_REG) || (ins2->opcode == OP_STORE_MEMBASE_REG)) {
3164                                                 /* Continue */
3165                                         } else if (((ins2->opcode == OP_ICONST) || (ins2->opcode == OP_I8CONST)) && (ins2->dreg == ins->dreg) && (ins2->inst_c0 == 0)) {
3166                                                 NULLIFY_INS (ins2);
3167                                                 /* Continue */
3168                                         } else {
3169                                                 break;
3170                                         }
3171                                 }
3172                         }
3173                         break;
3174                 case OP_COMPARE_IMM:
3175                 case OP_LCOMPARE_IMM:
3176                         /* OP_COMPARE_IMM (reg, 0)
3177                          * -->
3178                          * OP_AMD64_TEST_NULL (reg)
3179                          */
3180                         if (!ins->inst_imm)
3181                                 ins->opcode = OP_AMD64_TEST_NULL;
3182                         break;
3183                 case OP_ICOMPARE_IMM:
3184                         if (!ins->inst_imm)
3185                                 ins->opcode = OP_X86_TEST_NULL;
3186                         break;
3187                 case OP_AMD64_ICOMPARE_MEMBASE_IMM:
3188                         /*
3189                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
3190                          * OP_X86_COMPARE_MEMBASE_IMM offset(basereg), imm
3191                          * -->
3192                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
3193                          * OP_COMPARE_IMM reg, imm
3194                          *
3195                          * Note: if imm = 0 then OP_COMPARE_IMM replaced with OP_X86_TEST_NULL
3196                          */
3197                         if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_REG) &&
3198                             ins->inst_basereg == last_ins->inst_destbasereg &&
3199                             ins->inst_offset == last_ins->inst_offset) {
3200                                         ins->opcode = OP_ICOMPARE_IMM;
3201                                         ins->sreg1 = last_ins->sreg1;
3202
3203                                         /* check if we can remove cmp reg,0 with test null */
3204                                         if (!ins->inst_imm)
3205                                                 ins->opcode = OP_X86_TEST_NULL;
3206                                 }
3207
3208                         break;
3209                 }
3210
3211                 mono_peephole_ins (bb, ins);
3212         }
3213 }
3214
3215 void
3216 mono_arch_peephole_pass_2 (MonoCompile *cfg, MonoBasicBlock *bb)
3217 {
3218         MonoInst *ins, *n;
3219
3220         MONO_BB_FOR_EACH_INS_SAFE (bb, n, ins) {
3221                 switch (ins->opcode) {
3222                 case OP_ICONST:
3223                 case OP_I8CONST: {
3224                         /* reg = 0 -> XOR (reg, reg) */
3225                         /* XOR sets cflags on x86, so we cant do it always */
3226                         if (ins->inst_c0 == 0 && (!ins->next || (ins->next && INST_IGNORES_CFLAGS (ins->next->opcode)))) {
3227                                 ins->opcode = OP_LXOR;
3228                                 ins->sreg1 = ins->dreg;
3229                                 ins->sreg2 = ins->dreg;
3230                                 /* Fall through */
3231                         } else {
3232                                 break;
3233                         }
3234                 }
3235                 case OP_LXOR:
3236                         /*
3237                          * Use IXOR to avoid a rex prefix if possible. The cpu will sign extend the
3238                          * 0 result into 64 bits.
3239                          */
3240                         if ((ins->sreg1 == ins->sreg2) && (ins->sreg1 == ins->dreg)) {
3241                                 ins->opcode = OP_IXOR;
3242                         }
3243                         /* Fall through */
3244                 case OP_IXOR:
3245                         if ((ins->sreg1 == ins->sreg2) && (ins->sreg1 == ins->dreg)) {
3246                                 MonoInst *ins2;
3247
3248                                 /*
3249                                  * Replace STORE_MEMBASE_IMM 0 with STORE_MEMBASE_REG since
3250                                  * the latter has length 2-3 instead of 6 (reverse constant
3251                                  * propagation). These instruction sequences are very common
3252                                  * in the initlocals bblock.
3253                                  */
3254                                 for (ins2 = ins->next; ins2; ins2 = ins2->next) {
3255                                         if (((ins2->opcode == OP_STORE_MEMBASE_IMM) || (ins2->opcode == OP_STOREI4_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_IMM) || (ins2->opcode == OP_STORE_MEMBASE_IMM)) && (ins2->inst_imm == 0)) {
3256                                                 ins2->opcode = store_membase_imm_to_store_membase_reg (ins2->opcode);
3257                                                 ins2->sreg1 = ins->dreg;
3258                                         } else if ((ins2->opcode == OP_STOREI1_MEMBASE_IMM) || (ins2->opcode == OP_STOREI2_MEMBASE_IMM) || (ins2->opcode == OP_STOREI4_MEMBASE_REG) || (ins2->opcode == OP_STOREI8_MEMBASE_REG) || (ins2->opcode == OP_STORE_MEMBASE_REG) || (ins2->opcode == OP_LIVERANGE_START) || (ins2->opcode == OP_GC_LIVENESS_DEF) || (ins2->opcode == OP_GC_LIVENESS_USE)) {
3259                                                 /* Continue */
3260                                         } else if (((ins2->opcode == OP_ICONST) || (ins2->opcode == OP_I8CONST)) && (ins2->dreg == ins->dreg) && (ins2->inst_c0 == 0)) {
3261                                                 NULLIFY_INS (ins2);
3262                                                 /* Continue */
3263                                         } else {
3264                                                 break;
3265                                         }
3266                                 }
3267                         }
3268                         break;
3269                 case OP_IADD_IMM:
3270                         if ((ins->inst_imm == 1) && (ins->dreg == ins->sreg1))
3271                                 ins->opcode = OP_X86_INC_REG;
3272                         break;
3273                 case OP_ISUB_IMM:
3274                         if ((ins->inst_imm == 1) && (ins->dreg == ins->sreg1))
3275                                 ins->opcode = OP_X86_DEC_REG;
3276                         break;
3277                 }
3278
3279                 mono_peephole_ins (bb, ins);
3280         }
3281 }
3282
3283 #define NEW_INS(cfg,ins,dest,op) do {   \
3284                 MONO_INST_NEW ((cfg), (dest), (op)); \
3285         (dest)->cil_code = (ins)->cil_code; \
3286         mono_bblock_insert_before_ins (bb, ins, (dest)); \
3287         } while (0)
3288
3289 /*
3290  * mono_arch_lowering_pass:
3291  *
3292  *  Converts complex opcodes into simpler ones so that each IR instruction
3293  * corresponds to one machine instruction.
3294  */
3295 void
3296 mono_arch_lowering_pass (MonoCompile *cfg, MonoBasicBlock *bb)
3297 {
3298         MonoInst *ins, *n, *temp;
3299
3300         /*
3301          * FIXME: Need to add more instructions, but the current machine
3302          * description can't model some parts of the composite instructions like
3303          * cdq.
3304          */
3305         MONO_BB_FOR_EACH_INS_SAFE (bb, n, ins) {
3306                 switch (ins->opcode) {
3307                 case OP_DIV_IMM:
3308                 case OP_REM_IMM:
3309                 case OP_IDIV_IMM:
3310                 case OP_IDIV_UN_IMM:
3311                 case OP_IREM_UN_IMM:
3312                         mono_decompose_op_imm (cfg, bb, ins);
3313                         break;
3314                 case OP_IREM_IMM:
3315                         /* Keep the opcode if we can implement it efficiently */
3316                         if (!((ins->inst_imm > 0) && (mono_is_power_of_two (ins->inst_imm) != -1)))
3317                                 mono_decompose_op_imm (cfg, bb, ins);
3318                         break;
3319                 case OP_COMPARE_IMM:
3320                 case OP_LCOMPARE_IMM:
3321                         if (!amd64_is_imm32 (ins->inst_imm)) {
3322                                 NEW_INS (cfg, ins, temp, OP_I8CONST);
3323                                 temp->inst_c0 = ins->inst_imm;
3324                                 temp->dreg = mono_alloc_ireg (cfg);
3325                                 ins->opcode = OP_COMPARE;
3326                                 ins->sreg2 = temp->dreg;
3327                         }
3328                         break;
3329 #ifndef __mono_ilp32__
3330                 case OP_LOAD_MEMBASE:
3331 #endif
3332                 case OP_LOADI8_MEMBASE:
3333 #ifndef __native_client_codegen__
3334                 /*  Don't generate memindex opcodes (to simplify */
3335                 /*  read sandboxing) */
3336                         if (!amd64_is_imm32 (ins->inst_offset)) {
3337                                 NEW_INS (cfg, ins, temp, OP_I8CONST);
3338                                 temp->inst_c0 = ins->inst_offset;
3339                                 temp->dreg = mono_alloc_ireg (cfg);
3340                                 ins->opcode = OP_AMD64_LOADI8_MEMINDEX;
3341                                 ins->inst_indexreg = temp->dreg;
3342                         }
3343 #endif
3344                         break;
3345 #ifndef __mono_ilp32__
3346                 case OP_STORE_MEMBASE_IMM:
3347 #endif
3348                 case OP_STOREI8_MEMBASE_IMM:
3349                         if (!amd64_is_imm32 (ins->inst_imm)) {
3350                                 NEW_INS (cfg, ins, temp, OP_I8CONST);
3351                                 temp->inst_c0 = ins->inst_imm;
3352                                 temp->dreg = mono_alloc_ireg (cfg);
3353                                 ins->opcode = OP_STOREI8_MEMBASE_REG;
3354                                 ins->sreg1 = temp->dreg;
3355                         }
3356                         break;
3357 #ifdef MONO_ARCH_SIMD_INTRINSICS
3358                 case OP_EXPAND_I1: {
3359                                 int temp_reg1 = mono_alloc_ireg (cfg);
3360                                 int temp_reg2 = mono_alloc_ireg (cfg);
3361                                 int original_reg = ins->sreg1;
3362
3363                                 NEW_INS (cfg, ins, temp, OP_ICONV_TO_U1);
3364                                 temp->sreg1 = original_reg;
3365                                 temp->dreg = temp_reg1;
3366
3367                                 NEW_INS (cfg, ins, temp, OP_SHL_IMM);
3368                                 temp->sreg1 = temp_reg1;
3369                                 temp->dreg = temp_reg2;
3370                                 temp->inst_imm = 8;
3371
3372                                 NEW_INS (cfg, ins, temp, OP_LOR);
3373                                 temp->sreg1 = temp->dreg = temp_reg2;
3374                                 temp->sreg2 = temp_reg1;
3375
3376                                 ins->opcode = OP_EXPAND_I2;
3377                                 ins->sreg1 = temp_reg2;
3378                         }
3379                         break;
3380 #endif
3381                 default:
3382                         break;
3383                 }
3384         }
3385
3386         bb->max_vreg = cfg->next_vreg;
3387 }
3388
3389 static const int
3390 branch_cc_table [] = {
3391         X86_CC_EQ, X86_CC_GE, X86_CC_GT, X86_CC_LE, X86_CC_LT,
3392         X86_CC_NE, X86_CC_GE, X86_CC_GT, X86_CC_LE, X86_CC_LT,
3393         X86_CC_O, X86_CC_NO, X86_CC_C, X86_CC_NC
3394 };
3395
3396 /* Maps CMP_... constants to X86_CC_... constants */
3397 static const int
3398 cc_table [] = {
3399         X86_CC_EQ, X86_CC_NE, X86_CC_LE, X86_CC_GE, X86_CC_LT, X86_CC_GT,
3400         X86_CC_LE, X86_CC_GE, X86_CC_LT, X86_CC_GT
3401 };
3402
3403 static const int
3404 cc_signed_table [] = {
3405         TRUE, TRUE, TRUE, TRUE, TRUE, TRUE,
3406         FALSE, FALSE, FALSE, FALSE
3407 };
3408
3409 /*#include "cprop.c"*/
3410
3411 static unsigned char*
3412 emit_float_to_int (MonoCompile *cfg, guchar *code, int dreg, int sreg, int size, gboolean is_signed)
3413 {
3414         amd64_sse_cvttsd2si_reg_reg (code, dreg, sreg);
3415
3416         if (size == 1)
3417                 amd64_widen_reg (code, dreg, dreg, is_signed, FALSE);
3418         else if (size == 2)
3419                 amd64_widen_reg (code, dreg, dreg, is_signed, TRUE);
3420         return code;
3421 }
3422
3423 static unsigned char*
3424 mono_emit_stack_alloc (MonoCompile *cfg, guchar *code, MonoInst* tree)
3425 {
3426         int sreg = tree->sreg1;
3427         int need_touch = FALSE;
3428
3429 #if defined(HOST_WIN32) || defined(MONO_ARCH_SIGSEGV_ON_ALTSTACK)
3430         if (!tree->flags & MONO_INST_INIT)
3431                 need_touch = TRUE;
3432 #endif
3433
3434         if (need_touch) {
3435                 guint8* br[5];
3436
3437                 /*
3438                  * Under Windows:
3439                  * If requested stack size is larger than one page,
3440                  * perform stack-touch operation
3441                  */
3442                 /*
3443                  * Generate stack probe code.
3444                  * Under Windows, it is necessary to allocate one page at a time,
3445                  * "touching" stack after each successful sub-allocation. This is
3446                  * because of the way stack growth is implemented - there is a
3447                  * guard page before the lowest stack page that is currently commited.
3448                  * Stack normally grows sequentially so OS traps access to the
3449                  * guard page and commits more pages when needed.
3450                  */
3451                 amd64_test_reg_imm (code, sreg, ~0xFFF);
3452                 br[0] = code; x86_branch8 (code, X86_CC_Z, 0, FALSE);
3453
3454                 br[2] = code; /* loop */
3455                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 0x1000);
3456                 amd64_test_membase_reg (code, AMD64_RSP, 0, AMD64_RSP);
3457                 amd64_alu_reg_imm (code, X86_SUB, sreg, 0x1000);
3458                 amd64_alu_reg_imm (code, X86_CMP, sreg, 0x1000);
3459                 br[3] = code; x86_branch8 (code, X86_CC_AE, 0, FALSE);
3460                 amd64_patch (br[3], br[2]);
3461                 amd64_test_reg_reg (code, sreg, sreg);
3462                 br[4] = code; x86_branch8 (code, X86_CC_Z, 0, FALSE);
3463                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, sreg);
3464
3465                 br[1] = code; x86_jump8 (code, 0);
3466
3467                 amd64_patch (br[0], code);
3468                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, sreg);
3469                 amd64_patch (br[1], code);
3470                 amd64_patch (br[4], code);
3471         }
3472         else
3473                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, tree->sreg1);
3474
3475         if (tree->flags & MONO_INST_INIT) {
3476                 int offset = 0;
3477                 if (tree->dreg != AMD64_RAX && sreg != AMD64_RAX) {
3478                         amd64_push_reg (code, AMD64_RAX);
3479                         offset += 8;
3480                 }
3481                 if (tree->dreg != AMD64_RCX && sreg != AMD64_RCX) {
3482                         amd64_push_reg (code, AMD64_RCX);
3483                         offset += 8;
3484                 }
3485                 if (tree->dreg != AMD64_RDI && sreg != AMD64_RDI) {
3486                         amd64_push_reg (code, AMD64_RDI);
3487                         offset += 8;
3488                 }
3489
3490                 amd64_shift_reg_imm (code, X86_SHR, sreg, 3);
3491                 if (sreg != AMD64_RCX)
3492                         amd64_mov_reg_reg (code, AMD64_RCX, sreg, 8);
3493                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
3494
3495                 amd64_lea_membase (code, AMD64_RDI, AMD64_RSP, offset);
3496                 if (cfg->param_area && cfg->arch.no_pushes)
3497                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RDI, cfg->param_area);
3498                 amd64_cld (code);
3499 #if defined(__default_codegen__)
3500                 amd64_prefix (code, X86_REP_PREFIX);
3501                 amd64_stosl (code);
3502 #elif defined(__native_client_codegen__)
3503                 /* NaCl stos pseudo-instruction */
3504                 amd64_codegen_pre(code);
3505                 /* First, clear the upper 32 bits of RDI (mov %edi, %edi)  */
3506                 amd64_mov_reg_reg (code, AMD64_RDI, AMD64_RDI, 4);
3507                 /* Add %r15 to %rdi using lea, condition flags unaffected. */
3508                 amd64_lea_memindex_size (code, AMD64_RDI, AMD64_R15, 0, AMD64_RDI, 0, 8);
3509                 amd64_prefix (code, X86_REP_PREFIX);
3510                 amd64_stosl (code);
3511                 amd64_codegen_post(code);
3512 #endif /* __native_client_codegen__ */
3513
3514                 if (tree->dreg != AMD64_RDI && sreg != AMD64_RDI)
3515                         amd64_pop_reg (code, AMD64_RDI);
3516                 if (tree->dreg != AMD64_RCX && sreg != AMD64_RCX)
3517                         amd64_pop_reg (code, AMD64_RCX);
3518                 if (tree->dreg != AMD64_RAX && sreg != AMD64_RAX)
3519                         amd64_pop_reg (code, AMD64_RAX);
3520         }
3521         return code;
3522 }
3523
3524 static guint8*
3525 emit_move_return_value (MonoCompile *cfg, MonoInst *ins, guint8 *code)
3526 {
3527         CallInfo *cinfo;
3528         guint32 quad;
3529
3530         /* Move return value to the target register */
3531         /* FIXME: do this in the local reg allocator */
3532         switch (ins->opcode) {
3533         case OP_CALL:
3534         case OP_CALL_REG:
3535         case OP_CALL_MEMBASE:
3536         case OP_LCALL:
3537         case OP_LCALL_REG:
3538         case OP_LCALL_MEMBASE:
3539                 g_assert (ins->dreg == AMD64_RAX);
3540                 break;
3541         case OP_FCALL:
3542         case OP_FCALL_REG:
3543         case OP_FCALL_MEMBASE:
3544                 if (((MonoCallInst*)ins)->signature->ret->type == MONO_TYPE_R4) {
3545                         amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, AMD64_XMM0);
3546                 }
3547                 else {
3548                         if (ins->dreg != AMD64_XMM0)
3549                                 amd64_sse_movsd_reg_reg (code, ins->dreg, AMD64_XMM0);
3550                 }
3551                 break;
3552         case OP_VCALL:
3553         case OP_VCALL_REG:
3554         case OP_VCALL_MEMBASE:
3555         case OP_VCALL2:
3556         case OP_VCALL2_REG:
3557         case OP_VCALL2_MEMBASE:
3558                 cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, ((MonoCallInst*)ins)->signature);
3559                 if (cinfo->ret.storage == ArgValuetypeInReg) {
3560                         MonoInst *loc = cfg->arch.vret_addr_loc;
3561
3562                         /* Load the destination address */
3563                         g_assert (loc->opcode == OP_REGOFFSET);
3564                         amd64_mov_reg_membase (code, AMD64_RCX, loc->inst_basereg, loc->inst_offset, sizeof(gpointer));
3565
3566                         for (quad = 0; quad < 2; quad ++) {
3567                                 switch (cinfo->ret.pair_storage [quad]) {
3568                                 case ArgInIReg:
3569                                         amd64_mov_membase_reg (code, AMD64_RCX, (quad * sizeof(mgreg_t)), cinfo->ret.pair_regs [quad], sizeof(mgreg_t));
3570                                         break;
3571                                 case ArgInFloatSSEReg:
3572                                         amd64_movss_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad]);
3573                                         break;
3574                                 case ArgInDoubleSSEReg:
3575                                         amd64_movsd_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad]);
3576                                         break;
3577                                 case ArgNone:
3578                                         break;
3579                                 default:
3580                                         NOT_IMPLEMENTED;
3581                                 }
3582                         }
3583                 }
3584                 break;
3585         }
3586
3587         return code;
3588 }
3589
3590 #endif /* DISABLE_JIT */
3591
3592 #ifdef __APPLE__
3593 static int tls_gs_offset;
3594 #endif
3595
3596 gboolean
3597 mono_amd64_have_tls_get (void)
3598 {
3599 #ifdef __APPLE__
3600         static gboolean have_tls_get = FALSE;
3601         static gboolean inited = FALSE;
3602         guint8 *ins;
3603
3604         if (inited)
3605                 return have_tls_get;
3606
3607         ins = (guint8*)pthread_getspecific;
3608
3609         /*
3610          * We're looking for these two instructions:
3611          *
3612          * mov    %gs:[offset](,%rdi,8),%rax
3613          * retq
3614          */
3615         have_tls_get = ins [0] == 0x65 &&
3616                        ins [1] == 0x48 &&
3617                        ins [2] == 0x8b &&
3618                        ins [3] == 0x04 &&
3619                        ins [4] == 0xfd &&
3620                        ins [6] == 0x00 &&
3621                        ins [7] == 0x00 &&
3622                        ins [8] == 0x00 &&
3623                        ins [9] == 0xc3;
3624
3625         inited = TRUE;
3626
3627         tls_gs_offset = ins[5];
3628
3629         return have_tls_get;
3630 #else
3631         return TRUE;
3632 #endif
3633 }
3634
3635 /*
3636  * mono_amd64_emit_tls_get:
3637  * @code: buffer to store code to
3638  * @dreg: hard register where to place the result
3639  * @tls_offset: offset info
3640  *
3641  * mono_amd64_emit_tls_get emits in @code the native code that puts in
3642  * the dreg register the item in the thread local storage identified
3643  * by tls_offset.
3644  *
3645  * Returns: a pointer to the end of the stored code
3646  */
3647 guint8*
3648 mono_amd64_emit_tls_get (guint8* code, int dreg, int tls_offset)
3649 {
3650 #ifdef HOST_WIN32
3651         g_assert (tls_offset < 64);
3652         x86_prefix (code, X86_GS_PREFIX);
3653         amd64_mov_reg_mem (code, dreg, (tls_offset * 8) + 0x1480, 8);
3654 #elif defined(__APPLE__)
3655         x86_prefix (code, X86_GS_PREFIX);
3656         amd64_mov_reg_mem (code, dreg, tls_gs_offset + (tls_offset * 8), 8);
3657 #else
3658         if (optimize_for_xen) {
3659                 x86_prefix (code, X86_FS_PREFIX);
3660                 amd64_mov_reg_mem (code, dreg, 0, 8);
3661                 amd64_mov_reg_membase (code, dreg, dreg, tls_offset, 8);
3662         } else {
3663                 x86_prefix (code, X86_FS_PREFIX);
3664                 amd64_mov_reg_mem (code, dreg, tls_offset, 8);
3665         }
3666 #endif
3667         return code;
3668 }
3669
3670 /*
3671  * emit_setup_lmf:
3672  *
3673  *   Emit code to initialize an LMF structure at LMF_OFFSET.
3674  */
3675 static guint8*
3676 emit_setup_lmf (MonoCompile *cfg, guint8 *code, gint32 lmf_offset, int cfa_offset)
3677 {
3678         int i;
3679
3680         /*
3681          * The ip field is not set, the exception handling code will obtain it from the stack location pointed to by the sp field.
3682          */
3683         /*
3684          * sp is saved right before calls but we need to save it here too so
3685          * async stack walks would work.
3686          */
3687         amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rsp), AMD64_RSP, 8);
3688         /* Skip method (only needed for trampoline LMF frames) */
3689         /* Save callee saved regs */
3690         for (i = 0; i < MONO_MAX_IREGS; ++i) {
3691                 int offset;
3692
3693                 switch (i) {
3694                 case AMD64_RBX: offset = G_STRUCT_OFFSET (MonoLMF, rbx); break;
3695                 case AMD64_RBP: offset = G_STRUCT_OFFSET (MonoLMF, rbp); break;
3696                 case AMD64_R12: offset = G_STRUCT_OFFSET (MonoLMF, r12); break;
3697                 case AMD64_R13: offset = G_STRUCT_OFFSET (MonoLMF, r13); break;
3698                 case AMD64_R14: offset = G_STRUCT_OFFSET (MonoLMF, r14); break;
3699 #ifndef __native_client_codegen__
3700                 case AMD64_R15: offset = G_STRUCT_OFFSET (MonoLMF, r15); break;
3701 #endif
3702 #ifdef HOST_WIN32
3703                 case AMD64_RDI: offset = G_STRUCT_OFFSET (MonoLMF, rdi); break;
3704                 case AMD64_RSI: offset = G_STRUCT_OFFSET (MonoLMF, rsi); break;
3705 #endif
3706                 default:
3707                         offset = -1;
3708                         break;
3709                 }
3710
3711                 if (offset != -1) {
3712                         amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + offset, i, 8);
3713                         if ((cfg->arch.omit_fp || (i != AMD64_RBP)) && cfa_offset != -1)
3714                                 mono_emit_unwind_op_offset (cfg, code, i, - (cfa_offset - (lmf_offset + offset)));
3715                 }
3716         }
3717
3718         /* These can't contain refs */
3719         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), SLOT_NOREF);
3720         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, lmf_addr), SLOT_NOREF);
3721         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, method), SLOT_NOREF);
3722         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rip), SLOT_NOREF);
3723         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rsp), SLOT_NOREF);
3724
3725         /* These are handled automatically by the stack marking code */
3726         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbx), SLOT_NOREF);
3727         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbp), SLOT_NOREF);
3728         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r12), SLOT_NOREF);
3729         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r13), SLOT_NOREF);
3730         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r14), SLOT_NOREF);
3731         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r15), SLOT_NOREF);
3732 #ifdef HOST_WIN32
3733         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rdi), SLOT_NOREF);
3734         mini_gc_set_slot_type_from_fp (cfg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rsi), SLOT_NOREF);
3735 #endif
3736
3737         return code;
3738 }
3739
3740 /*
3741  * emit_save_lmf:
3742  *
3743  *   Emit code to push an LMF structure on the LMF stack.
3744  */
3745 static guint8*
3746 emit_save_lmf (MonoCompile *cfg, guint8 *code, gint32 lmf_offset, gboolean *args_clobbered)
3747 {
3748         if ((lmf_tls_offset != -1) && !optimize_for_xen) {
3749                 /*
3750                  * Optimized version which uses the mono_lmf TLS variable instead of
3751                  * indirection through the mono_lmf_addr TLS variable.
3752                  */
3753                 /* %rax = previous_lmf */
3754                 x86_prefix (code, X86_FS_PREFIX);
3755                 amd64_mov_reg_mem (code, AMD64_RAX, lmf_tls_offset, 8);
3756
3757                 /* Save previous_lmf */
3758                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), AMD64_RAX, 8);
3759                 /* Set new lmf */
3760                 if (lmf_offset == 0) {
3761                         x86_prefix (code, X86_FS_PREFIX);
3762                         amd64_mov_mem_reg (code, lmf_tls_offset, cfg->frame_reg, 8);
3763                 } else {
3764                         amd64_lea_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset);
3765                         x86_prefix (code, X86_FS_PREFIX);
3766                         amd64_mov_mem_reg (code, lmf_tls_offset, AMD64_R11, 8);
3767                 }
3768         } else {
3769                 if (lmf_addr_tls_offset != -1) {
3770                         /* Load lmf quicky using the FS register */
3771                         code = mono_amd64_emit_tls_get (code, AMD64_RAX, lmf_addr_tls_offset);
3772 #ifdef HOST_WIN32
3773                         /* The TLS key actually contains a pointer to the MonoJitTlsData structure */
3774                         /* FIXME: Add a separate key for LMF to avoid this */
3775                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RAX, G_STRUCT_OFFSET (MonoJitTlsData, lmf));
3776 #endif
3777                 }
3778                 else {
3779                         /*
3780                          * The call might clobber argument registers, but they are already
3781                          * saved to the stack/global regs.
3782                          */
3783                         if (args_clobbered)
3784                                 *args_clobbered = TRUE;
3785                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
3786                                                           (gpointer)"mono_get_lmf_addr", TRUE);
3787                 }
3788
3789                 /* Save lmf_addr */
3790                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, lmf_addr), AMD64_RAX, sizeof(gpointer));
3791                 /* Save previous_lmf */
3792                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_RAX, 0, sizeof(gpointer));
3793                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), AMD64_R11, sizeof(gpointer));
3794                 /* Set new lmf */
3795                 amd64_lea_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset);
3796                 amd64_mov_membase_reg (code, AMD64_RAX, 0, AMD64_R11, sizeof(gpointer));
3797         }
3798
3799         return code;
3800 }
3801
3802 /*
3803  * emit_save_lmf:
3804  *
3805  *   Emit code to pop an LMF structure from the LMF stack.
3806  */
3807 static guint8*
3808 emit_restore_lmf (MonoCompile *cfg, guint8 *code, gint32 lmf_offset)
3809 {
3810         if ((lmf_tls_offset != -1) && !optimize_for_xen) {
3811                 /*
3812                  * Optimized version which uses the mono_lmf TLS variable instead of indirection
3813                  * through the mono_lmf_addr TLS variable.
3814                  */
3815                 /* reg = previous_lmf */
3816                 amd64_mov_reg_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), sizeof(gpointer));
3817                 x86_prefix (code, X86_FS_PREFIX);
3818                 amd64_mov_mem_reg (code, lmf_tls_offset, AMD64_R11, 8);
3819         } else {
3820                 /* Restore previous lmf */
3821                 amd64_mov_reg_membase (code, AMD64_RCX, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), sizeof(gpointer));
3822                 amd64_mov_reg_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, lmf_addr), sizeof(gpointer));
3823                 amd64_mov_membase_reg (code, AMD64_R11, 0, AMD64_RCX, sizeof(gpointer));
3824         }
3825
3826         return code;
3827 }
3828
3829 #define REAL_PRINT_REG(text,reg) \
3830 mono_assert (reg >= 0); \
3831 amd64_push_reg (code, AMD64_RAX); \
3832 amd64_push_reg (code, AMD64_RDX); \
3833 amd64_push_reg (code, AMD64_RCX); \
3834 amd64_push_reg (code, reg); \
3835 amd64_push_imm (code, reg); \
3836 amd64_push_imm (code, text " %d %p\n"); \
3837 amd64_mov_reg_imm (code, AMD64_RAX, printf); \
3838 amd64_call_reg (code, AMD64_RAX); \
3839 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 3*4); \
3840 amd64_pop_reg (code, AMD64_RCX); \
3841 amd64_pop_reg (code, AMD64_RDX); \
3842 amd64_pop_reg (code, AMD64_RAX);
3843
3844 /* benchmark and set based on cpu */
3845 #define LOOP_ALIGNMENT 8
3846 #define bb_is_loop_start(bb) ((bb)->loop_body_start && (bb)->nesting)
3847
3848 #ifndef DISABLE_JIT
3849 void
3850 mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
3851 {
3852         MonoInst *ins;
3853         MonoCallInst *call;
3854         guint offset;
3855         guint8 *code = cfg->native_code + cfg->code_len;
3856         MonoInst *last_ins = NULL;
3857         guint last_offset = 0;
3858         int max_len;
3859
3860         /* Fix max_offset estimate for each successor bb */
3861         if (cfg->opt & MONO_OPT_BRANCH) {
3862                 int current_offset = cfg->code_len;
3863                 MonoBasicBlock *current_bb;
3864                 for (current_bb = bb; current_bb != NULL; current_bb = current_bb->next_bb) {
3865                         current_bb->max_offset = current_offset;
3866                         current_offset += current_bb->max_length;
3867                 }
3868         }
3869
3870         if (cfg->opt & MONO_OPT_LOOP) {
3871                 int pad, align = LOOP_ALIGNMENT;
3872                 /* set alignment depending on cpu */
3873                 if (bb_is_loop_start (bb) && (pad = (cfg->code_len & (align - 1)))) {
3874                         pad = align - pad;
3875                         /*g_print ("adding %d pad at %x to loop in %s\n", pad, cfg->code_len, cfg->method->name);*/
3876                         amd64_padding (code, pad);
3877                         cfg->code_len += pad;
3878                         bb->native_offset = cfg->code_len;
3879                 }
3880         }
3881
3882 #if defined(__native_client_codegen__)
3883         /* For Native Client, all indirect call/jump targets must be */
3884         /* 32-byte aligned.  Exception handler blocks are jumped to  */
3885         /* indirectly as well.                                       */
3886         gboolean bb_needs_alignment = (bb->flags & BB_INDIRECT_JUMP_TARGET) ||
3887                                       (bb->flags & BB_EXCEPTION_HANDLER);
3888
3889         if ( bb_needs_alignment && ((cfg->code_len & kNaClAlignmentMask) != 0)) {
3890                 int pad = kNaClAlignment - (cfg->code_len & kNaClAlignmentMask);
3891                 if (pad != kNaClAlignment) code = mono_arch_nacl_pad(code, pad);
3892                 cfg->code_len += pad;
3893                 bb->native_offset = cfg->code_len;
3894         }
3895 #endif  /*__native_client_codegen__*/
3896
3897         if (cfg->verbose_level > 2)
3898                 g_print ("Basic block %d starting at offset 0x%x\n", bb->block_num, bb->native_offset);
3899
3900         if (cfg->prof_options & MONO_PROFILE_COVERAGE) {
3901                 MonoProfileCoverageInfo *cov = cfg->coverage_info;
3902                 g_assert (!cfg->compile_aot);
3903
3904                 cov->data [bb->dfn].cil_code = bb->cil_code;
3905                 amd64_mov_reg_imm (code, AMD64_R11, (guint64)&cov->data [bb->dfn].count);
3906                 /* this is not thread save, but good enough */
3907                 amd64_inc_membase (code, AMD64_R11, 0);
3908         }
3909
3910         offset = code - cfg->native_code;
3911
3912         mono_debug_open_block (cfg, bb, offset);
3913
3914     if (mono_break_at_bb_method && mono_method_desc_full_match (mono_break_at_bb_method, cfg->method) && bb->block_num == mono_break_at_bb_bb_num)
3915                 x86_breakpoint (code);
3916
3917         MONO_BB_FOR_EACH_INS (bb, ins) {
3918                 offset = code - cfg->native_code;
3919
3920                 max_len = ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
3921
3922 #define EXTRA_CODE_SPACE (NACL_SIZE (16, 16 + kNaClAlignment))
3923
3924                 if (G_UNLIKELY (offset > (cfg->code_size - max_len - EXTRA_CODE_SPACE))) {
3925                         cfg->code_size *= 2;
3926                         cfg->native_code = mono_realloc_native_code(cfg);
3927                         code = cfg->native_code + offset;
3928                         cfg->stat_code_reallocs++;
3929                 }
3930
3931                 if (cfg->debug_info)
3932                         mono_debug_record_line_number (cfg, ins, offset);
3933
3934                 switch (ins->opcode) {
3935                 case OP_BIGMUL:
3936                         amd64_mul_reg (code, ins->sreg2, TRUE);
3937                         break;
3938                 case OP_BIGMUL_UN:
3939                         amd64_mul_reg (code, ins->sreg2, FALSE);
3940                         break;
3941                 case OP_X86_SETEQ_MEMBASE:
3942                         amd64_set_membase (code, X86_CC_EQ, ins->inst_basereg, ins->inst_offset, TRUE);
3943                         break;
3944                 case OP_STOREI1_MEMBASE_IMM:
3945                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 1);
3946                         break;
3947                 case OP_STOREI2_MEMBASE_IMM:
3948                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 2);
3949                         break;
3950                 case OP_STOREI4_MEMBASE_IMM:
3951                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 4);
3952                         break;
3953                 case OP_STOREI1_MEMBASE_REG:
3954                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 1);
3955                         break;
3956                 case OP_STOREI2_MEMBASE_REG:
3957                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 2);
3958                         break;
3959                 /* In AMD64 NaCl, pointers are 4 bytes, */
3960                 /*  so STORE_* != STOREI8_*. Likewise below. */
3961                 case OP_STORE_MEMBASE_REG:
3962                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, sizeof(gpointer));
3963                         break;
3964                 case OP_STOREI8_MEMBASE_REG:
3965                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 8);
3966                         break;
3967                 case OP_STOREI4_MEMBASE_REG:
3968                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 4);
3969                         break;
3970                 case OP_STORE_MEMBASE_IMM:
3971 #ifndef __native_client_codegen__
3972                         /* In NaCl, this could be a PCONST type, which could */
3973                         /* mean a pointer type was copied directly into the  */
3974                         /* lower 32-bits of inst_imm, so for InvalidPtr==-1  */
3975                         /* the value would be 0x00000000FFFFFFFF which is    */
3976                         /* not proper for an imm32 unless you cast it.       */
3977                         g_assert (amd64_is_imm32 (ins->inst_imm));
3978 #endif
3979                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, (gint32)ins->inst_imm, sizeof(gpointer));
3980                         break;
3981                 case OP_STOREI8_MEMBASE_IMM:
3982                         g_assert (amd64_is_imm32 (ins->inst_imm));
3983                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 8);
3984                         break;
3985                 case OP_LOAD_MEM:
3986 #ifdef __mono_ilp32__
3987                         /* In ILP32, pointers are 4 bytes, so separate these */
3988                         /* cases, use literal 8 below where we really want 8 */
3989                         amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
3990                         amd64_mov_reg_membase (code, ins->dreg, ins->dreg, 0, sizeof(gpointer));
3991                         break;
3992 #endif
3993                 case OP_LOADI8_MEM:
3994                         // FIXME: Decompose this earlier
3995                         if (amd64_is_imm32 (ins->inst_imm))
3996                                 amd64_mov_reg_mem (code, ins->dreg, ins->inst_imm, 8);
3997                         else {
3998                                 amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
3999                                 amd64_mov_reg_membase (code, ins->dreg, ins->dreg, 0, 8);
4000                         }
4001                         break;
4002                 case OP_LOADI4_MEM:
4003                         amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
4004                         amd64_movsxd_reg_membase (code, ins->dreg, ins->dreg, 0);
4005                         break;
4006                 case OP_LOADU4_MEM:
4007                         // FIXME: Decompose this earlier
4008                         if (amd64_is_imm32 (ins->inst_imm))
4009                                 amd64_mov_reg_mem (code, ins->dreg, ins->inst_imm, 4);
4010                         else {
4011                                 amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
4012                                 amd64_mov_reg_membase (code, ins->dreg, ins->dreg, 0, 4);
4013                         }
4014                         break;
4015                 case OP_LOADU1_MEM:
4016                         amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
4017                         amd64_widen_membase (code, ins->dreg, ins->dreg, 0, FALSE, FALSE);
4018                         break;
4019                 case OP_LOADU2_MEM:
4020                         /* For NaCl, pointers are 4 bytes, so separate these */
4021                         /* cases, use literal 8 below where we really want 8 */
4022                         amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
4023                         amd64_widen_membase (code, ins->dreg, ins->dreg, 0, FALSE, TRUE);
4024                         break;
4025                 case OP_LOAD_MEMBASE:
4026                         g_assert (amd64_is_imm32 (ins->inst_offset));
4027                         amd64_mov_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, sizeof(gpointer));
4028                         break;
4029                 case OP_LOADI8_MEMBASE:
4030                         /* Use literal 8 instead of sizeof pointer or */
4031                         /* register, we really want 8 for this opcode */
4032                         g_assert (amd64_is_imm32 (ins->inst_offset));
4033                         amd64_mov_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, 8);
4034                         break;
4035                 case OP_LOADI4_MEMBASE:
4036                         amd64_movsxd_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
4037                         break;
4038                 case OP_LOADU4_MEMBASE:
4039                         amd64_mov_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, 4);
4040                         break;
4041                 case OP_LOADU1_MEMBASE:
4042                         /* The cpu zero extends the result into 64 bits */
4043                         amd64_widen_membase_size (code, ins->dreg, ins->inst_basereg, ins->inst_offset, FALSE, FALSE, 4);
4044                         break;
4045                 case OP_LOADI1_MEMBASE:
4046                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, TRUE, FALSE);
4047                         break;
4048                 case OP_LOADU2_MEMBASE:
4049                         /* The cpu zero extends the result into 64 bits */
4050                         amd64_widen_membase_size (code, ins->dreg, ins->inst_basereg, ins->inst_offset, FALSE, TRUE, 4);
4051                         break;
4052                 case OP_LOADI2_MEMBASE:
4053                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, TRUE, TRUE);
4054                         break;
4055                 case OP_AMD64_LOADI8_MEMINDEX:
4056                         amd64_mov_reg_memindex_size (code, ins->dreg, ins->inst_basereg, 0, ins->inst_indexreg, 0, 8);
4057                         break;
4058                 case OP_LCONV_TO_I1:
4059                 case OP_ICONV_TO_I1:
4060                 case OP_SEXT_I1:
4061                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, FALSE);
4062                         break;
4063                 case OP_LCONV_TO_I2:
4064                 case OP_ICONV_TO_I2:
4065                 case OP_SEXT_I2:
4066                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, TRUE);
4067                         break;
4068                 case OP_LCONV_TO_U1:
4069                 case OP_ICONV_TO_U1:
4070                         amd64_widen_reg (code, ins->dreg, ins->sreg1, FALSE, FALSE);
4071                         break;
4072                 case OP_LCONV_TO_U2:
4073                 case OP_ICONV_TO_U2:
4074                         amd64_widen_reg (code, ins->dreg, ins->sreg1, FALSE, TRUE);
4075                         break;
4076                 case OP_ZEXT_I4:
4077                         /* Clean out the upper word */
4078                         amd64_mov_reg_reg_size (code, ins->dreg, ins->sreg1, 4);
4079                         break;
4080                 case OP_SEXT_I4:
4081                         amd64_movsxd_reg_reg (code, ins->dreg, ins->sreg1);
4082                         break;
4083                 case OP_COMPARE:
4084                 case OP_LCOMPARE:
4085                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
4086                         break;
4087                 case OP_COMPARE_IMM:
4088 #if defined(__mono_ilp32__)
4089                         /* Comparison of pointer immediates should be 4 bytes to avoid sign-extend problems */
4090                         g_assert (amd64_is_imm32 (ins->inst_imm));
4091                         amd64_alu_reg_imm_size (code, X86_CMP, ins->sreg1, ins->inst_imm, 4);
4092                         break;
4093 #endif
4094                 case OP_LCOMPARE_IMM:
4095                         g_assert (amd64_is_imm32 (ins->inst_imm));
4096                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, ins->inst_imm);
4097                         break;
4098                 case OP_X86_COMPARE_REG_MEMBASE:
4099                         amd64_alu_reg_membase (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset);
4100                         break;
4101                 case OP_X86_TEST_NULL:
4102                         amd64_test_reg_reg_size (code, ins->sreg1, ins->sreg1, 4);
4103                         break;
4104                 case OP_AMD64_TEST_NULL:
4105                         amd64_test_reg_reg (code, ins->sreg1, ins->sreg1);
4106                         break;
4107
4108                 case OP_X86_ADD_REG_MEMBASE:
4109                         amd64_alu_reg_membase_size (code, X86_ADD, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
4110                         break;
4111                 case OP_X86_SUB_REG_MEMBASE:
4112                         amd64_alu_reg_membase_size (code, X86_SUB, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
4113                         break;
4114                 case OP_X86_AND_REG_MEMBASE:
4115                         amd64_alu_reg_membase_size (code, X86_AND, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
4116                         break;
4117                 case OP_X86_OR_REG_MEMBASE:
4118                         amd64_alu_reg_membase_size (code, X86_OR, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
4119                         break;
4120                 case OP_X86_XOR_REG_MEMBASE:
4121                         amd64_alu_reg_membase_size (code, X86_XOR, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
4122                         break;
4123
4124                 case OP_X86_ADD_MEMBASE_IMM:
4125                         /* FIXME: Make a 64 version too */
4126                         amd64_alu_membase_imm_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
4127                         break;
4128                 case OP_X86_SUB_MEMBASE_IMM:
4129                         g_assert (amd64_is_imm32 (ins->inst_imm));
4130                         amd64_alu_membase_imm_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
4131                         break;
4132                 case OP_X86_AND_MEMBASE_IMM:
4133                         g_assert (amd64_is_imm32 (ins->inst_imm));
4134                         amd64_alu_membase_imm_size (code, X86_AND, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
4135                         break;
4136                 case OP_X86_OR_MEMBASE_IMM:
4137                         g_assert (amd64_is_imm32 (ins->inst_imm));
4138                         amd64_alu_membase_imm_size (code, X86_OR, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
4139                         break;
4140                 case OP_X86_XOR_MEMBASE_IMM:
4141                         g_assert (amd64_is_imm32 (ins->inst_imm));
4142                         amd64_alu_membase_imm_size (code, X86_XOR, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
4143                         break;
4144                 case OP_X86_ADD_MEMBASE_REG:
4145                         amd64_alu_membase_reg_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
4146                         break;
4147                 case OP_X86_SUB_MEMBASE_REG:
4148                         amd64_alu_membase_reg_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
4149                         break;
4150                 case OP_X86_AND_MEMBASE_REG:
4151                         amd64_alu_membase_reg_size (code, X86_AND, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
4152                         break;
4153                 case OP_X86_OR_MEMBASE_REG:
4154                         amd64_alu_membase_reg_size (code, X86_OR, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
4155                         break;
4156                 case OP_X86_XOR_MEMBASE_REG:
4157                         amd64_alu_membase_reg_size (code, X86_XOR, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
4158                         break;
4159                 case OP_X86_INC_MEMBASE:
4160                         amd64_inc_membase_size (code, ins->inst_basereg, ins->inst_offset, 4);
4161                         break;
4162                 case OP_X86_INC_REG:
4163                         amd64_inc_reg_size (code, ins->dreg, 4);
4164                         break;
4165                 case OP_X86_DEC_MEMBASE:
4166                         amd64_dec_membase_size (code, ins->inst_basereg, ins->inst_offset, 4);
4167                         break;
4168                 case OP_X86_DEC_REG:
4169                         amd64_dec_reg_size (code, ins->dreg, 4);
4170                         break;
4171                 case OP_X86_MUL_REG_MEMBASE:
4172                 case OP_X86_MUL_MEMBASE_REG:
4173                         amd64_imul_reg_membase_size (code, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
4174                         break;
4175                 case OP_AMD64_ICOMPARE_MEMBASE_REG:
4176                         amd64_alu_membase_reg_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
4177                         break;
4178                 case OP_AMD64_ICOMPARE_MEMBASE_IMM:
4179                         amd64_alu_membase_imm_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
4180                         break;
4181                 case OP_AMD64_COMPARE_MEMBASE_REG:
4182                         amd64_alu_membase_reg_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
4183                         break;
4184                 case OP_AMD64_COMPARE_MEMBASE_IMM:
4185                         g_assert (amd64_is_imm32 (ins->inst_imm));
4186                         amd64_alu_membase_imm_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
4187                         break;
4188                 case OP_X86_COMPARE_MEMBASE8_IMM:
4189                         amd64_alu_membase8_imm_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
4190                         break;
4191                 case OP_AMD64_ICOMPARE_REG_MEMBASE:
4192                         amd64_alu_reg_membase_size (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
4193                         break;
4194                 case OP_AMD64_COMPARE_REG_MEMBASE:
4195                         amd64_alu_reg_membase_size (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
4196                         break;
4197
4198                 case OP_AMD64_ADD_REG_MEMBASE:
4199                         amd64_alu_reg_membase_size (code, X86_ADD, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
4200                         break;
4201                 case OP_AMD64_SUB_REG_MEMBASE:
4202                         amd64_alu_reg_membase_size (code, X86_SUB, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
4203                         break;
4204                 case OP_AMD64_AND_REG_MEMBASE:
4205                         amd64_alu_reg_membase_size (code, X86_AND, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
4206                         break;
4207                 case OP_AMD64_OR_REG_MEMBASE:
4208                         amd64_alu_reg_membase_size (code, X86_OR, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
4209                         break;
4210                 case OP_AMD64_XOR_REG_MEMBASE:
4211                         amd64_alu_reg_membase_size (code, X86_XOR, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
4212                         break;
4213
4214                 case OP_AMD64_ADD_MEMBASE_REG:
4215                         amd64_alu_membase_reg_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
4216                         break;
4217                 case OP_AMD64_SUB_MEMBASE_REG:
4218                         amd64_alu_membase_reg_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
4219                         break;
4220                 case OP_AMD64_AND_MEMBASE_REG:
4221                         amd64_alu_membase_reg_size (code, X86_AND, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
4222                         break;
4223                 case OP_AMD64_OR_MEMBASE_REG:
4224                         amd64_alu_membase_reg_size (code, X86_OR, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
4225                         break;
4226                 case OP_AMD64_XOR_MEMBASE_REG:
4227                         amd64_alu_membase_reg_size (code, X86_XOR, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
4228                         break;
4229
4230                 case OP_AMD64_ADD_MEMBASE_IMM:
4231                         g_assert (amd64_is_imm32 (ins->inst_imm));
4232                         amd64_alu_membase_imm_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
4233                         break;
4234                 case OP_AMD64_SUB_MEMBASE_IMM:
4235                         g_assert (amd64_is_imm32 (ins->inst_imm));
4236                         amd64_alu_membase_imm_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
4237                         break;
4238                 case OP_AMD64_AND_MEMBASE_IMM:
4239                         g_assert (amd64_is_imm32 (ins->inst_imm));
4240                         amd64_alu_membase_imm_size (code, X86_AND, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
4241                         break;
4242                 case OP_AMD64_OR_MEMBASE_IMM:
4243                         g_assert (amd64_is_imm32 (ins->inst_imm));
4244                         amd64_alu_membase_imm_size (code, X86_OR, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
4245                         break;
4246                 case OP_AMD64_XOR_MEMBASE_IMM:
4247                         g_assert (amd64_is_imm32 (ins->inst_imm));
4248                         amd64_alu_membase_imm_size (code, X86_XOR, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
4249                         break;
4250
4251                 case OP_BREAK:
4252                         amd64_breakpoint (code);
4253                         break;
4254                 case OP_RELAXED_NOP:
4255                         x86_prefix (code, X86_REP_PREFIX);
4256                         x86_nop (code);
4257                         break;
4258                 case OP_HARD_NOP:
4259                         x86_nop (code);
4260                         break;
4261                 case OP_NOP:
4262                 case OP_DUMMY_USE:
4263                 case OP_DUMMY_STORE:
4264                 case OP_NOT_REACHED:
4265                 case OP_NOT_NULL:
4266                         break;
4267                 case OP_SEQ_POINT: {
4268                         int i;
4269
4270                         /*
4271                          * Read from the single stepping trigger page. This will cause a
4272                          * SIGSEGV when single stepping is enabled.
4273                          * We do this _before_ the breakpoint, so single stepping after
4274                          * a breakpoint is hit will step to the next IL offset.
4275                          */
4276                         if (ins->flags & MONO_INST_SINGLE_STEP_LOC) {
4277                                 MonoInst *var = cfg->arch.ss_trigger_page_var;
4278
4279                                 amd64_mov_reg_membase (code, AMD64_R11, var->inst_basereg, var->inst_offset, 8);
4280                                 amd64_alu_membase_imm_size (code, X86_CMP, AMD64_R11, 0, 0, 4);
4281                         }
4282
4283                         /*
4284                          * This is the address which is saved in seq points,
4285                          */
4286                         mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
4287
4288                         if (cfg->compile_aot) {
4289                                 guint32 offset = code - cfg->native_code;
4290                                 guint32 val;
4291                                 MonoInst *info_var = cfg->arch.seq_point_info_var;
4292
4293                                 /* Load info var */
4294                                 amd64_mov_reg_membase (code, AMD64_R11, info_var->inst_basereg, info_var->inst_offset, 8);
4295                                 val = ((offset) * sizeof (guint8*)) + G_STRUCT_OFFSET (SeqPointInfo, bp_addrs);
4296                                 /* Load the info->bp_addrs [offset], which is either a valid address or the address of a trigger page */
4297                                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_R11, val, 8);
4298                                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_R11, 0, 8);
4299                         } else {
4300                                 /*
4301                                  * A placeholder for a possible breakpoint inserted by
4302                                  * mono_arch_set_breakpoint ().
4303                                  */
4304                                 for (i = 0; i < breakpoint_size; ++i)
4305                                         x86_nop (code);
4306                         }
4307                         /*
4308                          * Add an additional nop so skipping the bp doesn't cause the ip to point
4309                          * to another IL offset.
4310                          */
4311                         x86_nop (code);
4312                         break;
4313                 }
4314                 case OP_ADDCC:
4315                 case OP_LADDCC:
4316                 case OP_LADD:
4317                         amd64_alu_reg_reg (code, X86_ADD, ins->sreg1, ins->sreg2);
4318                         break;
4319                 case OP_ADC:
4320                         amd64_alu_reg_reg (code, X86_ADC, ins->sreg1, ins->sreg2);
4321                         break;
4322                 case OP_ADD_IMM:
4323                 case OP_LADD_IMM:
4324                         g_assert (amd64_is_imm32 (ins->inst_imm));
4325                         amd64_alu_reg_imm (code, X86_ADD, ins->dreg, ins->inst_imm);
4326                         break;
4327                 case OP_ADC_IMM:
4328                         g_assert (amd64_is_imm32 (ins->inst_imm));
4329                         amd64_alu_reg_imm (code, X86_ADC, ins->dreg, ins->inst_imm);
4330                         break;
4331                 case OP_SUBCC:
4332                 case OP_LSUBCC:
4333                 case OP_LSUB:
4334                         amd64_alu_reg_reg (code, X86_SUB, ins->sreg1, ins->sreg2);
4335                         break;
4336                 case OP_SBB:
4337                         amd64_alu_reg_reg (code, X86_SBB, ins->sreg1, ins->sreg2);
4338                         break;
4339                 case OP_SUB_IMM:
4340                 case OP_LSUB_IMM:
4341                         g_assert (amd64_is_imm32 (ins->inst_imm));
4342                         amd64_alu_reg_imm (code, X86_SUB, ins->dreg, ins->inst_imm);
4343                         break;
4344                 case OP_SBB_IMM:
4345                         g_assert (amd64_is_imm32 (ins->inst_imm));
4346                         amd64_alu_reg_imm (code, X86_SBB, ins->dreg, ins->inst_imm);
4347                         break;
4348                 case OP_LAND:
4349                         amd64_alu_reg_reg (code, X86_AND, ins->sreg1, ins->sreg2);
4350                         break;
4351                 case OP_AND_IMM:
4352                 case OP_LAND_IMM:
4353                         g_assert (amd64_is_imm32 (ins->inst_imm));
4354                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ins->inst_imm);
4355                         break;
4356                 case OP_LMUL:
4357                         amd64_imul_reg_reg (code, ins->sreg1, ins->sreg2);
4358                         break;
4359                 case OP_MUL_IMM:
4360                 case OP_LMUL_IMM:
4361                 case OP_IMUL_IMM: {
4362                         guint32 size = (ins->opcode == OP_IMUL_IMM) ? 4 : 8;
4363
4364                         switch (ins->inst_imm) {
4365                         case 2:
4366                                 /* MOV r1, r2 */
4367                                 /* ADD r1, r1 */
4368                                 if (ins->dreg != ins->sreg1)
4369                                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, size);
4370                                 amd64_alu_reg_reg (code, X86_ADD, ins->dreg, ins->dreg);
4371                                 break;
4372                         case 3:
4373                                 /* LEA r1, [r2 + r2*2] */
4374                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 1);
4375                                 break;
4376                         case 5:
4377                                 /* LEA r1, [r2 + r2*4] */
4378                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
4379                                 break;
4380                         case 6:
4381                                 /* LEA r1, [r2 + r2*2] */
4382                                 /* ADD r1, r1          */
4383                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 1);
4384                                 amd64_alu_reg_reg (code, X86_ADD, ins->dreg, ins->dreg);
4385                                 break;
4386                         case 9:
4387                                 /* LEA r1, [r2 + r2*8] */
4388                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 3);
4389                                 break;
4390                         case 10:
4391                                 /* LEA r1, [r2 + r2*4] */
4392                                 /* ADD r1, r1          */
4393                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
4394                                 amd64_alu_reg_reg (code, X86_ADD, ins->dreg, ins->dreg);
4395                                 break;
4396                         case 12:
4397                                 /* LEA r1, [r2 + r2*2] */
4398                                 /* SHL r1, 2           */
4399                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 1);
4400                                 amd64_shift_reg_imm (code, X86_SHL, ins->dreg, 2);
4401                                 break;
4402                         case 25:
4403                                 /* LEA r1, [r2 + r2*4] */
4404                                 /* LEA r1, [r1 + r1*4] */
4405                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
4406                                 amd64_lea_memindex (code, ins->dreg, ins->dreg, 0, ins->dreg, 2);
4407                                 break;
4408                         case 100:
4409                                 /* LEA r1, [r2 + r2*4] */
4410                                 /* SHL r1, 2           */
4411                                 /* LEA r1, [r1 + r1*4] */
4412                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
4413                                 amd64_shift_reg_imm (code, X86_SHL, ins->dreg, 2);
4414                                 amd64_lea_memindex (code, ins->dreg, ins->dreg, 0, ins->dreg, 2);
4415                                 break;
4416                         default:
4417                                 amd64_imul_reg_reg_imm_size (code, ins->dreg, ins->sreg1, ins->inst_imm, size);
4418                                 break;
4419                         }
4420                         break;
4421                 }
4422                 case OP_LDIV:
4423                 case OP_LREM:
4424 #if defined( __native_client_codegen__ )
4425                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg2, 0);
4426                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_EQ, TRUE, "DivideByZeroException");
4427 #endif
4428                         /* Regalloc magic makes the div/rem cases the same */
4429                         if (ins->sreg2 == AMD64_RDX) {
4430                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
4431                                 amd64_cdq (code);
4432                                 amd64_div_membase (code, AMD64_RSP, -8, TRUE);
4433                         } else {
4434                                 amd64_cdq (code);
4435                                 amd64_div_reg (code, ins->sreg2, TRUE);
4436                         }
4437                         break;
4438                 case OP_LDIV_UN:
4439                 case OP_LREM_UN:
4440 #if defined( __native_client_codegen__ )
4441                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg2, 0);
4442                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_EQ, TRUE, "DivideByZeroException");
4443 #endif
4444                         if (ins->sreg2 == AMD64_RDX) {
4445                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
4446                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
4447                                 amd64_div_membase (code, AMD64_RSP, -8, FALSE);
4448                         } else {
4449                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
4450                                 amd64_div_reg (code, ins->sreg2, FALSE);
4451                         }
4452                         break;
4453                 case OP_IDIV:
4454                 case OP_IREM:
4455 #if defined( __native_client_codegen__ )
4456                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg2, 0);
4457                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_EQ, TRUE, "DivideByZeroException");
4458 #endif
4459                         if (ins->sreg2 == AMD64_RDX) {
4460                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
4461                                 amd64_cdq_size (code, 4);
4462                                 amd64_div_membase_size (code, AMD64_RSP, -8, TRUE, 4);
4463                         } else {
4464                                 amd64_cdq_size (code, 4);
4465                                 amd64_div_reg_size (code, ins->sreg2, TRUE, 4);
4466                         }
4467                         break;
4468                 case OP_IDIV_UN:
4469                 case OP_IREM_UN:
4470 #if defined( __native_client_codegen__ )
4471                         amd64_alu_reg_imm_size (code, X86_CMP, ins->sreg2, 0, 4);
4472                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_EQ, TRUE, "DivideByZeroException");
4473 #endif
4474                         if (ins->sreg2 == AMD64_RDX) {
4475                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
4476                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
4477                                 amd64_div_membase_size (code, AMD64_RSP, -8, FALSE, 4);
4478                         } else {
4479                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
4480                                 amd64_div_reg_size (code, ins->sreg2, FALSE, 4);
4481                         }
4482                         break;
4483                 case OP_IREM_IMM: {
4484                         int power = mono_is_power_of_two (ins->inst_imm);
4485
4486                         g_assert (ins->sreg1 == X86_EAX);
4487                         g_assert (ins->dreg == X86_EAX);
4488                         g_assert (power >= 0);
4489
4490                         if (power == 0) {
4491                                 amd64_mov_reg_imm (code, ins->dreg, 0);
4492                                 break;
4493                         }
4494
4495                         /* Based on gcc code */
4496
4497                         /* Add compensation for negative dividents */
4498                         amd64_mov_reg_reg_size (code, AMD64_RDX, AMD64_RAX, 4);
4499                         if (power > 1)
4500                                 amd64_shift_reg_imm_size (code, X86_SAR, AMD64_RDX, 31, 4);
4501                         amd64_shift_reg_imm_size (code, X86_SHR, AMD64_RDX, 32 - power, 4);
4502                         amd64_alu_reg_reg_size (code, X86_ADD, AMD64_RAX, AMD64_RDX, 4);
4503                         /* Compute remainder */
4504                         amd64_alu_reg_imm_size (code, X86_AND, AMD64_RAX, (1 << power) - 1, 4);
4505                         /* Remove compensation */
4506                         amd64_alu_reg_reg_size (code, X86_SUB, AMD64_RAX, AMD64_RDX, 4);
4507                         break;
4508                 }
4509                 case OP_LMUL_OVF:
4510                         amd64_imul_reg_reg (code, ins->sreg1, ins->sreg2);
4511                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
4512                         break;
4513                 case OP_LOR:
4514                         amd64_alu_reg_reg (code, X86_OR, ins->sreg1, ins->sreg2);
4515                         break;
4516                 case OP_OR_IMM:
4517                 case OP_LOR_IMM:
4518                         g_assert (amd64_is_imm32 (ins->inst_imm));
4519                         amd64_alu_reg_imm (code, X86_OR, ins->sreg1, ins->inst_imm);
4520                         break;
4521                 case OP_LXOR:
4522                         amd64_alu_reg_reg (code, X86_XOR, ins->sreg1, ins->sreg2);
4523                         break;
4524                 case OP_XOR_IMM:
4525                 case OP_LXOR_IMM:
4526                         g_assert (amd64_is_imm32 (ins->inst_imm));
4527                         amd64_alu_reg_imm (code, X86_XOR, ins->sreg1, ins->inst_imm);
4528                         break;
4529                 case OP_LSHL:
4530                         g_assert (ins->sreg2 == AMD64_RCX);
4531                         amd64_shift_reg (code, X86_SHL, ins->dreg);
4532                         break;
4533                 case OP_LSHR:
4534                         g_assert (ins->sreg2 == AMD64_RCX);
4535                         amd64_shift_reg (code, X86_SAR, ins->dreg);
4536                         break;
4537                 case OP_SHR_IMM:
4538                         g_assert (amd64_is_imm32 (ins->inst_imm));
4539                         amd64_shift_reg_imm_size (code, X86_SAR, ins->dreg, ins->inst_imm, 4);
4540                         break;
4541                 case OP_LSHR_IMM:
4542                         g_assert (amd64_is_imm32 (ins->inst_imm));
4543                         amd64_shift_reg_imm (code, X86_SAR, ins->dreg, ins->inst_imm);
4544                         break;
4545                 case OP_SHR_UN_IMM:
4546                         g_assert (amd64_is_imm32 (ins->inst_imm));
4547                         amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, ins->inst_imm, 4);
4548                         break;
4549                 case OP_LSHR_UN_IMM:
4550                         g_assert (amd64_is_imm32 (ins->inst_imm));
4551                         amd64_shift_reg_imm (code, X86_SHR, ins->dreg, ins->inst_imm);
4552                         break;
4553                 case OP_LSHR_UN:
4554                         g_assert (ins->sreg2 == AMD64_RCX);
4555                         amd64_shift_reg (code, X86_SHR, ins->dreg);
4556                         break;
4557                 case OP_SHL_IMM:
4558                         g_assert (amd64_is_imm32 (ins->inst_imm));
4559                         amd64_shift_reg_imm_size (code, X86_SHL, ins->dreg, ins->inst_imm, 4);
4560                         break;
4561                 case OP_LSHL_IMM:
4562                         g_assert (amd64_is_imm32 (ins->inst_imm));
4563                         amd64_shift_reg_imm (code, X86_SHL, ins->dreg, ins->inst_imm);
4564                         break;
4565
4566                 case OP_IADDCC:
4567                 case OP_IADD:
4568                         amd64_alu_reg_reg_size (code, X86_ADD, ins->sreg1, ins->sreg2, 4);
4569                         break;
4570                 case OP_IADC:
4571                         amd64_alu_reg_reg_size (code, X86_ADC, ins->sreg1, ins->sreg2, 4);
4572                         break;
4573                 case OP_IADD_IMM:
4574                         amd64_alu_reg_imm_size (code, X86_ADD, ins->dreg, ins->inst_imm, 4);
4575                         break;
4576                 case OP_IADC_IMM:
4577                         amd64_alu_reg_imm_size (code, X86_ADC, ins->dreg, ins->inst_imm, 4);
4578                         break;
4579                 case OP_ISUBCC:
4580                 case OP_ISUB:
4581                         amd64_alu_reg_reg_size (code, X86_SUB, ins->sreg1, ins->sreg2, 4);
4582                         break;
4583                 case OP_ISBB:
4584                         amd64_alu_reg_reg_size (code, X86_SBB, ins->sreg1, ins->sreg2, 4);
4585                         break;
4586                 case OP_ISUB_IMM:
4587                         amd64_alu_reg_imm_size (code, X86_SUB, ins->dreg, ins->inst_imm, 4);
4588                         break;
4589                 case OP_ISBB_IMM:
4590                         amd64_alu_reg_imm_size (code, X86_SBB, ins->dreg, ins->inst_imm, 4);
4591                         break;
4592                 case OP_IAND:
4593                         amd64_alu_reg_reg_size (code, X86_AND, ins->sreg1, ins->sreg2, 4);
4594                         break;
4595                 case OP_IAND_IMM:
4596                         amd64_alu_reg_imm_size (code, X86_AND, ins->sreg1, ins->inst_imm, 4);
4597                         break;
4598                 case OP_IOR:
4599                         amd64_alu_reg_reg_size (code, X86_OR, ins->sreg1, ins->sreg2, 4);
4600                         break;
4601                 case OP_IOR_IMM:
4602                         amd64_alu_reg_imm_size (code, X86_OR, ins->sreg1, ins->inst_imm, 4);
4603                         break;
4604                 case OP_IXOR:
4605                         amd64_alu_reg_reg_size (code, X86_XOR, ins->sreg1, ins->sreg2, 4);
4606                         break;
4607                 case OP_IXOR_IMM:
4608                         amd64_alu_reg_imm_size (code, X86_XOR, ins->sreg1, ins->inst_imm, 4);
4609                         break;
4610                 case OP_INEG:
4611                         amd64_neg_reg_size (code, ins->sreg1, 4);
4612                         break;
4613                 case OP_INOT:
4614                         amd64_not_reg_size (code, ins->sreg1, 4);
4615                         break;
4616                 case OP_ISHL:
4617                         g_assert (ins->sreg2 == AMD64_RCX);
4618                         amd64_shift_reg_size (code, X86_SHL, ins->dreg, 4);
4619                         break;
4620                 case OP_ISHR:
4621                         g_assert (ins->sreg2 == AMD64_RCX);
4622                         amd64_shift_reg_size (code, X86_SAR, ins->dreg, 4);
4623                         break;
4624                 case OP_ISHR_IMM:
4625                         amd64_shift_reg_imm_size (code, X86_SAR, ins->dreg, ins->inst_imm, 4);
4626                         break;
4627                 case OP_ISHR_UN_IMM:
4628                         amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, ins->inst_imm, 4);
4629                         break;
4630                 case OP_ISHR_UN:
4631                         g_assert (ins->sreg2 == AMD64_RCX);
4632                         amd64_shift_reg_size (code, X86_SHR, ins->dreg, 4);
4633                         break;
4634                 case OP_ISHL_IMM:
4635                         amd64_shift_reg_imm_size (code, X86_SHL, ins->dreg, ins->inst_imm, 4);
4636                         break;
4637                 case OP_IMUL:
4638                         amd64_imul_reg_reg_size (code, ins->sreg1, ins->sreg2, 4);
4639                         break;
4640                 case OP_IMUL_OVF:
4641                         amd64_imul_reg_reg_size (code, ins->sreg1, ins->sreg2, 4);
4642                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
4643                         break;
4644                 case OP_IMUL_OVF_UN:
4645                 case OP_LMUL_OVF_UN: {
4646                         /* the mul operation and the exception check should most likely be split */
4647                         int non_eax_reg, saved_eax = FALSE, saved_edx = FALSE;
4648                         int size = (ins->opcode == OP_IMUL_OVF_UN) ? 4 : 8;
4649                         /*g_assert (ins->sreg2 == X86_EAX);
4650                         g_assert (ins->dreg == X86_EAX);*/
4651                         if (ins->sreg2 == X86_EAX) {
4652                                 non_eax_reg = ins->sreg1;
4653                         } else if (ins->sreg1 == X86_EAX) {
4654                                 non_eax_reg = ins->sreg2;
4655                         } else {
4656                                 /* no need to save since we're going to store to it anyway */
4657                                 if (ins->dreg != X86_EAX) {
4658                                         saved_eax = TRUE;
4659                                         amd64_push_reg (code, X86_EAX);
4660                                 }
4661                                 amd64_mov_reg_reg (code, X86_EAX, ins->sreg1, size);
4662                                 non_eax_reg = ins->sreg2;
4663                         }
4664                         if (ins->dreg == X86_EDX) {
4665                                 if (!saved_eax) {
4666                                         saved_eax = TRUE;
4667                                         amd64_push_reg (code, X86_EAX);
4668                                 }
4669                         } else {
4670                                 saved_edx = TRUE;
4671                                 amd64_push_reg (code, X86_EDX);
4672                         }
4673                         amd64_mul_reg_size (code, non_eax_reg, FALSE, size);
4674                         /* save before the check since pop and mov don't change the flags */
4675                         if (ins->dreg != X86_EAX)
4676                                 amd64_mov_reg_reg (code, ins->dreg, X86_EAX, size);
4677                         if (saved_edx)
4678                                 amd64_pop_reg (code, X86_EDX);
4679                         if (saved_eax)
4680                                 amd64_pop_reg (code, X86_EAX);
4681                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
4682                         break;
4683                 }
4684                 case OP_ICOMPARE:
4685                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
4686                         break;
4687                 case OP_ICOMPARE_IMM:
4688                         amd64_alu_reg_imm_size (code, X86_CMP, ins->sreg1, ins->inst_imm, 4);
4689                         break;
4690                 case OP_IBEQ:
4691                 case OP_IBLT:
4692                 case OP_IBGT:
4693                 case OP_IBGE:
4694                 case OP_IBLE:
4695                 case OP_LBEQ:
4696                 case OP_LBLT:
4697                 case OP_LBGT:
4698                 case OP_LBGE:
4699                 case OP_LBLE:
4700                 case OP_IBNE_UN:
4701                 case OP_IBLT_UN:
4702                 case OP_IBGT_UN:
4703                 case OP_IBGE_UN:
4704                 case OP_IBLE_UN:
4705                 case OP_LBNE_UN:
4706                 case OP_LBLT_UN:
4707                 case OP_LBGT_UN:
4708                 case OP_LBGE_UN:
4709                 case OP_LBLE_UN:
4710                         EMIT_COND_BRANCH (ins, cc_table [mono_opcode_to_cond (ins->opcode)], cc_signed_table [mono_opcode_to_cond (ins->opcode)]);
4711                         break;
4712
4713                 case OP_CMOV_IEQ:
4714                 case OP_CMOV_IGE:
4715                 case OP_CMOV_IGT:
4716                 case OP_CMOV_ILE:
4717                 case OP_CMOV_ILT:
4718                 case OP_CMOV_INE_UN:
4719                 case OP_CMOV_IGE_UN:
4720                 case OP_CMOV_IGT_UN:
4721                 case OP_CMOV_ILE_UN:
4722                 case OP_CMOV_ILT_UN:
4723                 case OP_CMOV_LEQ:
4724                 case OP_CMOV_LGE:
4725                 case OP_CMOV_LGT:
4726                 case OP_CMOV_LLE:
4727                 case OP_CMOV_LLT:
4728                 case OP_CMOV_LNE_UN:
4729                 case OP_CMOV_LGE_UN:
4730                 case OP_CMOV_LGT_UN:
4731                 case OP_CMOV_LLE_UN:
4732                 case OP_CMOV_LLT_UN:
4733                         g_assert (ins->dreg == ins->sreg1);
4734                         /* This needs to operate on 64 bit values */
4735                         amd64_cmov_reg (code, cc_table [mono_opcode_to_cond (ins->opcode)], cc_signed_table [mono_opcode_to_cond (ins->opcode)], ins->dreg, ins->sreg2);
4736                         break;
4737
4738                 case OP_LNOT:
4739                         amd64_not_reg (code, ins->sreg1);
4740                         break;
4741                 case OP_LNEG:
4742                         amd64_neg_reg (code, ins->sreg1);
4743                         break;
4744
4745                 case OP_ICONST:
4746                 case OP_I8CONST:
4747                         if ((((guint64)ins->inst_c0) >> 32) == 0)
4748                                 amd64_mov_reg_imm_size (code, ins->dreg, ins->inst_c0, 4);
4749                         else
4750                                 amd64_mov_reg_imm_size (code, ins->dreg, ins->inst_c0, 8);
4751                         break;
4752                 case OP_AOTCONST:
4753                         mono_add_patch_info (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
4754                         amd64_mov_reg_membase (code, ins->dreg, AMD64_RIP, 0, sizeof(gpointer));
4755                         break;
4756                 case OP_JUMP_TABLE:
4757                         mono_add_patch_info (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
4758                         amd64_mov_reg_imm_size (code, ins->dreg, 0, 8);
4759                         break;
4760                 case OP_MOVE:
4761                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, sizeof(mgreg_t));
4762                         break;
4763                 case OP_AMD64_SET_XMMREG_R4: {
4764                         amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg1);
4765                         break;
4766                 }
4767                 case OP_AMD64_SET_XMMREG_R8: {
4768                         if (ins->dreg != ins->sreg1)
4769                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
4770                         break;
4771                 }
4772                 case OP_TAILCALL: {
4773                         MonoCallInst *call = (MonoCallInst*)ins;
4774                         int pos = 0, i;
4775
4776                         /* FIXME: no tracing support... */
4777                         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
4778                                 code = mono_arch_instrument_epilog_full (cfg, mono_profiler_method_leave, code, FALSE, TRUE);
4779
4780                         g_assert (!cfg->method->save_lmf);
4781
4782                         if (cfg->arch.omit_fp) {
4783                                 guint32 save_offset = 0;
4784                                 /* Pop callee-saved registers */
4785                                 for (i = 0; i < AMD64_NREG; ++i)
4786                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
4787                                                 amd64_mov_reg_membase (code, i, AMD64_RSP, save_offset, 8);
4788                                                 save_offset += 8;
4789                                         }
4790                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, cfg->arch.stack_alloc_size);
4791
4792                                 // FIXME:
4793                                 if (call->stack_usage)
4794                                         NOT_IMPLEMENTED;
4795                         }
4796                         else {
4797                                 for (i = 0; i < AMD64_NREG; ++i)
4798                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i)))
4799                                                 pos -= sizeof(mgreg_t);
4800
4801                                 /* Restore callee-saved registers */
4802                                 for (i = AMD64_NREG - 1; i > 0; --i) {
4803                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
4804                                                 amd64_mov_reg_membase (code, i, AMD64_RBP, pos, sizeof(mgreg_t));
4805                                                 pos += sizeof(mgreg_t);
4806                                         }
4807                                 }
4808
4809                                 /* Copy arguments on the stack to our argument area */
4810                                 for (i = 0; i < call->stack_usage; i += sizeof(mgreg_t)) {
4811                                         amd64_mov_reg_membase (code, AMD64_RAX, AMD64_RSP, i, sizeof(mgreg_t));
4812                                         amd64_mov_membase_reg (code, AMD64_RBP, 16 + i, AMD64_RAX, sizeof(mgreg_t));
4813                                 }
4814
4815                                 if (pos)
4816                                         amd64_lea_membase (code, AMD64_RSP, AMD64_RBP, pos);
4817
4818                                 amd64_leave (code);
4819                         }
4820
4821                         offset = code - cfg->native_code;
4822                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_METHOD_JUMP, ins->inst_p0);
4823                         if (cfg->compile_aot)
4824                                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_RIP, 0, 8);
4825                         else
4826                                 amd64_set_reg_template (code, AMD64_R11);
4827                         amd64_jump_reg (code, AMD64_R11);
4828                         ins->flags |= MONO_INST_GC_CALLSITE;
4829                         ins->backend.pc_offset = code - cfg->native_code;
4830                         break;
4831                 }
4832                 case OP_CHECK_THIS:
4833                         /* ensure ins->sreg1 is not NULL */
4834                         amd64_alu_membase_imm_size (code, X86_CMP, ins->sreg1, 0, 0, 4);
4835                         break;
4836                 case OP_ARGLIST: {
4837                         amd64_lea_membase (code, AMD64_R11, cfg->frame_reg, cfg->sig_cookie);
4838                         amd64_mov_membase_reg (code, ins->sreg1, 0, AMD64_R11, sizeof(gpointer));
4839                         break;
4840                 }
4841                 case OP_CALL:
4842                 case OP_FCALL:
4843                 case OP_LCALL:
4844                 case OP_VCALL:
4845                 case OP_VCALL2:
4846                 case OP_VOIDCALL:
4847                         call = (MonoCallInst*)ins;
4848                         /*
4849                          * The AMD64 ABI forces callers to know about varargs.
4850                          */
4851                         if ((call->signature->call_convention == MONO_CALL_VARARG) && (call->signature->pinvoke))
4852                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
4853                         else if ((cfg->method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) && (cfg->method->klass->image != mono_defaults.corlib)) {
4854                                 /*
4855                                  * Since the unmanaged calling convention doesn't contain a
4856                                  * 'vararg' entry, we have to treat every pinvoke call as a
4857                                  * potential vararg call.
4858                                  */
4859                                 guint32 nregs, i;
4860                                 nregs = 0;
4861                                 for (i = 0; i < AMD64_XMM_NREG; ++i)
4862                                         if (call->used_fregs & (1 << i))
4863                                                 nregs ++;
4864                                 if (!nregs)
4865                                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
4866                                 else
4867                                         amd64_mov_reg_imm (code, AMD64_RAX, nregs);
4868                         }
4869
4870                         if (ins->flags & MONO_INST_HAS_METHOD)
4871                                 code = emit_call (cfg, code, MONO_PATCH_INFO_METHOD, call->method, FALSE);
4872                         else
4873                                 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, call->fptr, FALSE);
4874                         ins->flags |= MONO_INST_GC_CALLSITE;
4875                         ins->backend.pc_offset = code - cfg->native_code;
4876                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention) && !cfg->arch.no_pushes)
4877                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
4878                         code = emit_move_return_value (cfg, ins, code);
4879                         break;
4880                 case OP_FCALL_REG:
4881                 case OP_LCALL_REG:
4882                 case OP_VCALL_REG:
4883                 case OP_VCALL2_REG:
4884                 case OP_VOIDCALL_REG:
4885                 case OP_CALL_REG:
4886                         call = (MonoCallInst*)ins;
4887
4888                         if (AMD64_IS_ARGUMENT_REG (ins->sreg1)) {
4889                                 amd64_mov_reg_reg (code, AMD64_R11, ins->sreg1, 8);
4890                                 ins->sreg1 = AMD64_R11;
4891                         }
4892
4893                         /*
4894                          * The AMD64 ABI forces callers to know about varargs.
4895                          */
4896                         if ((call->signature->call_convention == MONO_CALL_VARARG) && (call->signature->pinvoke)) {
4897                                 if (ins->sreg1 == AMD64_RAX) {
4898                                         amd64_mov_reg_reg (code, AMD64_R11, AMD64_RAX, 8);
4899                                         ins->sreg1 = AMD64_R11;
4900                                 }
4901                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
4902                         } else if ((cfg->method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) && (cfg->method->klass->image != mono_defaults.corlib)) {
4903                                 /*
4904                                  * Since the unmanaged calling convention doesn't contain a
4905                                  * 'vararg' entry, we have to treat every pinvoke call as a
4906                                  * potential vararg call.
4907                                  */
4908                                 guint32 nregs, i;
4909                                 nregs = 0;
4910                                 for (i = 0; i < AMD64_XMM_NREG; ++i)
4911                                         if (call->used_fregs & (1 << i))
4912                                                 nregs ++;
4913                                 if (ins->sreg1 == AMD64_RAX) {
4914                                         amd64_mov_reg_reg (code, AMD64_R11, AMD64_RAX, 8);
4915                                         ins->sreg1 = AMD64_R11;
4916                                 }
4917                                 if (!nregs)
4918                                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
4919                                 else
4920                                         amd64_mov_reg_imm (code, AMD64_RAX, nregs);
4921                         }
4922
4923                         amd64_call_reg (code, ins->sreg1);
4924                         ins->flags |= MONO_INST_GC_CALLSITE;
4925                         ins->backend.pc_offset = code - cfg->native_code;
4926                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention) && !cfg->arch.no_pushes)
4927                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
4928                         code = emit_move_return_value (cfg, ins, code);
4929                         break;
4930                 case OP_FCALL_MEMBASE:
4931                 case OP_LCALL_MEMBASE:
4932                 case OP_VCALL_MEMBASE:
4933                 case OP_VCALL2_MEMBASE:
4934                 case OP_VOIDCALL_MEMBASE:
4935                 case OP_CALL_MEMBASE:
4936                         call = (MonoCallInst*)ins;
4937
4938                         amd64_call_membase (code, ins->sreg1, ins->inst_offset);
4939                         ins->flags |= MONO_INST_GC_CALLSITE;
4940                         ins->backend.pc_offset = code - cfg->native_code;
4941                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention) && !cfg->arch.no_pushes)
4942                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
4943                         code = emit_move_return_value (cfg, ins, code);
4944                         break;
4945                 case OP_DYN_CALL: {
4946                         int i;
4947                         MonoInst *var = cfg->dyn_call_var;
4948
4949                         g_assert (var->opcode == OP_REGOFFSET);
4950
4951                         /* r11 = args buffer filled by mono_arch_get_dyn_call_args () */
4952                         amd64_mov_reg_reg (code, AMD64_R11, ins->sreg1, 8);
4953                         /* r10 = ftn */
4954                         amd64_mov_reg_reg (code, AMD64_R10, ins->sreg2, 8);
4955
4956                         /* Save args buffer */
4957                         amd64_mov_membase_reg (code, var->inst_basereg, var->inst_offset, AMD64_R11, 8);
4958
4959                         /* Set argument registers */
4960                         for (i = 0; i < PARAM_REGS; ++i)
4961                                 amd64_mov_reg_membase (code, param_regs [i], AMD64_R11, i * sizeof(mgreg_t), sizeof(mgreg_t));
4962
4963                         /* Make the call */
4964                         amd64_call_reg (code, AMD64_R10);
4965
4966                         ins->flags |= MONO_INST_GC_CALLSITE;
4967                         ins->backend.pc_offset = code - cfg->native_code;
4968
4969                         /* Save result */
4970                         amd64_mov_reg_membase (code, AMD64_R11, var->inst_basereg, var->inst_offset, 8);
4971                         amd64_mov_membase_reg (code, AMD64_R11, G_STRUCT_OFFSET (DynCallArgs, res), AMD64_RAX, 8);
4972                         break;
4973                 }
4974                 case OP_AMD64_SAVE_SP_TO_LMF: {
4975                         MonoInst *lmf_var = cfg->arch.lmf_var;
4976                         amd64_mov_membase_reg (code, cfg->frame_reg, lmf_var->inst_offset + G_STRUCT_OFFSET (MonoLMF, rsp), AMD64_RSP, 8);
4977                         break;
4978                 }
4979                 case OP_X86_PUSH:
4980                         g_assert (!cfg->arch.no_pushes);
4981                         amd64_push_reg (code, ins->sreg1);
4982                         break;
4983                 case OP_X86_PUSH_IMM:
4984                         g_assert (!cfg->arch.no_pushes);
4985                         g_assert (amd64_is_imm32 (ins->inst_imm));
4986                         amd64_push_imm (code, ins->inst_imm);
4987                         break;
4988                 case OP_X86_PUSH_MEMBASE:
4989                         g_assert (!cfg->arch.no_pushes);
4990                         amd64_push_membase (code, ins->inst_basereg, ins->inst_offset);
4991                         break;
4992                 case OP_X86_PUSH_OBJ: {
4993                         int size = ALIGN_TO (ins->inst_imm, 8);
4994
4995                         g_assert (!cfg->arch.no_pushes);
4996
4997                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, size);
4998                         amd64_push_reg (code, AMD64_RDI);
4999                         amd64_push_reg (code, AMD64_RSI);
5000                         amd64_push_reg (code, AMD64_RCX);
5001                         if (ins->inst_offset)
5002                                 amd64_lea_membase (code, AMD64_RSI, ins->inst_basereg, ins->inst_offset);
5003                         else
5004                                 amd64_mov_reg_reg (code, AMD64_RSI, ins->inst_basereg, 8);
5005                         amd64_lea_membase (code, AMD64_RDI, AMD64_RSP, (3 * 8));
5006                         amd64_mov_reg_imm (code, AMD64_RCX, (size >> 3));
5007                         amd64_cld (code);
5008                         amd64_prefix (code, X86_REP_PREFIX);
5009                         amd64_movsd (code);
5010                         amd64_pop_reg (code, AMD64_RCX);
5011                         amd64_pop_reg (code, AMD64_RSI);
5012                         amd64_pop_reg (code, AMD64_RDI);
5013                         break;
5014                 }
5015                 case OP_X86_LEA:
5016                         amd64_lea_memindex (code, ins->dreg, ins->sreg1, ins->inst_imm, ins->sreg2, ins->backend.shift_amount);
5017                         break;
5018                 case OP_X86_LEA_MEMBASE:
5019                         amd64_lea_membase (code, ins->dreg, ins->sreg1, ins->inst_imm);
5020                         break;
5021                 case OP_X86_XCHG:
5022                         amd64_xchg_reg_reg (code, ins->sreg1, ins->sreg2, 4);
5023                         break;
5024                 case OP_LOCALLOC:
5025                         /* keep alignment */
5026                         amd64_alu_reg_imm (code, X86_ADD, ins->sreg1, MONO_ARCH_FRAME_ALIGNMENT - 1);
5027                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ~(MONO_ARCH_FRAME_ALIGNMENT - 1));
5028                         code = mono_emit_stack_alloc (cfg, code, ins);
5029                         amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
5030                         if (cfg->param_area && cfg->arch.no_pushes)
5031                                 amd64_alu_reg_imm (code, X86_ADD, ins->dreg, cfg->param_area);
5032                         break;
5033                 case OP_LOCALLOC_IMM: {
5034                         guint32 size = ins->inst_imm;
5035                         size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
5036
5037                         if (ins->flags & MONO_INST_INIT) {
5038                                 if (size < 64) {
5039                                         int i;
5040
5041                                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, size);
5042                                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
5043
5044                                         for (i = 0; i < size; i += 8)
5045                                                 amd64_mov_membase_reg (code, AMD64_RSP, i, ins->dreg, 8);
5046                                         amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
5047                                 } else {
5048                                         amd64_mov_reg_imm (code, ins->dreg, size);
5049                                         ins->sreg1 = ins->dreg;
5050
5051                                         code = mono_emit_stack_alloc (cfg, code, ins);
5052                                         amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
5053                                 }
5054                         } else {
5055                                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, size);
5056                                 amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
5057                         }
5058                         if (cfg->param_area && cfg->arch.no_pushes)
5059                                 amd64_alu_reg_imm (code, X86_ADD, ins->dreg, cfg->param_area);
5060                         break;
5061                 }
5062                 case OP_THROW: {
5063                         amd64_mov_reg_reg (code, AMD64_ARG_REG1, ins->sreg1, 8);
5064                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
5065                                              (gpointer)"mono_arch_throw_exception", FALSE);
5066                         ins->flags |= MONO_INST_GC_CALLSITE;
5067                         ins->backend.pc_offset = code - cfg->native_code;
5068                         break;
5069                 }
5070                 case OP_RETHROW: {
5071                         amd64_mov_reg_reg (code, AMD64_ARG_REG1, ins->sreg1, 8);
5072                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
5073                                              (gpointer)"mono_arch_rethrow_exception", FALSE);
5074                         ins->flags |= MONO_INST_GC_CALLSITE;
5075                         ins->backend.pc_offset = code - cfg->native_code;
5076                         break;
5077                 }
5078                 case OP_CALL_HANDLER:
5079                         /* Align stack */
5080                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
5081                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_BB, ins->inst_target_bb);
5082                         amd64_call_imm (code, 0);
5083                         mono_cfg_add_try_hole (cfg, ins->inst_eh_block, code, bb);
5084                         /* Restore stack alignment */
5085                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
5086                         break;
5087                 case OP_START_HANDLER: {
5088                         /* Even though we're saving RSP, use sizeof */
5089                         /* gpointer because spvar is of type IntPtr */
5090                         /* see: mono_create_spvar_for_region */
5091                         MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
5092                         amd64_mov_membase_reg (code, spvar->inst_basereg, spvar->inst_offset, AMD64_RSP, sizeof(gpointer));
5093
5094                         if ((MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_FINALLY) ||
5095                                  MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_FINALLY)) &&
5096                                 cfg->param_area && cfg->arch.no_pushes) {
5097                                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, ALIGN_TO (cfg->param_area, MONO_ARCH_FRAME_ALIGNMENT));
5098                         }
5099                         break;
5100                 }
5101                 case OP_ENDFINALLY: {
5102                         MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
5103                         amd64_mov_reg_membase (code, AMD64_RSP, spvar->inst_basereg, spvar->inst_offset, sizeof(gpointer));
5104                         amd64_ret (code);
5105                         break;
5106                 }
5107                 case OP_ENDFILTER: {
5108                         MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
5109                         amd64_mov_reg_membase (code, AMD64_RSP, spvar->inst_basereg, spvar->inst_offset, sizeof(gpointer));
5110                         /* The local allocator will put the result into RAX */
5111                         amd64_ret (code);
5112                         break;
5113                 }
5114
5115                 case OP_LABEL:
5116                         ins->inst_c0 = code - cfg->native_code;
5117                         break;
5118                 case OP_BR:
5119                         //g_print ("target: %p, next: %p, curr: %p, last: %p\n", ins->inst_target_bb, bb->next_bb, ins, bb->last_ins);
5120                         //if ((ins->inst_target_bb == bb->next_bb) && ins == bb->last_ins)
5121                         //break;
5122                                 if (ins->inst_target_bb->native_offset) {
5123                                         amd64_jump_code (code, cfg->native_code + ins->inst_target_bb->native_offset);
5124                                 } else {
5125                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb);
5126                                         if ((cfg->opt & MONO_OPT_BRANCH) &&
5127                                             x86_is_imm8 (ins->inst_target_bb->max_offset - offset))
5128                                                 x86_jump8 (code, 0);
5129                                         else
5130                                                 x86_jump32 (code, 0);
5131                         }
5132                         break;
5133                 case OP_BR_REG:
5134                         amd64_jump_reg (code, ins->sreg1);
5135                         break;
5136                 case OP_CEQ:
5137                 case OP_LCEQ:
5138                 case OP_ICEQ:
5139                 case OP_CLT:
5140                 case OP_LCLT:
5141                 case OP_ICLT:
5142                 case OP_CGT:
5143                 case OP_ICGT:
5144                 case OP_LCGT:
5145                 case OP_CLT_UN:
5146                 case OP_LCLT_UN:
5147                 case OP_ICLT_UN:
5148                 case OP_CGT_UN:
5149                 case OP_LCGT_UN:
5150                 case OP_ICGT_UN:
5151                         amd64_set_reg (code, cc_table [mono_opcode_to_cond (ins->opcode)], ins->dreg, cc_signed_table [mono_opcode_to_cond (ins->opcode)]);
5152                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
5153                         break;
5154                 case OP_COND_EXC_EQ:
5155                 case OP_COND_EXC_NE_UN:
5156                 case OP_COND_EXC_LT:
5157                 case OP_COND_EXC_LT_UN:
5158                 case OP_COND_EXC_GT:
5159                 case OP_COND_EXC_GT_UN:
5160                 case OP_COND_EXC_GE:
5161                 case OP_COND_EXC_GE_UN:
5162                 case OP_COND_EXC_LE:
5163                 case OP_COND_EXC_LE_UN:
5164                 case OP_COND_EXC_IEQ:
5165                 case OP_COND_EXC_INE_UN:
5166                 case OP_COND_EXC_ILT:
5167                 case OP_COND_EXC_ILT_UN:
5168                 case OP_COND_EXC_IGT:
5169                 case OP_COND_EXC_IGT_UN:
5170                 case OP_COND_EXC_IGE:
5171                 case OP_COND_EXC_IGE_UN:
5172                 case OP_COND_EXC_ILE:
5173                 case OP_COND_EXC_ILE_UN:
5174                         EMIT_COND_SYSTEM_EXCEPTION (cc_table [mono_opcode_to_cond (ins->opcode)], cc_signed_table [mono_opcode_to_cond (ins->opcode)], ins->inst_p1);
5175                         break;
5176                 case OP_COND_EXC_OV:
5177                 case OP_COND_EXC_NO:
5178                 case OP_COND_EXC_C:
5179                 case OP_COND_EXC_NC:
5180                         EMIT_COND_SYSTEM_EXCEPTION (branch_cc_table [ins->opcode - OP_COND_EXC_EQ],
5181                                                     (ins->opcode < OP_COND_EXC_NE_UN), ins->inst_p1);
5182                         break;
5183                 case OP_COND_EXC_IOV:
5184                 case OP_COND_EXC_INO:
5185                 case OP_COND_EXC_IC:
5186                 case OP_COND_EXC_INC:
5187                         EMIT_COND_SYSTEM_EXCEPTION (branch_cc_table [ins->opcode - OP_COND_EXC_IEQ],
5188                                                     (ins->opcode < OP_COND_EXC_INE_UN), ins->inst_p1);
5189                         break;
5190
5191                 /* floating point opcodes */
5192                 case OP_R8CONST: {
5193                         double d = *(double *)ins->inst_p0;
5194
5195                         if ((d == 0.0) && (mono_signbit (d) == 0)) {
5196                                 amd64_sse_xorpd_reg_reg (code, ins->dreg, ins->dreg);
5197                         }
5198                         else {
5199                                 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, ins->inst_p0);
5200                                 amd64_sse_movsd_reg_membase (code, ins->dreg, AMD64_RIP, 0);
5201                         }
5202                         break;
5203                 }
5204                 case OP_R4CONST: {
5205                         float f = *(float *)ins->inst_p0;
5206
5207                         if ((f == 0.0) && (mono_signbit (f) == 0)) {
5208                                 amd64_sse_xorpd_reg_reg (code, ins->dreg, ins->dreg);
5209                         }
5210                         else {
5211                                 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R4, ins->inst_p0);
5212                                 amd64_sse_movss_reg_membase (code, ins->dreg, AMD64_RIP, 0);
5213                                 amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
5214                         }
5215                         break;
5216                 }
5217                 case OP_STORER8_MEMBASE_REG:
5218                         amd64_sse_movsd_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1);
5219                         break;
5220                 case OP_LOADR8_MEMBASE:
5221                         amd64_sse_movsd_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
5222                         break;
5223                 case OP_STORER4_MEMBASE_REG:
5224                         /* This requires a double->single conversion */
5225                         amd64_sse_cvtsd2ss_reg_reg (code, AMD64_XMM15, ins->sreg1);
5226                         amd64_sse_movss_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, AMD64_XMM15);
5227                         break;
5228                 case OP_LOADR4_MEMBASE:
5229                         amd64_sse_movss_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
5230                         amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
5231                         break;
5232                 case OP_ICONV_TO_R4: /* FIXME: change precision */
5233                 case OP_ICONV_TO_R8:
5234                         amd64_sse_cvtsi2sd_reg_reg_size (code, ins->dreg, ins->sreg1, 4);
5235                         break;
5236                 case OP_LCONV_TO_R4: /* FIXME: change precision */
5237                 case OP_LCONV_TO_R8:
5238                         amd64_sse_cvtsi2sd_reg_reg (code, ins->dreg, ins->sreg1);
5239                         break;
5240                 case OP_FCONV_TO_R4:
5241                         /* FIXME: nothing to do ?? */
5242                         break;
5243                 case OP_FCONV_TO_I1:
5244                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 1, TRUE);
5245                         break;
5246                 case OP_FCONV_TO_U1:
5247                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 1, FALSE);
5248                         break;
5249                 case OP_FCONV_TO_I2:
5250                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 2, TRUE);
5251                         break;
5252                 case OP_FCONV_TO_U2:
5253                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 2, FALSE);
5254                         break;
5255                 case OP_FCONV_TO_U4:
5256                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 4, FALSE);
5257                         break;
5258                 case OP_FCONV_TO_I4:
5259                 case OP_FCONV_TO_I:
5260                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 4, TRUE);
5261                         break;
5262                 case OP_FCONV_TO_I8:
5263                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 8, TRUE);
5264                         break;
5265                 case OP_LCONV_TO_R_UN: {
5266                         guint8 *br [2];
5267
5268                         /* Based on gcc code */
5269                         amd64_test_reg_reg (code, ins->sreg1, ins->sreg1);
5270                         br [0] = code; x86_branch8 (code, X86_CC_S, 0, TRUE);
5271
5272                         /* Positive case */
5273                         amd64_sse_cvtsi2sd_reg_reg (code, ins->dreg, ins->sreg1);
5274                         br [1] = code; x86_jump8 (code, 0);
5275                         amd64_patch (br [0], code);
5276
5277                         /* Negative case */
5278                         /* Save to the red zone */
5279                         amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RAX, 8);
5280                         amd64_mov_membase_reg (code, AMD64_RSP, -16, AMD64_RCX, 8);
5281                         amd64_mov_reg_reg (code, AMD64_RCX, ins->sreg1, 8);
5282                         amd64_mov_reg_reg (code, AMD64_RAX, ins->sreg1, 8);
5283                         amd64_alu_reg_imm (code, X86_AND, AMD64_RCX, 1);
5284                         amd64_shift_reg_imm (code, X86_SHR, AMD64_RAX, 1);
5285                         amd64_alu_reg_imm (code, X86_OR, AMD64_RAX, AMD64_RCX);
5286                         amd64_sse_cvtsi2sd_reg_reg (code, ins->dreg, AMD64_RAX);
5287                         amd64_sse_addsd_reg_reg (code, ins->dreg, ins->dreg);
5288                         /* Restore */
5289                         amd64_mov_reg_membase (code, AMD64_RCX, AMD64_RSP, -16, 8);
5290                         amd64_mov_reg_membase (code, AMD64_RAX, AMD64_RSP, -8, 8);
5291                         amd64_patch (br [1], code);
5292                         break;
5293                 }
5294                 case OP_LCONV_TO_OVF_U4:
5295                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, 0);
5296                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_LT, TRUE, "OverflowException");
5297                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, 8);
5298                         break;
5299                 case OP_LCONV_TO_OVF_I4_UN:
5300                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, 0x7fffffff);
5301                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_GT, FALSE, "OverflowException");
5302                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, 8);
5303                         break;
5304                 case OP_FMOVE:
5305                         if (ins->dreg != ins->sreg1)
5306                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
5307                         break;
5308                 case OP_FADD:
5309                         amd64_sse_addsd_reg_reg (code, ins->dreg, ins->sreg2);
5310                         break;
5311                 case OP_FSUB:
5312                         amd64_sse_subsd_reg_reg (code, ins->dreg, ins->sreg2);
5313                         break;
5314                 case OP_FMUL:
5315                         amd64_sse_mulsd_reg_reg (code, ins->dreg, ins->sreg2);
5316                         break;
5317                 case OP_FDIV:
5318                         amd64_sse_divsd_reg_reg (code, ins->dreg, ins->sreg2);
5319                         break;
5320                 case OP_FNEG: {
5321                         static double r8_0 = -0.0;
5322
5323                         g_assert (ins->sreg1 == ins->dreg);
5324
5325                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, &r8_0);
5326                         amd64_sse_xorpd_reg_membase (code, ins->dreg, AMD64_RIP, 0);
5327                         break;
5328                 }
5329                 case OP_SIN:
5330                         EMIT_SSE2_FPFUNC (code, fsin, ins->dreg, ins->sreg1);
5331                         break;
5332                 case OP_COS:
5333                         EMIT_SSE2_FPFUNC (code, fcos, ins->dreg, ins->sreg1);
5334                         break;
5335                 case OP_ABS: {
5336                         static guint64 d = 0x7fffffffffffffffUL;
5337
5338                         g_assert (ins->sreg1 == ins->dreg);
5339
5340                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, &d);
5341                         amd64_sse_andpd_reg_membase (code, ins->dreg, AMD64_RIP, 0);
5342                         break;
5343                 }
5344                 case OP_SQRT:
5345                         EMIT_SSE2_FPFUNC (code, fsqrt, ins->dreg, ins->sreg1);
5346                         break;
5347                 case OP_IMIN:
5348                         g_assert (cfg->opt & MONO_OPT_CMOV);
5349                         g_assert (ins->dreg == ins->sreg1);
5350                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
5351                         amd64_cmov_reg_size (code, X86_CC_GT, TRUE, ins->dreg, ins->sreg2, 4);
5352                         break;
5353                 case OP_IMIN_UN:
5354                         g_assert (cfg->opt & MONO_OPT_CMOV);
5355                         g_assert (ins->dreg == ins->sreg1);
5356                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
5357                         amd64_cmov_reg_size (code, X86_CC_GT, FALSE, ins->dreg, ins->sreg2, 4);
5358                         break;
5359                 case OP_IMAX:
5360                         g_assert (cfg->opt & MONO_OPT_CMOV);
5361                         g_assert (ins->dreg == ins->sreg1);
5362                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
5363                         amd64_cmov_reg_size (code, X86_CC_LT, TRUE, ins->dreg, ins->sreg2, 4);
5364                         break;
5365                 case OP_IMAX_UN:
5366                         g_assert (cfg->opt & MONO_OPT_CMOV);
5367                         g_assert (ins->dreg == ins->sreg1);
5368                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
5369                         amd64_cmov_reg_size (code, X86_CC_LT, FALSE, ins->dreg, ins->sreg2, 4);
5370                         break;
5371                 case OP_LMIN:
5372                         g_assert (cfg->opt & MONO_OPT_CMOV);
5373                         g_assert (ins->dreg == ins->sreg1);
5374                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
5375                         amd64_cmov_reg (code, X86_CC_GT, TRUE, ins->dreg, ins->sreg2);
5376                         break;
5377                 case OP_LMIN_UN:
5378                         g_assert (cfg->opt & MONO_OPT_CMOV);
5379                         g_assert (ins->dreg == ins->sreg1);
5380                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
5381                         amd64_cmov_reg (code, X86_CC_GT, FALSE, ins->dreg, ins->sreg2);
5382                         break;
5383                 case OP_LMAX:
5384                         g_assert (cfg->opt & MONO_OPT_CMOV);
5385                         g_assert (ins->dreg == ins->sreg1);
5386                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
5387                         amd64_cmov_reg (code, X86_CC_LT, TRUE, ins->dreg, ins->sreg2);
5388                         break;
5389                 case OP_LMAX_UN:
5390                         g_assert (cfg->opt & MONO_OPT_CMOV);
5391                         g_assert (ins->dreg == ins->sreg1);
5392                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
5393                         amd64_cmov_reg (code, X86_CC_LT, FALSE, ins->dreg, ins->sreg2);
5394                         break;
5395                 case OP_X86_FPOP:
5396                         break;
5397                 case OP_FCOMPARE:
5398                         /*
5399                          * The two arguments are swapped because the fbranch instructions
5400                          * depend on this for the non-sse case to work.
5401                          */
5402                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
5403                         break;
5404                 case OP_FCEQ: {
5405                         /* zeroing the register at the start results in
5406                          * shorter and faster code (we can also remove the widening op)
5407                          */
5408                         guchar *unordered_check;
5409                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
5410                         amd64_sse_comisd_reg_reg (code, ins->sreg1, ins->sreg2);
5411                         unordered_check = code;
5412                         x86_branch8 (code, X86_CC_P, 0, FALSE);
5413                         amd64_set_reg (code, X86_CC_EQ, ins->dreg, FALSE);
5414                         amd64_patch (unordered_check, code);
5415                         break;
5416                 }
5417                 case OP_FCLT:
5418                 case OP_FCLT_UN:
5419                         /* zeroing the register at the start results in
5420                          * shorter and faster code (we can also remove the widening op)
5421                          */
5422                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
5423                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
5424                         if (ins->opcode == OP_FCLT_UN) {
5425                                 guchar *unordered_check = code;
5426                                 guchar *jump_to_end;
5427                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
5428                                 amd64_set_reg (code, X86_CC_GT, ins->dreg, FALSE);
5429                                 jump_to_end = code;
5430                                 x86_jump8 (code, 0);
5431                                 amd64_patch (unordered_check, code);
5432                                 amd64_inc_reg (code, ins->dreg);
5433                                 amd64_patch (jump_to_end, code);
5434                         } else {
5435                                 amd64_set_reg (code, X86_CC_GT, ins->dreg, FALSE);
5436                         }
5437                         break;
5438                 case OP_FCGT:
5439                 case OP_FCGT_UN: {
5440                         /* zeroing the register at the start results in
5441                          * shorter and faster code (we can also remove the widening op)
5442                          */
5443                         guchar *unordered_check;
5444                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
5445                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
5446                         if (ins->opcode == OP_FCGT) {
5447                                 unordered_check = code;
5448                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
5449                                 amd64_set_reg (code, X86_CC_LT, ins->dreg, FALSE);
5450                                 amd64_patch (unordered_check, code);
5451                         } else {
5452                                 amd64_set_reg (code, X86_CC_LT, ins->dreg, FALSE);
5453                         }
5454                         break;
5455                 }
5456                 case OP_FCLT_MEMBASE:
5457                 case OP_FCGT_MEMBASE:
5458                 case OP_FCLT_UN_MEMBASE:
5459                 case OP_FCGT_UN_MEMBASE:
5460                 case OP_FCEQ_MEMBASE: {
5461                         guchar *unordered_check, *jump_to_end;
5462                         int x86_cond;
5463
5464                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
5465                         amd64_sse_comisd_reg_membase (code, ins->sreg1, ins->sreg2, ins->inst_offset);
5466
5467                         switch (ins->opcode) {
5468                         case OP_FCEQ_MEMBASE:
5469                                 x86_cond = X86_CC_EQ;
5470                                 break;
5471                         case OP_FCLT_MEMBASE:
5472                         case OP_FCLT_UN_MEMBASE:
5473                                 x86_cond = X86_CC_LT;
5474                                 break;
5475                         case OP_FCGT_MEMBASE:
5476                         case OP_FCGT_UN_MEMBASE:
5477                                 x86_cond = X86_CC_GT;
5478                                 break;
5479                         default:
5480                                 g_assert_not_reached ();
5481                         }
5482
5483                         unordered_check = code;
5484                         x86_branch8 (code, X86_CC_P, 0, FALSE);
5485                         amd64_set_reg (code, x86_cond, ins->dreg, FALSE);
5486
5487                         switch (ins->opcode) {
5488                         case OP_FCEQ_MEMBASE:
5489                         case OP_FCLT_MEMBASE:
5490                         case OP_FCGT_MEMBASE:
5491                                 amd64_patch (unordered_check, code);
5492                                 break;
5493                         case OP_FCLT_UN_MEMBASE:
5494                         case OP_FCGT_UN_MEMBASE:
5495                                 jump_to_end = code;
5496                                 x86_jump8 (code, 0);
5497                                 amd64_patch (unordered_check, code);
5498                                 amd64_inc_reg (code, ins->dreg);
5499                                 amd64_patch (jump_to_end, code);
5500                                 break;
5501                         default:
5502                                 break;
5503                         }
5504                         break;
5505                 }
5506                 case OP_FBEQ: {
5507                         guchar *jump = code;
5508                         x86_branch8 (code, X86_CC_P, 0, TRUE);
5509                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
5510                         amd64_patch (jump, code);
5511                         break;
5512                 }
5513                 case OP_FBNE_UN:
5514                         /* Branch if C013 != 100 */
5515                         /* branch if !ZF or (PF|CF) */
5516                         EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
5517                         EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
5518                         EMIT_COND_BRANCH (ins, X86_CC_B, FALSE);
5519                         break;
5520                 case OP_FBLT:
5521                         EMIT_COND_BRANCH (ins, X86_CC_GT, FALSE);
5522                         break;
5523                 case OP_FBLT_UN:
5524                         EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
5525                         EMIT_COND_BRANCH (ins, X86_CC_GT, FALSE);
5526                         break;
5527                 case OP_FBGT:
5528                 case OP_FBGT_UN:
5529                         if (ins->opcode == OP_FBGT) {
5530                                 guchar *br1;
5531
5532                                 /* skip branch if C1=1 */
5533                                 br1 = code;
5534                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
5535                                 /* branch if (C0 | C3) = 1 */
5536                                 EMIT_COND_BRANCH (ins, X86_CC_LT, FALSE);
5537                                 amd64_patch (br1, code);
5538                                 break;
5539                         } else {
5540                                 EMIT_COND_BRANCH (ins, X86_CC_LT, FALSE);
5541                         }
5542                         break;
5543                 case OP_FBGE: {
5544                         /* Branch if C013 == 100 or 001 */
5545                         guchar *br1;
5546
5547                         /* skip branch if C1=1 */
5548                         br1 = code;
5549                         x86_branch8 (code, X86_CC_P, 0, FALSE);
5550                         /* branch if (C0 | C3) = 1 */
5551                         EMIT_COND_BRANCH (ins, X86_CC_BE, FALSE);
5552                         amd64_patch (br1, code);
5553                         break;
5554                 }
5555                 case OP_FBGE_UN:
5556                         /* Branch if C013 == 000 */
5557                         EMIT_COND_BRANCH (ins, X86_CC_LE, FALSE);
5558                         break;
5559                 case OP_FBLE: {
5560                         /* Branch if C013=000 or 100 */
5561                         guchar *br1;
5562
5563                         /* skip branch if C1=1 */
5564                         br1 = code;
5565                         x86_branch8 (code, X86_CC_P, 0, FALSE);
5566                         /* branch if C0=0 */
5567                         EMIT_COND_BRANCH (ins, X86_CC_NB, FALSE);
5568                         amd64_patch (br1, code);
5569                         break;
5570                 }
5571                 case OP_FBLE_UN:
5572                         /* Branch if C013 != 001 */
5573                         EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
5574                         EMIT_COND_BRANCH (ins, X86_CC_GE, FALSE);
5575                         break;
5576                 case OP_CKFINITE:
5577                         /* Transfer value to the fp stack */
5578                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 16);
5579                         amd64_movsd_membase_reg (code, AMD64_RSP, 0, ins->sreg1);
5580                         amd64_fld_membase (code, AMD64_RSP, 0, TRUE);
5581
5582                         amd64_push_reg (code, AMD64_RAX);
5583                         amd64_fxam (code);
5584                         amd64_fnstsw (code);
5585                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, 0x4100);
5586                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
5587                         amd64_pop_reg (code, AMD64_RAX);
5588                         amd64_fstp (code, 0);
5589                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_EQ, FALSE, "ArithmeticException");
5590                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 16);
5591                         break;
5592                 case OP_TLS_GET: {
5593                         code = mono_amd64_emit_tls_get (code, ins->dreg, ins->inst_offset);
5594                         break;
5595                 }
5596                 case OP_MEMORY_BARRIER: {
5597                         switch (ins->backend.memory_barrier_kind) {
5598                         case StoreLoadBarrier:
5599                         case FullBarrier:
5600                                 /* http://blogs.sun.com/dave/resource/NHM-Pipeline-Blog-V2.txt */
5601                                 x86_prefix (code, X86_LOCK_PREFIX);
5602                                 amd64_alu_membase_imm (code, X86_ADD, AMD64_RSP, 0, 0);
5603                                 break;
5604                         }
5605                         break;
5606                 }
5607                 case OP_ATOMIC_ADD_I4:
5608                 case OP_ATOMIC_ADD_I8: {
5609                         int dreg = ins->dreg;
5610                         guint32 size = (ins->opcode == OP_ATOMIC_ADD_I4) ? 4 : 8;
5611
5612                         if (dreg == ins->inst_basereg)
5613                                 dreg = AMD64_R11;
5614
5615                         if (dreg != ins->sreg2)
5616                                 amd64_mov_reg_reg (code, ins->dreg, ins->sreg2, size);
5617
5618                         x86_prefix (code, X86_LOCK_PREFIX);
5619                         amd64_xadd_membase_reg (code, ins->inst_basereg, ins->inst_offset, dreg, size);
5620
5621                         if (dreg != ins->dreg)
5622                                 amd64_mov_reg_reg (code, ins->dreg, dreg, size);
5623
5624                         break;
5625                 }
5626                 case OP_ATOMIC_ADD_NEW_I4:
5627                 case OP_ATOMIC_ADD_NEW_I8: {
5628                         int dreg = ins->dreg;
5629                         guint32 size = (ins->opcode == OP_ATOMIC_ADD_NEW_I4) ? 4 : 8;
5630
5631                         if ((dreg == ins->sreg2) || (dreg == ins->inst_basereg))
5632                                 dreg = AMD64_R11;
5633
5634                         amd64_mov_reg_reg (code, dreg, ins->sreg2, size);
5635                         amd64_prefix (code, X86_LOCK_PREFIX);
5636                         amd64_xadd_membase_reg (code, ins->inst_basereg, ins->inst_offset, dreg, size);
5637                         /* dreg contains the old value, add with sreg2 value */
5638                         amd64_alu_reg_reg_size (code, X86_ADD, dreg, ins->sreg2, size);
5639
5640                         if (ins->dreg != dreg)
5641                                 amd64_mov_reg_reg (code, ins->dreg, dreg, size);
5642
5643                         break;
5644                 }
5645                 case OP_ATOMIC_EXCHANGE_I4:
5646                 case OP_ATOMIC_EXCHANGE_I8: {
5647                         guchar *br[2];
5648                         int sreg2 = ins->sreg2;
5649                         int breg = ins->inst_basereg;
5650                         guint32 size;
5651                         gboolean need_push = FALSE, rdx_pushed = FALSE;
5652
5653                         if (ins->opcode == OP_ATOMIC_EXCHANGE_I8)
5654                                 size = 8;
5655                         else
5656                                 size = 4;
5657
5658                         /*
5659                          * See http://msdn.microsoft.com/en-us/magazine/cc302329.aspx for
5660                          * an explanation of how this works.
5661                          */
5662
5663                         /* cmpxchg uses eax as comperand, need to make sure we can use it
5664                          * hack to overcome limits in x86 reg allocator
5665                          * (req: dreg == eax and sreg2 != eax and breg != eax)
5666                          */
5667                         g_assert (ins->dreg == AMD64_RAX);
5668
5669                         if (breg == AMD64_RAX && ins->sreg2 == AMD64_RAX)
5670                                 /* Highly unlikely, but possible */
5671                                 need_push = TRUE;
5672
5673                         /* The pushes invalidate rsp */
5674                         if ((breg == AMD64_RAX) || need_push) {
5675                                 amd64_mov_reg_reg (code, AMD64_R11, breg, 8);
5676                                 breg = AMD64_R11;
5677                         }
5678
5679                         /* We need the EAX reg for the comparand */
5680                         if (ins->sreg2 == AMD64_RAX) {
5681                                 if (breg != AMD64_R11) {
5682                                         amd64_mov_reg_reg (code, AMD64_R11, AMD64_RAX, 8);
5683                                         sreg2 = AMD64_R11;
5684                                 } else {
5685                                         g_assert (need_push);
5686                                         amd64_push_reg (code, AMD64_RDX);
5687                                         amd64_mov_reg_reg (code, AMD64_RDX, AMD64_RAX, size);
5688                                         sreg2 = AMD64_RDX;
5689                                         rdx_pushed = TRUE;
5690                                 }
5691                         }
5692
5693                         amd64_mov_reg_membase (code, AMD64_RAX, breg, ins->inst_offset, size);
5694
5695                         br [0] = code; amd64_prefix (code, X86_LOCK_PREFIX);
5696                         amd64_cmpxchg_membase_reg_size (code, breg, ins->inst_offset, sreg2, size);
5697                         br [1] = code; amd64_branch8 (code, X86_CC_NE, -1, FALSE);
5698                         amd64_patch (br [1], br [0]);
5699
5700                         if (rdx_pushed)
5701                                 amd64_pop_reg (code, AMD64_RDX);
5702
5703                         break;
5704                 }
5705                 case OP_ATOMIC_CAS_I4:
5706                 case OP_ATOMIC_CAS_I8: {
5707                         guint32 size;
5708
5709                         if (ins->opcode == OP_ATOMIC_CAS_I8)
5710                                 size = 8;
5711                         else
5712                                 size = 4;
5713
5714                         /*
5715                          * See http://msdn.microsoft.com/en-us/magazine/cc302329.aspx for
5716                          * an explanation of how this works.
5717                          */
5718                         g_assert (ins->sreg3 == AMD64_RAX);
5719                         g_assert (ins->sreg1 != AMD64_RAX);
5720                         g_assert (ins->sreg1 != ins->sreg2);
5721
5722                         amd64_prefix (code, X86_LOCK_PREFIX);
5723                         amd64_cmpxchg_membase_reg_size (code, ins->sreg1, ins->inst_offset, ins->sreg2, size);
5724
5725                         if (ins->dreg != AMD64_RAX)
5726                                 amd64_mov_reg_reg (code, ins->dreg, AMD64_RAX, size);
5727                         break;
5728                 }
5729                 case OP_CARD_TABLE_WBARRIER: {
5730                         int ptr = ins->sreg1;
5731                         int value = ins->sreg2;
5732                         guchar *br;
5733                         int nursery_shift, card_table_shift;
5734                         gpointer card_table_mask;
5735                         size_t nursery_size;
5736
5737                         gpointer card_table = mono_gc_get_card_table (&card_table_shift, &card_table_mask);
5738                         guint64 nursery_start = (guint64)mono_gc_get_nursery (&nursery_shift, &nursery_size);
5739                         guint64 shifted_nursery_start = nursery_start >> nursery_shift;
5740
5741                         /*If either point to the stack we can simply avoid the WB. This happens due to
5742                          * optimizations revealing a stack store that was not visible when op_cardtable was emited.
5743                          */
5744                         if (ins->sreg1 == AMD64_RSP || ins->sreg2 == AMD64_RSP)
5745                                 continue;
5746
5747                         /*
5748                          * We need one register we can clobber, we choose EDX and make sreg1
5749                          * fixed EAX to work around limitations in the local register allocator.
5750                          * sreg2 might get allocated to EDX, but that is not a problem since
5751                          * we use it before clobbering EDX.
5752                          */
5753                         g_assert (ins->sreg1 == AMD64_RAX);
5754
5755                         /*
5756                          * This is the code we produce:
5757                          *
5758                          *   edx = value
5759                          *   edx >>= nursery_shift
5760                          *   cmp edx, (nursery_start >> nursery_shift)
5761                          *   jne done
5762                          *   edx = ptr
5763                          *   edx >>= card_table_shift
5764                          *   edx += cardtable
5765                          *   [edx] = 1
5766                          * done:
5767                          */
5768
5769                         if (mono_gc_card_table_nursery_check ()) {
5770                                 if (value != AMD64_RDX)
5771                                         amd64_mov_reg_reg (code, AMD64_RDX, value, 8);
5772                                 amd64_shift_reg_imm (code, X86_SHR, AMD64_RDX, nursery_shift);
5773                                 if (shifted_nursery_start >> 31) {
5774                                         /*
5775                                          * The value we need to compare against is 64 bits, so we need
5776                                          * another spare register.  We use RBX, which we save and
5777                                          * restore.
5778                                          */
5779                                         amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RBX, 8);
5780                                         amd64_mov_reg_imm (code, AMD64_RBX, shifted_nursery_start);
5781                                         amd64_alu_reg_reg (code, X86_CMP, AMD64_RDX, AMD64_RBX);
5782                                         amd64_mov_reg_membase (code, AMD64_RBX, AMD64_RSP, -8, 8);
5783                                 } else {
5784                                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RDX, shifted_nursery_start);
5785                                 }
5786                                 br = code; x86_branch8 (code, X86_CC_NE, -1, FALSE);
5787                         }
5788                         amd64_mov_reg_reg (code, AMD64_RDX, ptr, 8);
5789                         amd64_shift_reg_imm (code, X86_SHR, AMD64_RDX, card_table_shift);
5790                         if (card_table_mask)
5791                                 amd64_alu_reg_imm (code, X86_AND, AMD64_RDX, (guint32)(guint64)card_table_mask);
5792
5793                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_GC_CARD_TABLE_ADDR, card_table);
5794                         amd64_alu_reg_membase (code, X86_ADD, AMD64_RDX, AMD64_RIP, 0);
5795
5796                         amd64_mov_membase_imm (code, AMD64_RDX, 0, 1, 1);
5797                         if (mono_gc_card_table_nursery_check ())
5798                                 x86_patch (br, code);
5799                         break;
5800                 }
5801 #ifdef MONO_ARCH_SIMD_INTRINSICS
5802                 /* TODO: Some of these IR opcodes are marked as no clobber when they indeed do. */
5803                 case OP_ADDPS:
5804                         amd64_sse_addps_reg_reg (code, ins->sreg1, ins->sreg2);
5805                         break;
5806                 case OP_DIVPS:
5807                         amd64_sse_divps_reg_reg (code, ins->sreg1, ins->sreg2);
5808                         break;
5809                 case OP_MULPS:
5810                         amd64_sse_mulps_reg_reg (code, ins->sreg1, ins->sreg2);
5811                         break;
5812                 case OP_SUBPS:
5813                         amd64_sse_subps_reg_reg (code, ins->sreg1, ins->sreg2);
5814                         break;
5815                 case OP_MAXPS:
5816                         amd64_sse_maxps_reg_reg (code, ins->sreg1, ins->sreg2);
5817                         break;
5818                 case OP_MINPS:
5819                         amd64_sse_minps_reg_reg (code, ins->sreg1, ins->sreg2);
5820                         break;
5821                 case OP_COMPPS:
5822                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 7);
5823                         amd64_sse_cmpps_reg_reg_imm (code, ins->sreg1, ins->sreg2, ins->inst_c0);
5824                         break;
5825                 case OP_ANDPS:
5826                         amd64_sse_andps_reg_reg (code, ins->sreg1, ins->sreg2);
5827                         break;
5828                 case OP_ANDNPS:
5829                         amd64_sse_andnps_reg_reg (code, ins->sreg1, ins->sreg2);
5830                         break;
5831                 case OP_ORPS:
5832                         amd64_sse_orps_reg_reg (code, ins->sreg1, ins->sreg2);
5833                         break;
5834                 case OP_XORPS:
5835                         amd64_sse_xorps_reg_reg (code, ins->sreg1, ins->sreg2);
5836                         break;
5837                 case OP_SQRTPS:
5838                         amd64_sse_sqrtps_reg_reg (code, ins->dreg, ins->sreg1);
5839                         break;
5840                 case OP_RSQRTPS:
5841                         amd64_sse_rsqrtps_reg_reg (code, ins->dreg, ins->sreg1);
5842                         break;
5843                 case OP_RCPPS:
5844                         amd64_sse_rcpps_reg_reg (code, ins->dreg, ins->sreg1);
5845                         break;
5846                 case OP_ADDSUBPS:
5847                         amd64_sse_addsubps_reg_reg (code, ins->sreg1, ins->sreg2);
5848                         break;
5849                 case OP_HADDPS:
5850                         amd64_sse_haddps_reg_reg (code, ins->sreg1, ins->sreg2);
5851                         break;
5852                 case OP_HSUBPS:
5853                         amd64_sse_hsubps_reg_reg (code, ins->sreg1, ins->sreg2);
5854                         break;
5855                 case OP_DUPPS_HIGH:
5856                         amd64_sse_movshdup_reg_reg (code, ins->dreg, ins->sreg1);
5857                         break;
5858                 case OP_DUPPS_LOW:
5859                         amd64_sse_movsldup_reg_reg (code, ins->dreg, ins->sreg1);
5860                         break;
5861
5862                 case OP_PSHUFLEW_HIGH:
5863                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 0xFF);
5864                         amd64_sse_pshufhw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
5865                         break;
5866                 case OP_PSHUFLEW_LOW:
5867                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 0xFF);
5868                         amd64_sse_pshuflw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
5869                         break;
5870                 case OP_PSHUFLED:
5871                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 0xFF);
5872                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
5873                         break;
5874                 case OP_SHUFPS:
5875                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 0xFF);
5876                         amd64_sse_shufps_reg_reg_imm (code, ins->sreg1, ins->sreg2, ins->inst_c0);
5877                         break;
5878                 case OP_SHUFPD:
5879                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 0x3);
5880                         amd64_sse_shufpd_reg_reg_imm (code, ins->sreg1, ins->sreg2, ins->inst_c0);
5881                         break;
5882
5883                 case OP_ADDPD:
5884                         amd64_sse_addpd_reg_reg (code, ins->sreg1, ins->sreg2);
5885                         break;
5886                 case OP_DIVPD:
5887                         amd64_sse_divpd_reg_reg (code, ins->sreg1, ins->sreg2);
5888                         break;
5889                 case OP_MULPD:
5890                         amd64_sse_mulpd_reg_reg (code, ins->sreg1, ins->sreg2);
5891                         break;
5892                 case OP_SUBPD:
5893                         amd64_sse_subpd_reg_reg (code, ins->sreg1, ins->sreg2);
5894                         break;
5895                 case OP_MAXPD:
5896                         amd64_sse_maxpd_reg_reg (code, ins->sreg1, ins->sreg2);
5897                         break;
5898                 case OP_MINPD:
5899                         amd64_sse_minpd_reg_reg (code, ins->sreg1, ins->sreg2);
5900                         break;
5901                 case OP_COMPPD:
5902                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 7);
5903                         amd64_sse_cmppd_reg_reg_imm (code, ins->sreg1, ins->sreg2, ins->inst_c0);
5904                         break;
5905                 case OP_ANDPD:
5906                         amd64_sse_andpd_reg_reg (code, ins->sreg1, ins->sreg2);
5907                         break;
5908                 case OP_ANDNPD:
5909                         amd64_sse_andnpd_reg_reg (code, ins->sreg1, ins->sreg2);
5910                         break;
5911                 case OP_ORPD:
5912                         amd64_sse_orpd_reg_reg (code, ins->sreg1, ins->sreg2);
5913                         break;
5914                 case OP_XORPD:
5915                         amd64_sse_xorpd_reg_reg (code, ins->sreg1, ins->sreg2);
5916                         break;
5917                 case OP_SQRTPD:
5918                         amd64_sse_sqrtpd_reg_reg (code, ins->dreg, ins->sreg1);
5919                         break;
5920                 case OP_ADDSUBPD:
5921                         amd64_sse_addsubpd_reg_reg (code, ins->sreg1, ins->sreg2);
5922                         break;
5923                 case OP_HADDPD:
5924                         amd64_sse_haddpd_reg_reg (code, ins->sreg1, ins->sreg2);
5925                         break;
5926                 case OP_HSUBPD:
5927                         amd64_sse_hsubpd_reg_reg (code, ins->sreg1, ins->sreg2);
5928                         break;
5929                 case OP_DUPPD:
5930                         amd64_sse_movddup_reg_reg (code, ins->dreg, ins->sreg1);
5931                         break;
5932
5933                 case OP_EXTRACT_MASK:
5934                         amd64_sse_pmovmskb_reg_reg (code, ins->dreg, ins->sreg1);
5935                         break;
5936
5937                 case OP_PAND:
5938                         amd64_sse_pand_reg_reg (code, ins->sreg1, ins->sreg2);
5939                         break;
5940                 case OP_POR:
5941                         amd64_sse_por_reg_reg (code, ins->sreg1, ins->sreg2);
5942                         break;
5943                 case OP_PXOR:
5944                         amd64_sse_pxor_reg_reg (code, ins->sreg1, ins->sreg2);
5945                         break;
5946
5947                 case OP_PADDB:
5948                         amd64_sse_paddb_reg_reg (code, ins->sreg1, ins->sreg2);
5949                         break;
5950                 case OP_PADDW:
5951                         amd64_sse_paddw_reg_reg (code, ins->sreg1, ins->sreg2);
5952                         break;
5953                 case OP_PADDD:
5954                         amd64_sse_paddd_reg_reg (code, ins->sreg1, ins->sreg2);
5955                         break;
5956                 case OP_PADDQ:
5957                         amd64_sse_paddq_reg_reg (code, ins->sreg1, ins->sreg2);
5958                         break;
5959
5960                 case OP_PSUBB:
5961                         amd64_sse_psubb_reg_reg (code, ins->sreg1, ins->sreg2);
5962                         break;
5963                 case OP_PSUBW:
5964                         amd64_sse_psubw_reg_reg (code, ins->sreg1, ins->sreg2);
5965                         break;
5966                 case OP_PSUBD:
5967                         amd64_sse_psubd_reg_reg (code, ins->sreg1, ins->sreg2);
5968                         break;
5969                 case OP_PSUBQ:
5970                         amd64_sse_psubq_reg_reg (code, ins->sreg1, ins->sreg2);
5971                         break;
5972
5973                 case OP_PMAXB_UN:
5974                         amd64_sse_pmaxub_reg_reg (code, ins->sreg1, ins->sreg2);
5975                         break;
5976                 case OP_PMAXW_UN:
5977                         amd64_sse_pmaxuw_reg_reg (code, ins->sreg1, ins->sreg2);
5978                         break;
5979                 case OP_PMAXD_UN:
5980                         amd64_sse_pmaxud_reg_reg (code, ins->sreg1, ins->sreg2);
5981                         break;
5982
5983                 case OP_PMAXB:
5984                         amd64_sse_pmaxsb_reg_reg (code, ins->sreg1, ins->sreg2);
5985                         break;
5986                 case OP_PMAXW:
5987                         amd64_sse_pmaxsw_reg_reg (code, ins->sreg1, ins->sreg2);
5988                         break;
5989                 case OP_PMAXD:
5990                         amd64_sse_pmaxsd_reg_reg (code, ins->sreg1, ins->sreg2);
5991                         break;
5992
5993                 case OP_PAVGB_UN:
5994                         amd64_sse_pavgb_reg_reg (code, ins->sreg1, ins->sreg2);
5995                         break;
5996                 case OP_PAVGW_UN:
5997                         amd64_sse_pavgw_reg_reg (code, ins->sreg1, ins->sreg2);
5998                         break;
5999
6000                 case OP_PMINB_UN:
6001                         amd64_sse_pminub_reg_reg (code, ins->sreg1, ins->sreg2);
6002                         break;
6003                 case OP_PMINW_UN:
6004                         amd64_sse_pminuw_reg_reg (code, ins->sreg1, ins->sreg2);
6005                         break;
6006                 case OP_PMIND_UN:
6007                         amd64_sse_pminud_reg_reg (code, ins->sreg1, ins->sreg2);
6008                         break;
6009
6010                 case OP_PMINB:
6011                         amd64_sse_pminsb_reg_reg (code, ins->sreg1, ins->sreg2);
6012                         break;
6013                 case OP_PMINW:
6014                         amd64_sse_pminsw_reg_reg (code, ins->sreg1, ins->sreg2);
6015                         break;
6016                 case OP_PMIND:
6017                         amd64_sse_pminsd_reg_reg (code, ins->sreg1, ins->sreg2);
6018                         break;
6019
6020                 case OP_PCMPEQB:
6021                         amd64_sse_pcmpeqb_reg_reg (code, ins->sreg1, ins->sreg2);
6022                         break;
6023                 case OP_PCMPEQW:
6024                         amd64_sse_pcmpeqw_reg_reg (code, ins->sreg1, ins->sreg2);
6025                         break;
6026                 case OP_PCMPEQD:
6027                         amd64_sse_pcmpeqd_reg_reg (code, ins->sreg1, ins->sreg2);
6028                         break;
6029                 case OP_PCMPEQQ:
6030                         amd64_sse_pcmpeqq_reg_reg (code, ins->sreg1, ins->sreg2);
6031                         break;
6032
6033                 case OP_PCMPGTB:
6034                         amd64_sse_pcmpgtb_reg_reg (code, ins->sreg1, ins->sreg2);
6035                         break;
6036                 case OP_PCMPGTW:
6037                         amd64_sse_pcmpgtw_reg_reg (code, ins->sreg1, ins->sreg2);
6038                         break;
6039                 case OP_PCMPGTD:
6040                         amd64_sse_pcmpgtd_reg_reg (code, ins->sreg1, ins->sreg2);
6041                         break;
6042                 case OP_PCMPGTQ:
6043                         amd64_sse_pcmpgtq_reg_reg (code, ins->sreg1, ins->sreg2);
6044                         break;
6045
6046                 case OP_PSUM_ABS_DIFF:
6047                         amd64_sse_psadbw_reg_reg (code, ins->sreg1, ins->sreg2);
6048                         break;
6049
6050                 case OP_UNPACK_LOWB:
6051                         amd64_sse_punpcklbw_reg_reg (code, ins->sreg1, ins->sreg2);
6052                         break;
6053                 case OP_UNPACK_LOWW:
6054                         amd64_sse_punpcklwd_reg_reg (code, ins->sreg1, ins->sreg2);
6055                         break;
6056                 case OP_UNPACK_LOWD:
6057                         amd64_sse_punpckldq_reg_reg (code, ins->sreg1, ins->sreg2);
6058                         break;
6059                 case OP_UNPACK_LOWQ:
6060                         amd64_sse_punpcklqdq_reg_reg (code, ins->sreg1, ins->sreg2);
6061                         break;
6062                 case OP_UNPACK_LOWPS:
6063                         amd64_sse_unpcklps_reg_reg (code, ins->sreg1, ins->sreg2);
6064                         break;
6065                 case OP_UNPACK_LOWPD:
6066                         amd64_sse_unpcklpd_reg_reg (code, ins->sreg1, ins->sreg2);
6067                         break;
6068
6069                 case OP_UNPACK_HIGHB:
6070                         amd64_sse_punpckhbw_reg_reg (code, ins->sreg1, ins->sreg2);
6071                         break;
6072                 case OP_UNPACK_HIGHW:
6073                         amd64_sse_punpckhwd_reg_reg (code, ins->sreg1, ins->sreg2);
6074                         break;
6075                 case OP_UNPACK_HIGHD:
6076                         amd64_sse_punpckhdq_reg_reg (code, ins->sreg1, ins->sreg2);
6077                         break;
6078                 case OP_UNPACK_HIGHQ:
6079                         amd64_sse_punpckhqdq_reg_reg (code, ins->sreg1, ins->sreg2);
6080                         break;
6081                 case OP_UNPACK_HIGHPS:
6082                         amd64_sse_unpckhps_reg_reg (code, ins->sreg1, ins->sreg2);
6083                         break;
6084                 case OP_UNPACK_HIGHPD:
6085                         amd64_sse_unpckhpd_reg_reg (code, ins->sreg1, ins->sreg2);
6086                         break;
6087
6088                 case OP_PACKW:
6089                         amd64_sse_packsswb_reg_reg (code, ins->sreg1, ins->sreg2);
6090                         break;
6091                 case OP_PACKD:
6092                         amd64_sse_packssdw_reg_reg (code, ins->sreg1, ins->sreg2);
6093                         break;
6094                 case OP_PACKW_UN:
6095                         amd64_sse_packuswb_reg_reg (code, ins->sreg1, ins->sreg2);
6096                         break;
6097                 case OP_PACKD_UN:
6098                         amd64_sse_packusdw_reg_reg (code, ins->sreg1, ins->sreg2);
6099                         break;
6100
6101                 case OP_PADDB_SAT_UN:
6102                         amd64_sse_paddusb_reg_reg (code, ins->sreg1, ins->sreg2);
6103                         break;
6104                 case OP_PSUBB_SAT_UN:
6105                         amd64_sse_psubusb_reg_reg (code, ins->sreg1, ins->sreg2);
6106                         break;
6107                 case OP_PADDW_SAT_UN:
6108                         amd64_sse_paddusw_reg_reg (code, ins->sreg1, ins->sreg2);
6109                         break;
6110                 case OP_PSUBW_SAT_UN:
6111                         amd64_sse_psubusw_reg_reg (code, ins->sreg1, ins->sreg2);
6112                         break;
6113
6114                 case OP_PADDB_SAT:
6115                         amd64_sse_paddsb_reg_reg (code, ins->sreg1, ins->sreg2);
6116                         break;
6117                 case OP_PSUBB_SAT:
6118                         amd64_sse_psubsb_reg_reg (code, ins->sreg1, ins->sreg2);
6119                         break;
6120                 case OP_PADDW_SAT:
6121                         amd64_sse_paddsw_reg_reg (code, ins->sreg1, ins->sreg2);
6122                         break;
6123                 case OP_PSUBW_SAT:
6124                         amd64_sse_psubsw_reg_reg (code, ins->sreg1, ins->sreg2);
6125                         break;
6126
6127                 case OP_PMULW:
6128                         amd64_sse_pmullw_reg_reg (code, ins->sreg1, ins->sreg2);
6129                         break;
6130                 case OP_PMULD:
6131                         amd64_sse_pmulld_reg_reg (code, ins->sreg1, ins->sreg2);
6132                         break;
6133                 case OP_PMULQ:
6134                         amd64_sse_pmuludq_reg_reg (code, ins->sreg1, ins->sreg2);
6135                         break;
6136                 case OP_PMULW_HIGH_UN:
6137                         amd64_sse_pmulhuw_reg_reg (code, ins->sreg1, ins->sreg2);
6138                         break;
6139                 case OP_PMULW_HIGH:
6140                         amd64_sse_pmulhw_reg_reg (code, ins->sreg1, ins->sreg2);
6141                         break;
6142
6143                 case OP_PSHRW:
6144                         amd64_sse_psrlw_reg_imm (code, ins->dreg, ins->inst_imm);
6145                         break;
6146                 case OP_PSHRW_REG:
6147                         amd64_sse_psrlw_reg_reg (code, ins->dreg, ins->sreg2);
6148                         break;
6149
6150                 case OP_PSARW:
6151                         amd64_sse_psraw_reg_imm (code, ins->dreg, ins->inst_imm);
6152                         break;
6153                 case OP_PSARW_REG:
6154                         amd64_sse_psraw_reg_reg (code, ins->dreg, ins->sreg2);
6155                         break;
6156
6157                 case OP_PSHLW:
6158                         amd64_sse_psllw_reg_imm (code, ins->dreg, ins->inst_imm);
6159                         break;
6160                 case OP_PSHLW_REG:
6161                         amd64_sse_psllw_reg_reg (code, ins->dreg, ins->sreg2);
6162                         break;
6163
6164                 case OP_PSHRD:
6165                         amd64_sse_psrld_reg_imm (code, ins->dreg, ins->inst_imm);
6166                         break;
6167                 case OP_PSHRD_REG:
6168                         amd64_sse_psrld_reg_reg (code, ins->dreg, ins->sreg2);
6169                         break;
6170
6171                 case OP_PSARD:
6172                         amd64_sse_psrad_reg_imm (code, ins->dreg, ins->inst_imm);
6173                         break;
6174                 case OP_PSARD_REG:
6175                         amd64_sse_psrad_reg_reg (code, ins->dreg, ins->sreg2);
6176                         break;
6177
6178                 case OP_PSHLD:
6179                         amd64_sse_pslld_reg_imm (code, ins->dreg, ins->inst_imm);
6180                         break;
6181                 case OP_PSHLD_REG:
6182                         amd64_sse_pslld_reg_reg (code, ins->dreg, ins->sreg2);
6183                         break;
6184
6185                 case OP_PSHRQ:
6186                         amd64_sse_psrlq_reg_imm (code, ins->dreg, ins->inst_imm);
6187                         break;
6188                 case OP_PSHRQ_REG:
6189                         amd64_sse_psrlq_reg_reg (code, ins->dreg, ins->sreg2);
6190                         break;
6191
6192                 /*TODO: This is appart of the sse spec but not added
6193                 case OP_PSARQ:
6194                         amd64_sse_psraq_reg_imm (code, ins->dreg, ins->inst_imm);
6195                         break;
6196                 case OP_PSARQ_REG:
6197                         amd64_sse_psraq_reg_reg (code, ins->dreg, ins->sreg2);
6198                         break;
6199                 */
6200
6201                 case OP_PSHLQ:
6202                         amd64_sse_psllq_reg_imm (code, ins->dreg, ins->inst_imm);
6203                         break;
6204                 case OP_PSHLQ_REG:
6205                         amd64_sse_psllq_reg_reg (code, ins->dreg, ins->sreg2);
6206                         break;
6207                 case OP_CVTDQ2PD:
6208                         amd64_sse_cvtdq2pd_reg_reg (code, ins->dreg, ins->sreg1);
6209                         break;
6210                 case OP_CVTDQ2PS:
6211                         amd64_sse_cvtdq2ps_reg_reg (code, ins->dreg, ins->sreg1);
6212                         break;
6213                 case OP_CVTPD2DQ:
6214                         amd64_sse_cvtpd2dq_reg_reg (code, ins->dreg, ins->sreg1);
6215                         break;
6216                 case OP_CVTPD2PS:
6217                         amd64_sse_cvtpd2ps_reg_reg (code, ins->dreg, ins->sreg1);
6218                         break;
6219                 case OP_CVTPS2DQ:
6220                         amd64_sse_cvtps2dq_reg_reg (code, ins->dreg, ins->sreg1);
6221                         break;
6222                 case OP_CVTPS2PD:
6223                         amd64_sse_cvtps2pd_reg_reg (code, ins->dreg, ins->sreg1);
6224                         break;
6225                 case OP_CVTTPD2DQ:
6226                         amd64_sse_cvttpd2dq_reg_reg (code, ins->dreg, ins->sreg1);
6227                         break;
6228                 case OP_CVTTPS2DQ:
6229                         amd64_sse_cvttps2dq_reg_reg (code, ins->dreg, ins->sreg1);
6230                         break;
6231
6232                 case OP_ICONV_TO_X:
6233                         amd64_movd_xreg_reg_size (code, ins->dreg, ins->sreg1, 4);
6234                         break;
6235                 case OP_EXTRACT_I4:
6236                         amd64_movd_reg_xreg_size (code, ins->dreg, ins->sreg1, 4);
6237                         break;
6238                 case OP_EXTRACT_I8:
6239                         if (ins->inst_c0) {
6240                                 amd64_movhlps_reg_reg (code, AMD64_XMM15, ins->sreg1);
6241                                 amd64_movd_reg_xreg_size (code, ins->dreg, AMD64_XMM15, 8);
6242                         } else {
6243                                 amd64_movd_reg_xreg_size (code, ins->dreg, ins->sreg1, 8);
6244                         }
6245                         break;
6246                 case OP_EXTRACT_I1:
6247                 case OP_EXTRACT_U1:
6248                         amd64_movd_reg_xreg_size (code, ins->dreg, ins->sreg1, 4);
6249                         if (ins->inst_c0)
6250                                 amd64_shift_reg_imm (code, X86_SHR, ins->dreg, ins->inst_c0 * 8);
6251                         amd64_widen_reg (code, ins->dreg, ins->dreg, ins->opcode == OP_EXTRACT_I1, FALSE);
6252                         break;
6253                 case OP_EXTRACT_I2:
6254                 case OP_EXTRACT_U2:
6255                         /*amd64_movd_reg_xreg_size (code, ins->dreg, ins->sreg1, 4);
6256                         if (ins->inst_c0)
6257                                 amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, 16, 4);*/
6258                         amd64_sse_pextrw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
6259                         amd64_widen_reg_size (code, ins->dreg, ins->dreg, ins->opcode == OP_EXTRACT_I2, TRUE, 4);
6260                         break;
6261                 case OP_EXTRACT_R8:
6262                         if (ins->inst_c0)
6263                                 amd64_movhlps_reg_reg (code, ins->dreg, ins->sreg1);
6264                         else
6265                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
6266                         break;
6267                 case OP_INSERT_I2:
6268                         amd64_sse_pinsrw_reg_reg_imm (code, ins->sreg1, ins->sreg2, ins->inst_c0);
6269                         break;
6270                 case OP_EXTRACTX_U2:
6271                         amd64_sse_pextrw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
6272                         break;
6273                 case OP_INSERTX_U1_SLOW:
6274                         /*sreg1 is the extracted ireg (scratch)
6275                         /sreg2 is the to be inserted ireg (scratch)
6276                         /dreg is the xreg to receive the value*/
6277
6278                         /*clear the bits from the extracted word*/
6279                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ins->inst_c0 & 1 ? 0x00FF : 0xFF00);
6280                         /*shift the value to insert if needed*/
6281                         if (ins->inst_c0 & 1)
6282                                 amd64_shift_reg_imm_size (code, X86_SHL, ins->sreg2, 8, 4);
6283                         /*join them together*/
6284                         amd64_alu_reg_reg (code, X86_OR, ins->sreg1, ins->sreg2);
6285                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0 / 2);
6286                         break;
6287                 case OP_INSERTX_I4_SLOW:
6288                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg2, ins->inst_c0 * 2);
6289                         amd64_shift_reg_imm (code, X86_SHR, ins->sreg2, 16);
6290                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg2, ins->inst_c0 * 2 + 1);
6291                         break;
6292                 case OP_INSERTX_I8_SLOW:
6293                         amd64_movd_xreg_reg_size(code, AMD64_XMM15, ins->sreg2, 8);
6294                         if (ins->inst_c0)
6295                                 amd64_movlhps_reg_reg (code, ins->dreg, AMD64_XMM15);
6296                         else
6297                                 amd64_sse_movsd_reg_reg (code, ins->dreg, AMD64_XMM15);
6298                         break;
6299
6300                 case OP_INSERTX_R4_SLOW:
6301                         switch (ins->inst_c0) {
6302                         case 0:
6303                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg2);
6304                                 break;
6305                         case 1:
6306                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(1, 0, 2, 3));
6307                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg2);
6308                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(1, 0, 2, 3));
6309                                 break;
6310                         case 2:
6311                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(2, 1, 0, 3));
6312                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg2);
6313                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(2, 1, 0, 3));
6314                                 break;
6315                         case 3:
6316                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(3, 1, 2, 0));
6317                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg2);
6318                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(3, 1, 2, 0));
6319                                 break;
6320                         }
6321                         break;
6322                 case OP_INSERTX_R8_SLOW:
6323                         if (ins->inst_c0)
6324                                 amd64_movlhps_reg_reg (code, ins->dreg, ins->sreg2);
6325                         else
6326                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg2);
6327                         break;
6328                 case OP_STOREX_MEMBASE_REG:
6329                 case OP_STOREX_MEMBASE:
6330                         amd64_sse_movups_membase_reg (code, ins->dreg, ins->inst_offset, ins->sreg1);
6331                         break;
6332                 case OP_LOADX_MEMBASE:
6333                         amd64_sse_movups_reg_membase (code, ins->dreg, ins->sreg1, ins->inst_offset);
6334                         break;
6335                 case OP_LOADX_ALIGNED_MEMBASE:
6336                         amd64_sse_movaps_reg_membase (code, ins->dreg, ins->sreg1, ins->inst_offset);
6337                         break;
6338                 case OP_STOREX_ALIGNED_MEMBASE_REG:
6339                         amd64_sse_movaps_membase_reg (code, ins->dreg, ins->inst_offset, ins->sreg1);
6340                         break;
6341                 case OP_STOREX_NTA_MEMBASE_REG:
6342                         amd64_sse_movntps_reg_membase (code, ins->dreg, ins->sreg1, ins->inst_offset);
6343                         break;
6344                 case OP_PREFETCH_MEMBASE:
6345                         amd64_sse_prefetch_reg_membase (code, ins->backend.arg_info, ins->sreg1, ins->inst_offset);
6346                         break;
6347
6348                 case OP_XMOVE:
6349                         /*FIXME the peephole pass should have killed this*/
6350                         if (ins->dreg != ins->sreg1)
6351                                 amd64_sse_movaps_reg_reg (code, ins->dreg, ins->sreg1);
6352                         break;
6353                 case OP_XZERO:
6354                         amd64_sse_pxor_reg_reg (code, ins->dreg, ins->dreg);
6355                         break;
6356                 case OP_ICONV_TO_R8_RAW:
6357                         amd64_movd_xreg_reg_size (code, ins->dreg, ins->sreg1, 4);
6358                         amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
6359                         break;
6360
6361                 case OP_FCONV_TO_R8_X:
6362                         amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
6363                         break;
6364
6365                 case OP_XCONV_R8_TO_I4:
6366                         amd64_sse_cvttsd2si_reg_xreg_size (code, ins->dreg, ins->sreg1, 4);
6367                         switch (ins->backend.source_opcode) {
6368                         case OP_FCONV_TO_I1:
6369                                 amd64_widen_reg (code, ins->dreg, ins->dreg, TRUE, FALSE);
6370                                 break;
6371                         case OP_FCONV_TO_U1:
6372                                 amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
6373                                 break;
6374                         case OP_FCONV_TO_I2:
6375                                 amd64_widen_reg (code, ins->dreg, ins->dreg, TRUE, TRUE);
6376                                 break;
6377                         case OP_FCONV_TO_U2:
6378                                 amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, TRUE);
6379                                 break;
6380                         }
6381                         break;
6382
6383                 case OP_EXPAND_I2:
6384                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg1, 0);
6385                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg1, 1);
6386                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0);
6387                         break;
6388                 case OP_EXPAND_I4:
6389                         amd64_movd_xreg_reg_size (code, ins->dreg, ins->sreg1, 4);
6390                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0);
6391                         break;
6392                 case OP_EXPAND_I8:
6393                         amd64_movd_xreg_reg_size (code, ins->dreg, ins->sreg1, 8);
6394                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0x44);
6395                         break;
6396                 case OP_EXPAND_R4:
6397                         amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
6398                         amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->dreg);
6399                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0);
6400                         break;
6401                 case OP_EXPAND_R8:
6402                         amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
6403                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0x44);
6404                         break;
6405 #endif
6406                 case OP_LIVERANGE_START: {
6407                         if (cfg->verbose_level > 1)
6408                                 printf ("R%d START=0x%x\n", MONO_VARINFO (cfg, ins->inst_c0)->vreg, (int)(code - cfg->native_code));
6409                         MONO_VARINFO (cfg, ins->inst_c0)->live_range_start = code - cfg->native_code;
6410                         break;
6411                 }
6412                 case OP_LIVERANGE_END: {
6413                         if (cfg->verbose_level > 1)
6414                                 printf ("R%d END=0x%x\n", MONO_VARINFO (cfg, ins->inst_c0)->vreg, (int)(code - cfg->native_code));
6415                         MONO_VARINFO (cfg, ins->inst_c0)->live_range_end = code - cfg->native_code;
6416                         break;
6417                 }
6418                 case OP_NACL_GC_SAFE_POINT: {
6419 #if defined(__native_client_codegen__) && defined(__native_client_gc__)
6420                         if (cfg->compile_aot)
6421                                 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc, TRUE);
6422                         else {
6423                                 guint8 *br [1];
6424
6425                                 amd64_mov_reg_imm_size (code, AMD64_R11, (gpointer)&__nacl_thread_suspension_needed, 4);
6426                                 amd64_test_membase_imm_size (code, AMD64_R11, 0, 0xFFFFFFFF, 4);
6427                                 br[0] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE);
6428                                 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc, TRUE);
6429                                 amd64_patch (br[0], code);
6430                         }
6431 #endif
6432                         break;
6433                 }
6434                 case OP_GC_LIVENESS_DEF:
6435                 case OP_GC_LIVENESS_USE:
6436                 case OP_GC_PARAM_SLOT_LIVENESS_DEF:
6437                         ins->backend.pc_offset = code - cfg->native_code;
6438                         break;
6439                 case OP_GC_SPILL_SLOT_LIVENESS_DEF:
6440                         ins->backend.pc_offset = code - cfg->native_code;
6441                         bb->spill_slot_defs = g_slist_prepend_mempool (cfg->mempool, bb->spill_slot_defs, ins);
6442                         break;
6443                 default:
6444                         g_warning ("unknown opcode %s in %s()\n", mono_inst_name (ins->opcode), __FUNCTION__);
6445                         g_assert_not_reached ();
6446                 }
6447
6448                 if ((code - cfg->native_code - offset) > max_len) {
6449 #if !defined(__native_client_codegen__)
6450                         g_warning ("wrong maximal instruction length of instruction %s (expected %d, got %ld)",
6451                                    mono_inst_name (ins->opcode), max_len, code - cfg->native_code - offset);
6452                         g_assert_not_reached ();
6453 #endif
6454                 }
6455
6456                 last_ins = ins;
6457                 last_offset = offset;
6458         }
6459
6460         cfg->code_len = code - cfg->native_code;
6461 }
6462
6463 #endif /* DISABLE_JIT */
6464
6465 void
6466 mono_arch_register_lowlevel_calls (void)
6467 {
6468         /* The signature doesn't matter */
6469         mono_register_jit_icall (mono_amd64_throw_exception, "mono_amd64_throw_exception", mono_create_icall_signature ("void"), TRUE);
6470 }
6471
6472 void
6473 mono_arch_patch_code (MonoMethod *method, MonoDomain *domain, guint8 *code, MonoJumpInfo *ji, MonoCodeManager *dyn_code_mp, gboolean run_cctors)
6474 {
6475         MonoJumpInfo *patch_info;
6476         gboolean compile_aot = !run_cctors;
6477
6478         for (patch_info = ji; patch_info; patch_info = patch_info->next) {
6479                 unsigned char *ip = patch_info->ip.i + code;
6480                 unsigned char *target;
6481
6482                 target = mono_resolve_patch_target (method, domain, code, patch_info, run_cctors);
6483
6484                 if (compile_aot) {
6485                         switch (patch_info->type) {
6486                         case MONO_PATCH_INFO_BB:
6487                         case MONO_PATCH_INFO_LABEL:
6488                                 break;
6489                         default:
6490                                 /* No need to patch these */
6491                                 continue;
6492                         }
6493                 }
6494
6495                 switch (patch_info->type) {
6496                 case MONO_PATCH_INFO_NONE:
6497                         continue;
6498                 case MONO_PATCH_INFO_METHOD_REL:
6499                 case MONO_PATCH_INFO_R8:
6500                 case MONO_PATCH_INFO_R4:
6501                         g_assert_not_reached ();
6502                         continue;
6503                 case MONO_PATCH_INFO_BB:
6504                         break;
6505                 default:
6506                         break;
6507                 }
6508
6509                 /*
6510                  * Debug code to help track down problems where the target of a near call is
6511                  * is not valid.
6512                  */
6513                 if (amd64_is_near_call (ip)) {
6514                         gint64 disp = (guint8*)target - (guint8*)ip;
6515
6516                         if (!amd64_is_imm32 (disp)) {
6517                                 printf ("TYPE: %d\n", patch_info->type);
6518                                 switch (patch_info->type) {
6519                                 case MONO_PATCH_INFO_INTERNAL_METHOD:
6520                                         printf ("V: %s\n", patch_info->data.name);
6521                                         break;
6522                                 case MONO_PATCH_INFO_METHOD_JUMP:
6523                                 case MONO_PATCH_INFO_METHOD:
6524                                         printf ("V: %s\n", patch_info->data.method->name);
6525                                         break;
6526                                 default:
6527                                         break;
6528                                 }
6529                         }
6530                 }
6531
6532                 amd64_patch (ip, (gpointer)target);
6533         }
6534 }
6535
6536 #ifndef DISABLE_JIT
6537
6538 static int
6539 get_max_epilog_size (MonoCompile *cfg)
6540 {
6541         int max_epilog_size = 16;
6542
6543         if (cfg->method->save_lmf)
6544                 max_epilog_size += 256;
6545
6546         if (mono_jit_trace_calls != NULL)
6547                 max_epilog_size += 50;
6548
6549         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
6550                 max_epilog_size += 50;
6551
6552         max_epilog_size += (AMD64_NREG * 2);
6553
6554         return max_epilog_size;
6555 }
6556
6557 /*
6558  * This macro is used for testing whenever the unwinder works correctly at every point
6559  * where an async exception can happen.
6560  */
6561 /* This will generate a SIGSEGV at the given point in the code */
6562 #define async_exc_point(code) do { \
6563     if (mono_inject_async_exc_method && mono_method_desc_full_match (mono_inject_async_exc_method, cfg->method)) { \
6564          if (cfg->arch.async_point_count == mono_inject_async_exc_pos) \
6565              amd64_mov_reg_mem (code, AMD64_RAX, 0, 4); \
6566          cfg->arch.async_point_count ++; \
6567     } \
6568 } while (0)
6569
6570 guint8 *
6571 mono_arch_emit_prolog (MonoCompile *cfg)
6572 {
6573         MonoMethod *method = cfg->method;
6574         MonoBasicBlock *bb;
6575         MonoMethodSignature *sig;
6576         MonoInst *ins;
6577         int alloc_size, pos, i, cfa_offset, quad, max_epilog_size;
6578         guint8 *code;
6579         CallInfo *cinfo;
6580         MonoInst *lmf_var = cfg->arch.lmf_var;
6581         gboolean args_clobbered = FALSE;
6582         gboolean trace = FALSE;
6583 #ifdef __native_client_codegen__
6584         guint alignment_check;
6585 #endif
6586
6587         cfg->code_size =  MAX (cfg->header->code_size * 4, 10240);
6588
6589 #if defined(__default_codegen__)
6590         code = cfg->native_code = g_malloc (cfg->code_size);
6591 #elif defined(__native_client_codegen__)
6592         /* native_code_alloc is not 32-byte aligned, native_code is. */
6593         cfg->native_code_alloc = g_malloc (cfg->code_size + kNaClAlignment);
6594
6595         /* Align native_code to next nearest kNaclAlignment byte. */
6596         cfg->native_code = (uintptr_t)cfg->native_code_alloc + kNaClAlignment;
6597         cfg->native_code = (uintptr_t)cfg->native_code & ~kNaClAlignmentMask;
6598
6599         code = cfg->native_code;
6600
6601         alignment_check = (guint)cfg->native_code & kNaClAlignmentMask;
6602         g_assert (alignment_check == 0);
6603 #endif
6604
6605         if (mono_jit_trace_calls != NULL && mono_trace_eval (method))
6606                 trace = TRUE;
6607
6608         /* Amount of stack space allocated by register saving code */
6609         pos = 0;
6610
6611         /* Offset between RSP and the CFA */
6612         cfa_offset = 0;
6613
6614         /*
6615          * The prolog consists of the following parts:
6616          * FP present:
6617          * - push rbp, mov rbp, rsp
6618          * - save callee saved regs using pushes
6619          * - allocate frame
6620          * - save rgctx if needed
6621          * - save lmf if needed
6622          * FP not present:
6623          * - allocate frame
6624          * - save rgctx if needed
6625          * - save lmf if needed
6626          * - save callee saved regs using moves
6627          */
6628
6629         // CFA = sp + 8
6630         cfa_offset = 8;
6631         mono_emit_unwind_op_def_cfa (cfg, code, AMD64_RSP, 8);
6632         // IP saved at CFA - 8
6633         mono_emit_unwind_op_offset (cfg, code, AMD64_RIP, -cfa_offset);
6634         async_exc_point (code);
6635         mini_gc_set_slot_type_from_cfa (cfg, -cfa_offset, SLOT_NOREF);
6636
6637         if (!cfg->arch.omit_fp) {
6638                 amd64_push_reg (code, AMD64_RBP);
6639                 cfa_offset += 8;
6640                 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
6641                 mono_emit_unwind_op_offset (cfg, code, AMD64_RBP, - cfa_offset);
6642                 async_exc_point (code);
6643 #ifdef HOST_WIN32
6644                 mono_arch_unwindinfo_add_push_nonvol (&cfg->arch.unwindinfo, cfg->native_code, code, AMD64_RBP);
6645 #endif
6646                 /* These are handled automatically by the stack marking code */
6647                 mini_gc_set_slot_type_from_cfa (cfg, -cfa_offset, SLOT_NOREF);
6648
6649                 amd64_mov_reg_reg (code, AMD64_RBP, AMD64_RSP, sizeof(mgreg_t));
6650                 mono_emit_unwind_op_def_cfa_reg (cfg, code, AMD64_RBP);
6651                 async_exc_point (code);
6652 #ifdef HOST_WIN32
6653                 mono_arch_unwindinfo_add_set_fpreg (&cfg->arch.unwindinfo, cfg->native_code, code, AMD64_RBP);
6654 #endif
6655         }
6656
6657         /* Save callee saved registers */
6658         if (!cfg->arch.omit_fp && !method->save_lmf) {
6659                 int offset = cfa_offset;
6660
6661                 for (i = 0; i < AMD64_NREG; ++i)
6662                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
6663                                 amd64_push_reg (code, i);
6664                                 pos += 8; /* AMD64 push inst is always 8 bytes, no way to change it */
6665                                 offset += 8;
6666                                 mono_emit_unwind_op_offset (cfg, code, i, - offset);
6667                                 async_exc_point (code);
6668
6669                                 /* These are handled automatically by the stack marking code */
6670                                 mini_gc_set_slot_type_from_cfa (cfg, - offset, SLOT_NOREF);
6671                         }
6672         }
6673
6674         /* The param area is always at offset 0 from sp */
6675         /* This needs to be allocated here, since it has to come after the spill area */
6676         if (cfg->arch.no_pushes && cfg->param_area) {
6677                 if (cfg->arch.omit_fp)
6678                         // FIXME:
6679                         g_assert_not_reached ();
6680                 cfg->stack_offset += ALIGN_TO (cfg->param_area, sizeof(mgreg_t));
6681         }
6682
6683         if (cfg->arch.omit_fp) {
6684                 /*
6685                  * On enter, the stack is misaligned by the pushing of the return
6686                  * address. It is either made aligned by the pushing of %rbp, or by
6687                  * this.
6688                  */
6689                 alloc_size = ALIGN_TO (cfg->stack_offset, 8);
6690                 if ((alloc_size % 16) == 0) {
6691                         alloc_size += 8;
6692                         /* Mark the padding slot as NOREF */
6693                         mini_gc_set_slot_type_from_cfa (cfg, -cfa_offset - sizeof (mgreg_t), SLOT_NOREF);
6694                 }
6695         } else {
6696                 alloc_size = ALIGN_TO (cfg->stack_offset, MONO_ARCH_FRAME_ALIGNMENT);
6697                 if (cfg->stack_offset != alloc_size) {
6698                         /* Mark the padding slot as NOREF */
6699                         mini_gc_set_slot_type_from_fp (cfg, -alloc_size + cfg->param_area, SLOT_NOREF);
6700                 }
6701                 cfg->arch.sp_fp_offset = alloc_size;
6702                 alloc_size -= pos;
6703         }
6704
6705         cfg->arch.stack_alloc_size = alloc_size;
6706
6707         /* Allocate stack frame */
6708         if (alloc_size) {
6709                 /* See mono_emit_stack_alloc */
6710 #if defined(HOST_WIN32) || defined(MONO_ARCH_SIGSEGV_ON_ALTSTACK)
6711                 guint32 remaining_size = alloc_size;
6712                 /*FIXME handle unbounded code expansion, we should use a loop in case of more than X interactions*/
6713                 guint32 required_code_size = ((remaining_size / 0x1000) + 1) * 10; /*10 is the max size of amd64_alu_reg_imm + amd64_test_membase_reg*/
6714                 guint32 offset = code - cfg->native_code;
6715                 if (G_UNLIKELY (required_code_size >= (cfg->code_size - offset))) {
6716                         while (required_code_size >= (cfg->code_size - offset))
6717                                 cfg->code_size *= 2;
6718                         cfg->native_code = mono_realloc_native_code (cfg);
6719                         code = cfg->native_code + offset;
6720                         cfg->stat_code_reallocs++;
6721                 }
6722
6723                 while (remaining_size >= 0x1000) {
6724                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 0x1000);
6725                         if (cfg->arch.omit_fp) {
6726                                 cfa_offset += 0x1000;
6727                                 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
6728                         }
6729                         async_exc_point (code);
6730 #ifdef HOST_WIN32
6731                         if (cfg->arch.omit_fp)
6732                                 mono_arch_unwindinfo_add_alloc_stack (&cfg->arch.unwindinfo, cfg->native_code, code, 0x1000);
6733 #endif
6734
6735                         amd64_test_membase_reg (code, AMD64_RSP, 0, AMD64_RSP);
6736                         remaining_size -= 0x1000;
6737                 }
6738                 if (remaining_size) {
6739                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, remaining_size);
6740                         if (cfg->arch.omit_fp) {
6741                                 cfa_offset += remaining_size;
6742                                 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
6743                                 async_exc_point (code);
6744                         }
6745 #ifdef HOST_WIN32
6746                         if (cfg->arch.omit_fp)
6747                                 mono_arch_unwindinfo_add_alloc_stack (&cfg->arch.unwindinfo, cfg->native_code, code, remaining_size);
6748 #endif
6749                 }
6750 #else
6751                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, alloc_size);
6752                 if (cfg->arch.omit_fp) {
6753                         cfa_offset += alloc_size;
6754                         mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
6755                         async_exc_point (code);
6756                 }
6757 #endif
6758         }
6759
6760         /* Stack alignment check */
6761 #if 0
6762         {
6763                 amd64_mov_reg_reg (code, AMD64_RAX, AMD64_RSP, 8);
6764                 amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, 0xf);
6765                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0);
6766                 x86_branch8 (code, X86_CC_EQ, 2, FALSE);
6767                 amd64_breakpoint (code);
6768         }
6769 #endif
6770
6771 #ifndef TARGET_WIN32
6772         if (mini_get_debug_options ()->init_stacks) {
6773                 /* Fill the stack frame with a dummy value to force deterministic behavior */
6774
6775                 /* Save registers to the red zone */
6776                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDI, 8);
6777                 amd64_mov_membase_reg (code, AMD64_RSP, -16, AMD64_RCX, 8);
6778
6779                 amd64_mov_reg_imm (code, AMD64_RAX, 0x2a2a2a2a2a2a2a2a);
6780                 amd64_mov_reg_imm (code, AMD64_RCX, alloc_size / 8);
6781                 amd64_mov_reg_reg (code, AMD64_RDI, AMD64_RSP, 8);
6782
6783                 amd64_cld (code);
6784 #if defined(__default_codegen__)
6785                 amd64_prefix (code, X86_REP_PREFIX);
6786                 amd64_stosl (code);
6787 #elif defined(__native_client_codegen__)
6788                 /* NaCl stos pseudo-instruction */
6789                 amd64_codegen_pre (code);
6790                 /* First, clear the upper 32 bits of RDI (mov %edi, %edi)  */
6791                 amd64_mov_reg_reg (code, AMD64_RDI, AMD64_RDI, 4);
6792                 /* Add %r15 to %rdi using lea, condition flags unaffected. */
6793                 amd64_lea_memindex_size (code, AMD64_RDI, AMD64_R15, 0, AMD64_RDI, 0, 8);
6794                 amd64_prefix (code, X86_REP_PREFIX);
6795                 amd64_stosl (code);
6796                 amd64_codegen_post (code);
6797 #endif /* __native_client_codegen__ */
6798
6799                 amd64_mov_reg_membase (code, AMD64_RDI, AMD64_RSP, -8, 8);
6800                 amd64_mov_reg_membase (code, AMD64_RCX, AMD64_RSP, -16, 8);
6801         }
6802 #endif
6803
6804         /* Save LMF */
6805         if (method->save_lmf) {
6806                 code = emit_setup_lmf (cfg, code, lmf_var->inst_offset, cfa_offset);
6807         }
6808
6809         /* Save callee saved registers */
6810         if (cfg->arch.omit_fp && !method->save_lmf) {
6811                 gint32 save_area_offset = cfg->arch.reg_save_area_offset;
6812
6813                 /* Save caller saved registers after sp is adjusted */
6814                 /* The registers are saved at the bottom of the frame */
6815                 /* FIXME: Optimize this so the regs are saved at the end of the frame in increasing order */
6816                 for (i = 0; i < AMD64_NREG; ++i)
6817                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
6818                                 amd64_mov_membase_reg (code, AMD64_RSP, save_area_offset, i, 8);
6819                                 mono_emit_unwind_op_offset (cfg, code, i, - (cfa_offset - save_area_offset));
6820
6821                                 /* These are handled automatically by the stack marking code */
6822                                 mini_gc_set_slot_type_from_cfa (cfg, - (cfa_offset - save_area_offset), SLOT_NOREF);
6823
6824                                 save_area_offset += 8;
6825                                 async_exc_point (code);
6826                         }
6827         }
6828
6829         /* store runtime generic context */
6830         if (cfg->rgctx_var) {
6831                 g_assert (cfg->rgctx_var->opcode == OP_REGOFFSET &&
6832                                 (cfg->rgctx_var->inst_basereg == AMD64_RBP || cfg->rgctx_var->inst_basereg == AMD64_RSP));
6833
6834                 amd64_mov_membase_reg (code, cfg->rgctx_var->inst_basereg, cfg->rgctx_var->inst_offset, MONO_ARCH_RGCTX_REG, sizeof(gpointer));
6835
6836                 mono_add_var_location (cfg, cfg->rgctx_var, TRUE, MONO_ARCH_RGCTX_REG, 0, 0, code - cfg->native_code);
6837                 mono_add_var_location (cfg, cfg->rgctx_var, FALSE, cfg->rgctx_var->inst_basereg, cfg->rgctx_var->inst_offset, code - cfg->native_code, 0);
6838         }
6839
6840         /* compute max_length in order to use short forward jumps */
6841         max_epilog_size = get_max_epilog_size (cfg);
6842         if (cfg->opt & MONO_OPT_BRANCH) {
6843                 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6844                         MonoInst *ins;
6845                         int max_length = 0;
6846
6847                         if (cfg->prof_options & MONO_PROFILE_COVERAGE)
6848                                 max_length += 6;
6849                         /* max alignment for loops */
6850                         if ((cfg->opt & MONO_OPT_LOOP) && bb_is_loop_start (bb))
6851                                 max_length += LOOP_ALIGNMENT;
6852 #ifdef __native_client_codegen__
6853                         /* max alignment for native client */
6854                         max_length += kNaClAlignment;
6855 #endif
6856
6857                         MONO_BB_FOR_EACH_INS (bb, ins) {
6858 #ifdef __native_client_codegen__
6859                                 {
6860                                         int space_in_block = kNaClAlignment -
6861                                                 ((max_length + cfg->code_len) & kNaClAlignmentMask);
6862                                         int max_len = ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
6863                                         if (space_in_block < max_len && max_len < kNaClAlignment) {
6864                                                 max_length += space_in_block;
6865                                         }
6866                                 }
6867 #endif  /*__native_client_codegen__*/
6868                                 max_length += ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
6869                         }
6870
6871                         /* Take prolog and epilog instrumentation into account */
6872                         if (bb == cfg->bb_entry || bb == cfg->bb_exit)
6873                                 max_length += max_epilog_size;
6874
6875                         bb->max_length = max_length;
6876                 }
6877         }
6878
6879         sig = mono_method_signature (method);
6880         pos = 0;
6881
6882         cinfo = cfg->arch.cinfo;
6883
6884         if (sig->ret->type != MONO_TYPE_VOID) {
6885                 /* Save volatile arguments to the stack */
6886                 if (cfg->vret_addr && (cfg->vret_addr->opcode != OP_REGVAR))
6887                         amd64_mov_membase_reg (code, cfg->vret_addr->inst_basereg, cfg->vret_addr->inst_offset, cinfo->ret.reg, 8);
6888         }
6889
6890         /* Keep this in sync with emit_load_volatile_arguments */
6891         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
6892                 ArgInfo *ainfo = cinfo->args + i;
6893                 gint32 stack_offset;
6894                 MonoType *arg_type;
6895
6896                 ins = cfg->args [i];
6897
6898                 if ((ins->flags & MONO_INST_IS_DEAD) && !trace)
6899                         /* Unused arguments */
6900                         continue;
6901
6902                 if (sig->hasthis && (i == 0))
6903                         arg_type = &mono_defaults.object_class->byval_arg;
6904                 else
6905                         arg_type = sig->params [i - sig->hasthis];
6906
6907                 stack_offset = ainfo->offset + ARGS_OFFSET;
6908
6909                 if (cfg->globalra) {
6910                         /* All the other moves are done by the register allocator */
6911                         switch (ainfo->storage) {
6912                         case ArgInFloatSSEReg:
6913                                 amd64_sse_cvtss2sd_reg_reg (code, ainfo->reg, ainfo->reg);
6914                                 break;
6915                         case ArgValuetypeInReg:
6916                                 for (quad = 0; quad < 2; quad ++) {
6917                                         switch (ainfo->pair_storage [quad]) {
6918                                         case ArgInIReg:
6919                                                 amd64_mov_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof(mgreg_t)), ainfo->pair_regs [quad], sizeof(mgreg_t));
6920                                                 break;
6921                                         case ArgInFloatSSEReg:
6922                                                 amd64_movss_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof(mgreg_t)), ainfo->pair_regs [quad]);
6923                                                 break;
6924                                         case ArgInDoubleSSEReg:
6925                                                 amd64_movsd_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof(mgreg_t)), ainfo->pair_regs [quad]);
6926                                                 break;
6927                                         case ArgNone:
6928                                                 break;
6929                                         default:
6930                                                 g_assert_not_reached ();
6931                                         }
6932                                 }
6933                                 break;
6934                         default:
6935                                 break;
6936                         }
6937
6938                         continue;
6939                 }
6940
6941                 /* Save volatile arguments to the stack */
6942                 if (ins->opcode != OP_REGVAR) {
6943                         switch (ainfo->storage) {
6944                         case ArgInIReg: {
6945                                 guint32 size = 8;
6946
6947                                 /* FIXME: I1 etc */
6948                                 /*
6949                                 if (stack_offset & 0x1)
6950                                         size = 1;
6951                                 else if (stack_offset & 0x2)
6952                                         size = 2;
6953                                 else if (stack_offset & 0x4)
6954                                         size = 4;
6955                                 else
6956                                         size = 8;
6957                                 */
6958                                 amd64_mov_membase_reg (code, ins->inst_basereg, ins->inst_offset, ainfo->reg, size);
6959
6960                                 /*
6961                                  * Save the original location of 'this',
6962                                  * get_generic_info_from_stack_frame () needs this to properly look up
6963                                  * the argument value during the handling of async exceptions.
6964                                  */
6965                                 if (ins == cfg->args [0]) {
6966                                         mono_add_var_location (cfg, ins, TRUE, ainfo->reg, 0, 0, code - cfg->native_code);
6967                                         mono_add_var_location (cfg, ins, FALSE, ins->inst_basereg, ins->inst_offset, code - cfg->native_code, 0);
6968                                 }
6969                                 break;
6970                         }
6971                         case ArgInFloatSSEReg:
6972                                 amd64_movss_membase_reg (code, ins->inst_basereg, ins->inst_offset, ainfo->reg);
6973                                 break;
6974                         case ArgInDoubleSSEReg:
6975                                 amd64_movsd_membase_reg (code, ins->inst_basereg, ins->inst_offset, ainfo->reg);
6976                                 break;
6977                         case ArgValuetypeInReg:
6978                                 for (quad = 0; quad < 2; quad ++) {
6979                                         switch (ainfo->pair_storage [quad]) {
6980                                         case ArgInIReg:
6981                                                 amd64_mov_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof(mgreg_t)), ainfo->pair_regs [quad], sizeof(mgreg_t));
6982                                                 break;
6983                                         case ArgInFloatSSEReg:
6984                                                 amd64_movss_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof(mgreg_t)), ainfo->pair_regs [quad]);
6985                                                 break;
6986                                         case ArgInDoubleSSEReg:
6987                                                 amd64_movsd_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof(mgreg_t)), ainfo->pair_regs [quad]);
6988                                                 break;
6989                                         case ArgNone:
6990                                                 break;
6991                                         default:
6992                                                 g_assert_not_reached ();
6993                                         }
6994                                 }
6995                                 break;
6996                         case ArgValuetypeAddrInIReg:
6997                                 if (ainfo->pair_storage [0] == ArgInIReg)
6998                                         amd64_mov_membase_reg (code, ins->inst_left->inst_basereg, ins->inst_left->inst_offset, ainfo->pair_regs [0],  sizeof (gpointer));
6999                                 break;
7000                         default:
7001                                 break;
7002                         }
7003                 } else {
7004                         /* Argument allocated to (non-volatile) register */
7005                         switch (ainfo->storage) {
7006                         case ArgInIReg:
7007                                 amd64_mov_reg_reg (code, ins->dreg, ainfo->reg, 8);
7008                                 break;
7009                         case ArgOnStack:
7010                                 amd64_mov_reg_membase (code, ins->dreg, AMD64_RBP, ARGS_OFFSET + ainfo->offset, 8);
7011                                 break;
7012                         default:
7013                                 g_assert_not_reached ();
7014                         }
7015
7016                         if (ins == cfg->args [0]) {
7017                                 mono_add_var_location (cfg, ins, TRUE, ainfo->reg, 0, 0, code - cfg->native_code);
7018                                 mono_add_var_location (cfg, ins, TRUE, ins->dreg, 0, code - cfg->native_code, 0);
7019                         }
7020                 }
7021         }
7022
7023         if (method->save_lmf) {
7024                 code = emit_save_lmf (cfg, code, lmf_var->inst_offset, &args_clobbered);
7025         }
7026
7027         if (trace) {
7028                 args_clobbered = TRUE;
7029                 code = mono_arch_instrument_prolog (cfg, mono_trace_enter_method, code, TRUE);
7030         }
7031
7032         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
7033                 args_clobbered = TRUE;
7034
7035         /*
7036          * Optimize the common case of the first bblock making a call with the same
7037          * arguments as the method. This works because the arguments are still in their
7038          * original argument registers.
7039          * FIXME: Generalize this
7040          */
7041         if (!args_clobbered) {
7042                 MonoBasicBlock *first_bb = cfg->bb_entry;
7043                 MonoInst *next;
7044
7045                 next = mono_bb_first_ins (first_bb);
7046                 if (!next && first_bb->next_bb) {
7047                         first_bb = first_bb->next_bb;
7048                         next = mono_bb_first_ins (first_bb);
7049                 }
7050
7051                 if (first_bb->in_count > 1)
7052                         next = NULL;
7053
7054                 for (i = 0; next && i < sig->param_count + sig->hasthis; ++i) {
7055                         ArgInfo *ainfo = cinfo->args + i;
7056                         gboolean match = FALSE;
7057
7058                         ins = cfg->args [i];
7059                         if (ins->opcode != OP_REGVAR) {
7060                                 switch (ainfo->storage) {
7061                                 case ArgInIReg: {
7062                                         if (((next->opcode == OP_LOAD_MEMBASE) || (next->opcode == OP_LOADI4_MEMBASE)) && next->inst_basereg == ins->inst_basereg && next->inst_offset == ins->inst_offset) {
7063                                                 if (next->dreg == ainfo->reg) {
7064                                                         NULLIFY_INS (next);
7065                                                         match = TRUE;
7066                                                 } else {
7067                                                         next->opcode = OP_MOVE;
7068                                                         next->sreg1 = ainfo->reg;
7069                                                         /* Only continue if the instruction doesn't change argument regs */
7070                                                         if (next->dreg == ainfo->reg || next->dreg == AMD64_RAX)
7071                                                                 match = TRUE;
7072                                                 }
7073                                         }
7074                                         break;
7075                                 }
7076                                 default:
7077                                         break;
7078                                 }
7079                         } else {
7080                                 /* Argument allocated to (non-volatile) register */
7081                                 switch (ainfo->storage) {
7082                                 case ArgInIReg:
7083                                         if (next->opcode == OP_MOVE && next->sreg1 == ins->dreg && next->dreg == ainfo->reg) {
7084                                                 NULLIFY_INS (next);
7085                                                 match = TRUE;
7086                                         }
7087                                         break;
7088                                 default:
7089                                         break;
7090                                 }
7091                         }
7092
7093                         if (match) {
7094                                 next = next->next;
7095                                 //next = mono_inst_list_next (&next->node, &first_bb->ins_list);
7096                                 if (!next)
7097                                         break;
7098                         }
7099                 }
7100         }
7101
7102         if (cfg->gen_seq_points) {
7103                 MonoInst *info_var = cfg->arch.seq_point_info_var;
7104
7105                 /* Initialize seq_point_info_var */
7106                 if (cfg->compile_aot) {
7107                         /* Initialize the variable from a GOT slot */
7108                         /* Same as OP_AOTCONST */
7109                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_SEQ_POINT_INFO, cfg->method);
7110                         amd64_mov_reg_membase (code, AMD64_R11, AMD64_RIP, 0, sizeof(gpointer));
7111                         g_assert (info_var->opcode == OP_REGOFFSET);
7112                         amd64_mov_membase_reg (code, info_var->inst_basereg, info_var->inst_offset, AMD64_R11, 8);
7113                 }
7114
7115                 /* Initialize ss_trigger_page_var */
7116                 ins = cfg->arch.ss_trigger_page_var;
7117
7118                 g_assert (ins->opcode == OP_REGOFFSET);
7119
7120                 if (cfg->compile_aot) {
7121                         amd64_mov_reg_membase (code, AMD64_R11, info_var->inst_basereg, info_var->inst_offset, 8);
7122                         amd64_mov_reg_membase (code, AMD64_R11, AMD64_R11, G_STRUCT_OFFSET (SeqPointInfo, ss_trigger_page), 8);
7123                 } else {
7124                         amd64_mov_reg_imm (code, AMD64_R11, (guint64)ss_trigger_page);
7125                 }
7126                 amd64_mov_membase_reg (code, ins->inst_basereg, ins->inst_offset, AMD64_R11, 8);
7127         }
7128
7129         cfg->code_len = code - cfg->native_code;
7130
7131         g_assert (cfg->code_len < cfg->code_size);
7132
7133         return code;
7134 }
7135
7136 void
7137 mono_arch_emit_epilog (MonoCompile *cfg)
7138 {
7139         MonoMethod *method = cfg->method;
7140         int quad, pos, i;
7141         guint8 *code;
7142         int max_epilog_size;
7143         CallInfo *cinfo;
7144         gint32 lmf_offset = cfg->arch.lmf_var ? ((MonoInst*)cfg->arch.lmf_var)->inst_offset : -1;
7145
7146         max_epilog_size = get_max_epilog_size (cfg);
7147
7148         while (cfg->code_len + max_epilog_size > (cfg->code_size - 16)) {
7149                 cfg->code_size *= 2;
7150                 cfg->native_code = mono_realloc_native_code (cfg);
7151                 cfg->stat_code_reallocs++;
7152         }
7153
7154         code = cfg->native_code + cfg->code_len;
7155
7156         if (mono_jit_trace_calls != NULL && mono_trace_eval (method))
7157                 code = mono_arch_instrument_epilog (cfg, mono_trace_leave_method, code, TRUE);
7158
7159         /* the code restoring the registers must be kept in sync with OP_JMP */
7160         pos = 0;
7161
7162         if (method->save_lmf) {
7163                 /* check if we need to restore protection of the stack after a stack overflow */
7164                 if (mono_get_jit_tls_offset () != -1) {
7165                         guint8 *patch;
7166                         code = mono_amd64_emit_tls_get (code, AMD64_RCX, mono_get_jit_tls_offset ());
7167                         /* we load the value in a separate instruction: this mechanism may be
7168                          * used later as a safer way to do thread interruption
7169                          */
7170                         amd64_mov_reg_membase (code, AMD64_RCX, AMD64_RCX, G_STRUCT_OFFSET (MonoJitTlsData, restore_stack_prot), 8);
7171                         x86_alu_reg_imm (code, X86_CMP, X86_ECX, 0);
7172                         patch = code;
7173                         x86_branch8 (code, X86_CC_Z, 0, FALSE);
7174                         /* note that the call trampoline will preserve eax/edx */
7175                         x86_call_reg (code, X86_ECX);
7176                         x86_patch (patch, code);
7177                 } else {
7178                         /* FIXME: maybe save the jit tls in the prolog */
7179                 }
7180
7181                 code = emit_restore_lmf (cfg, code, lmf_offset);
7182
7183                 /* Restore caller saved regs */
7184                 if (cfg->used_int_regs & (1 << AMD64_RBP)) {
7185                         amd64_mov_reg_membase (code, AMD64_RBP, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbp), 8);
7186                 }
7187                 if (cfg->used_int_regs & (1 << AMD64_RBX)) {
7188                         amd64_mov_reg_membase (code, AMD64_RBX, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbx), 8);
7189                 }
7190                 if (cfg->used_int_regs & (1 << AMD64_R12)) {
7191                         amd64_mov_reg_membase (code, AMD64_R12, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r12), 8);
7192                 }
7193                 if (cfg->used_int_regs & (1 << AMD64_R13)) {
7194                         amd64_mov_reg_membase (code, AMD64_R13, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r13), 8);
7195                 }
7196                 if (cfg->used_int_regs & (1 << AMD64_R14)) {
7197                         amd64_mov_reg_membase (code, AMD64_R14, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r14), 8);
7198                 }
7199                 if (cfg->used_int_regs & (1 << AMD64_R15)) {
7200 #if defined(__default_codegen__)
7201                         amd64_mov_reg_membase (code, AMD64_R15, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r15), 8);
7202 #elif defined(__native_client_codegen__)
7203                         g_assert_not_reached();
7204 #endif
7205                 }
7206 #ifdef HOST_WIN32
7207                 if (cfg->used_int_regs & (1 << AMD64_RDI)) {
7208                         amd64_mov_reg_membase (code, AMD64_RDI, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rdi), 8);
7209                 }
7210                 if (cfg->used_int_regs & (1 << AMD64_RSI)) {
7211                         amd64_mov_reg_membase (code, AMD64_RSI, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rsi), 8);
7212                 }
7213 #endif
7214         } else {
7215
7216                 if (cfg->arch.omit_fp) {
7217                         gint32 save_area_offset = cfg->arch.reg_save_area_offset;
7218
7219                         for (i = 0; i < AMD64_NREG; ++i)
7220                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
7221                                         amd64_mov_reg_membase (code, i, AMD64_RSP, save_area_offset, 8);
7222                                         save_area_offset += 8;
7223                                 }
7224                 }
7225                 else {
7226                         for (i = 0; i < AMD64_NREG; ++i)
7227                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i)))
7228                                         pos -= sizeof(mgreg_t);
7229
7230                         if (pos) {
7231                                 if (pos == - sizeof(mgreg_t)) {
7232                                         /* Only one register, so avoid lea */
7233                                         for (i = AMD64_NREG - 1; i > 0; --i)
7234                                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
7235                                                         amd64_mov_reg_membase (code, i, AMD64_RBP, pos, 8);
7236                                                 }
7237                                 }
7238                                 else {
7239                                         amd64_lea_membase (code, AMD64_RSP, AMD64_RBP, pos);
7240
7241                                         /* Pop registers in reverse order */
7242                                         for (i = AMD64_NREG - 1; i > 0; --i)
7243                                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
7244                                                         amd64_pop_reg (code, i);
7245                                                 }
7246                                 }
7247                         }
7248                 }
7249         }
7250
7251         /* Load returned vtypes into registers if needed */
7252         cinfo = cfg->arch.cinfo;
7253         if (cinfo->ret.storage == ArgValuetypeInReg) {
7254                 ArgInfo *ainfo = &cinfo->ret;
7255                 MonoInst *inst = cfg->ret;
7256
7257                 for (quad = 0; quad < 2; quad ++) {
7258                         switch (ainfo->pair_storage [quad]) {
7259                         case ArgInIReg:
7260                                 amd64_mov_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof(mgreg_t)), sizeof(mgreg_t));
7261                                 break;
7262                         case ArgInFloatSSEReg:
7263                                 amd64_movss_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof(mgreg_t)));
7264                                 break;
7265                         case ArgInDoubleSSEReg:
7266                                 amd64_movsd_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof(mgreg_t)));
7267                                 break;
7268                         case ArgNone:
7269                                 break;
7270                         default:
7271                                 g_assert_not_reached ();
7272                         }
7273                 }
7274         }
7275
7276         if (cfg->arch.omit_fp) {
7277                 if (cfg->arch.stack_alloc_size)
7278                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, cfg->arch.stack_alloc_size);
7279         } else {
7280                 amd64_leave (code);
7281         }
7282         async_exc_point (code);
7283         amd64_ret (code);
7284
7285         cfg->code_len = code - cfg->native_code;
7286
7287         g_assert (cfg->code_len < cfg->code_size);
7288 }
7289
7290 void
7291 mono_arch_emit_exceptions (MonoCompile *cfg)
7292 {
7293         MonoJumpInfo *patch_info;
7294         int nthrows, i;
7295         guint8 *code;
7296         MonoClass *exc_classes [16];
7297         guint8 *exc_throw_start [16], *exc_throw_end [16];
7298         guint32 code_size = 0;
7299
7300         /* Compute needed space */
7301         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
7302                 if (patch_info->type == MONO_PATCH_INFO_EXC)
7303                         code_size += 40;
7304                 if (patch_info->type == MONO_PATCH_INFO_R8)
7305                         code_size += 8 + 15; /* sizeof (double) + alignment */
7306                 if (patch_info->type == MONO_PATCH_INFO_R4)
7307                         code_size += 4 + 15; /* sizeof (float) + alignment */
7308                 if (patch_info->type == MONO_PATCH_INFO_GC_CARD_TABLE_ADDR)
7309                         code_size += 8 + 7; /*sizeof (void*) + alignment */
7310         }
7311
7312 #ifdef __native_client_codegen__
7313         /* Give us extra room on Native Client.  This could be   */
7314         /* more carefully calculated, but bundle alignment makes */
7315         /* it much trickier, so *2 like other places is good.    */
7316         code_size *= 2;
7317 #endif
7318
7319         while (cfg->code_len + code_size > (cfg->code_size - 16)) {
7320                 cfg->code_size *= 2;
7321                 cfg->native_code = mono_realloc_native_code (cfg);
7322                 cfg->stat_code_reallocs++;
7323         }
7324
7325         code = cfg->native_code + cfg->code_len;
7326
7327         /* add code to raise exceptions */
7328         nthrows = 0;
7329         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
7330                 switch (patch_info->type) {
7331                 case MONO_PATCH_INFO_EXC: {
7332                         MonoClass *exc_class;
7333                         guint8 *buf, *buf2;
7334                         guint32 throw_ip;
7335
7336                         amd64_patch (patch_info->ip.i + cfg->native_code, code);
7337
7338                         exc_class = mono_class_from_name (mono_defaults.corlib, "System", patch_info->data.name);
7339                         g_assert (exc_class);
7340                         throw_ip = patch_info->ip.i;
7341
7342                         //x86_breakpoint (code);
7343                         /* Find a throw sequence for the same exception class */
7344                         for (i = 0; i < nthrows; ++i)
7345                                 if (exc_classes [i] == exc_class)
7346                                         break;
7347                         if (i < nthrows) {
7348                                 amd64_mov_reg_imm (code, AMD64_ARG_REG2, (exc_throw_end [i] - cfg->native_code) - throw_ip);
7349                                 x86_jump_code (code, exc_throw_start [i]);
7350                                 patch_info->type = MONO_PATCH_INFO_NONE;
7351                         }
7352                         else {
7353                                 buf = code;
7354                                 amd64_mov_reg_imm_size (code, AMD64_ARG_REG2, 0xf0f0f0f0, 4);
7355                                 buf2 = code;
7356
7357                                 if (nthrows < 16) {
7358                                         exc_classes [nthrows] = exc_class;
7359                                         exc_throw_start [nthrows] = code;
7360                                 }
7361                                 amd64_mov_reg_imm (code, AMD64_ARG_REG1, exc_class->type_token - MONO_TOKEN_TYPE_DEF);
7362
7363                                 patch_info->type = MONO_PATCH_INFO_NONE;
7364
7365                                 code = emit_call_body (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_corlib_exception");
7366
7367                                 amd64_mov_reg_imm (buf, AMD64_ARG_REG2, (code - cfg->native_code) - throw_ip);
7368                                 while (buf < buf2)
7369                                         x86_nop (buf);
7370
7371                                 if (nthrows < 16) {
7372                                         exc_throw_end [nthrows] = code;
7373                                         nthrows ++;
7374                                 }
7375                         }
7376                         break;
7377                 }
7378                 default:
7379                         /* do nothing */
7380                         break;
7381                 }
7382                 g_assert(code < cfg->native_code + cfg->code_size);
7383         }
7384
7385         /* Handle relocations with RIP relative addressing */
7386         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
7387                 gboolean remove = FALSE;
7388                 guint8 *orig_code = code;
7389
7390                 switch (patch_info->type) {
7391                 case MONO_PATCH_INFO_R8:
7392                 case MONO_PATCH_INFO_R4: {
7393                         guint8 *pos, *patch_pos;
7394                         guint32 target_pos;
7395
7396                         /* The SSE opcodes require a 16 byte alignment */
7397 #if defined(__default_codegen__)
7398                         code = (guint8*)ALIGN_TO (code, 16);
7399 #elif defined(__native_client_codegen__)
7400                         {
7401                                 /* Pad this out with HLT instructions  */
7402                                 /* or we can get garbage bytes emitted */
7403                                 /* which will fail validation          */
7404                                 guint8 *aligned_code;
7405                                 /* extra align to make room for  */
7406                                 /* mov/push below                      */
7407                                 int extra_align = patch_info->type == MONO_PATCH_INFO_R8 ? 2 : 1;
7408                                 aligned_code = (guint8*)ALIGN_TO (code + extra_align, 16);
7409                                 /* The technique of hiding data in an  */
7410                                 /* instruction has a problem here: we  */
7411                                 /* need the data aligned to a 16-byte  */
7412                                 /* boundary but the instruction cannot */
7413                                 /* cross the bundle boundary. so only  */
7414                                 /* odd multiples of 16 can be used     */
7415                                 if ((intptr_t)aligned_code % kNaClAlignment == 0) {
7416                                         aligned_code += 16;
7417                                 }
7418                                 while (code < aligned_code) {
7419                                         *(code++) = 0xf4; /* hlt */
7420                                 }
7421                         }
7422 #endif
7423
7424                         pos = cfg->native_code + patch_info->ip.i;
7425                         if (IS_REX (pos [1])) {
7426                                 patch_pos = pos + 5;
7427                                 target_pos = code - pos - 9;
7428                         }
7429                         else {
7430                                 patch_pos = pos + 4;
7431                                 target_pos = code - pos - 8;
7432                         }
7433
7434                         if (patch_info->type == MONO_PATCH_INFO_R8) {
7435 #ifdef __native_client_codegen__
7436                                 /* Hide 64-bit data in a         */
7437                                 /* "mov imm64, r11" instruction. */
7438                                 /* write it before the start of  */
7439                                 /* the data*/
7440                                 *(code-2) = 0x49; /* prefix      */
7441                                 *(code-1) = 0xbb; /* mov X, %r11 */
7442 #endif
7443                                 *(double*)code = *(double*)patch_info->data.target;
7444                                 code += sizeof (double);
7445                         } else {
7446 #ifdef __native_client_codegen__
7447                                 /* Hide 32-bit data in a        */
7448                                 /* "push imm32" instruction.    */
7449                                 *(code-1) = 0x68; /* push */
7450 #endif
7451                                 *(float*)code = *(float*)patch_info->data.target;
7452                                 code += sizeof (float);
7453                         }
7454
7455                         *(guint32*)(patch_pos) = target_pos;
7456
7457                         remove = TRUE;
7458                         break;
7459                 }
7460                 case MONO_PATCH_INFO_GC_CARD_TABLE_ADDR: {
7461                         guint8 *pos;
7462
7463                         if (cfg->compile_aot)
7464                                 continue;
7465
7466                         /*loading is faster against aligned addresses.*/
7467                         code = (guint8*)ALIGN_TO (code, 8);
7468                         memset (orig_code, 0, code - orig_code);
7469
7470                         pos = cfg->native_code + patch_info->ip.i;
7471
7472                         /*alu_op [rex] modr/m imm32 - 7 or 8 bytes */
7473                         if (IS_REX (pos [1]))
7474                                 *(guint32*)(pos + 4) = (guint8*)code - pos - 8;
7475                         else
7476                                 *(guint32*)(pos + 3) = (guint8*)code - pos - 7;
7477
7478                         *(gpointer*)code = (gpointer)patch_info->data.target;
7479                         code += sizeof (gpointer);
7480
7481                         remove = TRUE;
7482                         break;
7483                 }
7484                 default:
7485                         break;
7486                 }
7487
7488                 if (remove) {
7489                         if (patch_info == cfg->patch_info)
7490                                 cfg->patch_info = patch_info->next;
7491                         else {
7492                                 MonoJumpInfo *tmp;
7493
7494                                 for (tmp = cfg->patch_info; tmp->next != patch_info; tmp = tmp->next)
7495                                         ;
7496                                 tmp->next = patch_info->next;
7497                         }
7498                 }
7499                 g_assert (code < cfg->native_code + cfg->code_size);
7500         }
7501
7502         cfg->code_len = code - cfg->native_code;
7503
7504         g_assert (cfg->code_len < cfg->code_size);
7505
7506 }
7507
7508 #endif /* DISABLE_JIT */
7509
7510 void*
7511 mono_arch_instrument_prolog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
7512 {
7513         guchar *code = p;
7514         CallInfo *cinfo = NULL;
7515         MonoMethodSignature *sig;
7516         MonoInst *inst;
7517         int i, n, stack_area = 0;
7518
7519         /* Keep this in sync with mono_arch_get_argument_info */
7520
7521         if (enable_arguments) {
7522                 /* Allocate a new area on the stack and save arguments there */
7523                 sig = mono_method_signature (cfg->method);
7524
7525                 cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig);
7526
7527                 n = sig->param_count + sig->hasthis;
7528
7529                 stack_area = ALIGN_TO (n * 8, 16);
7530
7531                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, stack_area);
7532
7533                 for (i = 0; i < n; ++i) {
7534                         inst = cfg->args [i];
7535
7536                         if (inst->opcode == OP_REGVAR)
7537                                 amd64_mov_membase_reg (code, AMD64_RSP, (i * 8), inst->dreg, 8);
7538                         else {
7539                                 amd64_mov_reg_membase (code, AMD64_R11, inst->inst_basereg, inst->inst_offset, 8);
7540                                 amd64_mov_membase_reg (code, AMD64_RSP, (i * 8), AMD64_R11, 8);
7541                         }
7542                 }
7543         }
7544
7545         mono_add_patch_info (cfg, code-cfg->native_code, MONO_PATCH_INFO_METHODCONST, cfg->method);
7546         amd64_set_reg_template (code, AMD64_ARG_REG1);
7547         amd64_mov_reg_reg (code, AMD64_ARG_REG2, AMD64_RSP, 8);
7548         code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)func, TRUE);
7549
7550         if (enable_arguments)
7551                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, stack_area);
7552
7553         return code;
7554 }
7555
7556 enum {
7557         SAVE_NONE,
7558         SAVE_STRUCT,
7559         SAVE_EAX,
7560         SAVE_EAX_EDX,
7561         SAVE_XMM
7562 };
7563
7564 void*
7565 mono_arch_instrument_epilog_full (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments, gboolean preserve_argument_registers)
7566 {
7567         guchar *code = p;
7568         int save_mode = SAVE_NONE;
7569         MonoMethod *method = cfg->method;
7570         MonoType *ret_type = mini_type_get_underlying_type (NULL, mono_method_signature (method)->ret);
7571         int i;
7572
7573         switch (ret_type->type) {
7574         case MONO_TYPE_VOID:
7575                 /* special case string .ctor icall */
7576                 if (strcmp (".ctor", method->name) && method->klass == mono_defaults.string_class)
7577                         save_mode = SAVE_EAX;
7578                 else
7579                         save_mode = SAVE_NONE;
7580                 break;
7581         case MONO_TYPE_I8:
7582         case MONO_TYPE_U8:
7583                 save_mode = SAVE_EAX;
7584                 break;
7585         case MONO_TYPE_R4:
7586         case MONO_TYPE_R8:
7587                 save_mode = SAVE_XMM;
7588                 break;
7589         case MONO_TYPE_GENERICINST:
7590                 if (!mono_type_generic_inst_is_valuetype (ret_type)) {
7591                         save_mode = SAVE_EAX;
7592                         break;
7593                 }
7594                 /* Fall through */
7595         case MONO_TYPE_VALUETYPE:
7596                 save_mode = SAVE_STRUCT;
7597                 break;
7598         default:
7599                 save_mode = SAVE_EAX;
7600                 break;
7601         }
7602
7603         /* Save the result and copy it into the proper argument register */
7604         switch (save_mode) {
7605         case SAVE_EAX:
7606                 amd64_push_reg (code, AMD64_RAX);
7607                 /* Align stack */
7608                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
7609                 if (enable_arguments)
7610                         amd64_mov_reg_reg (code, AMD64_ARG_REG2, AMD64_RAX, 8);
7611                 break;
7612         case SAVE_STRUCT:
7613                 /* FIXME: */
7614                 if (enable_arguments)
7615                         amd64_mov_reg_imm (code, AMD64_ARG_REG2, 0);
7616                 break;
7617         case SAVE_XMM:
7618                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
7619                 amd64_movsd_membase_reg (code, AMD64_RSP, 0, AMD64_XMM0);
7620                 /* Align stack */
7621                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
7622                 /*
7623                  * The result is already in the proper argument register so no copying
7624                  * needed.
7625                  */
7626                 break;
7627         case SAVE_NONE:
7628                 break;
7629         default:
7630                 g_assert_not_reached ();
7631         }
7632
7633         /* Set %al since this is a varargs call */
7634         if (save_mode == SAVE_XMM)
7635                 amd64_mov_reg_imm (code, AMD64_RAX, 1);
7636         else
7637                 amd64_mov_reg_imm (code, AMD64_RAX, 0);
7638
7639         if (preserve_argument_registers) {
7640                 for (i = 0; i < PARAM_REGS; ++i)
7641                         amd64_push_reg (code, param_regs [i]);
7642         }
7643
7644         mono_add_patch_info (cfg, code-cfg->native_code, MONO_PATCH_INFO_METHODCONST, method);
7645         amd64_set_reg_template (code, AMD64_ARG_REG1);
7646         code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)func, TRUE);
7647
7648         if (preserve_argument_registers) {
7649                 for (i = PARAM_REGS - 1; i >= 0; --i)
7650                         amd64_pop_reg (code, param_regs [i]);
7651         }
7652
7653         /* Restore result */
7654         switch (save_mode) {
7655         case SAVE_EAX:
7656                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
7657                 amd64_pop_reg (code, AMD64_RAX);
7658                 break;
7659         case SAVE_STRUCT:
7660                 /* FIXME: */
7661                 break;
7662         case SAVE_XMM:
7663                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
7664                 amd64_movsd_reg_membase (code, AMD64_XMM0, AMD64_RSP, 0);
7665                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
7666                 break;
7667         case SAVE_NONE:
7668                 break;
7669         default:
7670                 g_assert_not_reached ();
7671         }
7672
7673         return code;
7674 }
7675
7676 void
7677 mono_arch_flush_icache (guint8 *code, gint size)
7678 {
7679         /* Not needed */
7680 }
7681
7682 void
7683 mono_arch_flush_register_windows (void)
7684 {
7685 }
7686
7687 gboolean
7688 mono_arch_is_inst_imm (gint64 imm)
7689 {
7690         return amd64_is_imm32 (imm);
7691 }
7692
7693 /*
7694  * Determine whenever the trap whose info is in SIGINFO is caused by
7695  * integer overflow.
7696  */
7697 gboolean
7698 mono_arch_is_int_overflow (void *sigctx, void *info)
7699 {
7700         MonoContext ctx;
7701         guint8* rip;
7702         int reg;
7703         gint64 value;
7704
7705         mono_arch_sigctx_to_monoctx (sigctx, &ctx);
7706
7707         rip = (guint8*)ctx.rip;
7708
7709         if (IS_REX (rip [0])) {
7710                 reg = amd64_rex_b (rip [0]);
7711                 rip ++;
7712         }
7713         else
7714                 reg = 0;
7715
7716         if ((rip [0] == 0xf7) && (x86_modrm_mod (rip [1]) == 0x3) && (x86_modrm_reg (rip [1]) == 0x7)) {
7717                 /* idiv REG */
7718                 reg += x86_modrm_rm (rip [1]);
7719
7720                 switch (reg) {
7721                 case AMD64_RAX:
7722                         value = ctx.rax;
7723                         break;
7724                 case AMD64_RBX:
7725                         value = ctx.rbx;
7726                         break;
7727                 case AMD64_RCX:
7728                         value = ctx.rcx;
7729                         break;
7730                 case AMD64_RDX:
7731                         value = ctx.rdx;
7732                         break;
7733                 case AMD64_RBP:
7734                         value = ctx.rbp;
7735                         break;
7736                 case AMD64_RSP:
7737                         value = ctx.rsp;
7738                         break;
7739                 case AMD64_RSI:
7740                         value = ctx.rsi;
7741                         break;
7742                 case AMD64_RDI:
7743                         value = ctx.rdi;
7744                         break;
7745                 case AMD64_R12:
7746                         value = ctx.r12;
7747                         break;
7748                 case AMD64_R13:
7749                         value = ctx.r13;
7750                         break;
7751                 case AMD64_R14:
7752                         value = ctx.r14;
7753                         break;
7754                 case AMD64_R15:
7755                         value = ctx.r15;
7756                         break;
7757                 default:
7758                         g_assert_not_reached ();
7759                         reg = -1;
7760                 }
7761
7762                 if (value == -1)
7763                         return TRUE;
7764         }
7765
7766         return FALSE;
7767 }
7768
7769 guint32
7770 mono_arch_get_patch_offset (guint8 *code)
7771 {
7772         return 3;
7773 }
7774
7775 /**
7776  * mono_breakpoint_clean_code:
7777  *
7778  * Copy @size bytes from @code - @offset to the buffer @buf. If the debugger inserted software
7779  * breakpoints in the original code, they are removed in the copy.
7780  *
7781  * Returns TRUE if no sw breakpoint was present.
7782  */
7783 gboolean
7784 mono_breakpoint_clean_code (guint8 *method_start, guint8 *code, int offset, guint8 *buf, int size)
7785 {
7786         int i;
7787         gboolean can_write = TRUE;
7788         /*
7789          * If method_start is non-NULL we need to perform bound checks, since we access memory
7790          * at code - offset we could go before the start of the method and end up in a different
7791          * page of memory that is not mapped or read incorrect data anyway. We zero-fill the bytes
7792          * instead.
7793          */
7794         if (!method_start || code - offset >= method_start) {
7795                 memcpy (buf, code - offset, size);
7796         } else {
7797                 int diff = code - method_start;
7798                 memset (buf, 0, size);
7799                 memcpy (buf + offset - diff, method_start, diff + size - offset);
7800         }
7801         code -= offset;
7802         for (i = 0; i < MONO_BREAKPOINT_ARRAY_SIZE; ++i) {
7803                 int idx = mono_breakpoint_info_index [i];
7804                 guint8 *ptr;
7805                 if (idx < 1)
7806                         continue;
7807                 ptr = mono_breakpoint_info [idx].address;
7808                 if (ptr >= code && ptr < code + size) {
7809                         guint8 saved_byte = mono_breakpoint_info [idx].saved_byte;
7810                         can_write = FALSE;
7811                         /*g_print ("patching %p with 0x%02x (was: 0x%02x)\n", ptr, saved_byte, buf [ptr - code]);*/
7812                         buf [ptr - code] = saved_byte;
7813                 }
7814         }
7815         return can_write;
7816 }
7817
7818 #if defined(__native_client_codegen__)
7819 /* For membase calls, we want the base register. for Native Client,  */
7820 /* all indirect calls have the following sequence with the given sizes: */
7821 /* mov %eXX,%eXX                                [2-3]   */
7822 /* mov disp(%r15,%rXX,scale),%r11d              [4-8]   */
7823 /* and $0xffffffffffffffe0,%r11d                [4]     */
7824 /* add %r15,%r11                                [3]     */
7825 /* callq *%r11                                  [3]     */
7826
7827
7828 /* Determine if code points to a NaCl call-through-register sequence, */
7829 /* (i.e., the last 3 instructions listed above) */
7830 int
7831 is_nacl_call_reg_sequence(guint8* code)
7832 {
7833         const char *sequence = "\x41\x83\xe3\xe0" /* and */
7834                                "\x4d\x03\xdf"     /* add */
7835                                "\x41\xff\xd3";   /* call */
7836         return memcmp(code, sequence, 10) == 0;
7837 }
7838
7839 /* Determine if code points to the first opcode of the mov membase component */
7840 /* of an indirect call sequence (i.e. the first 2 instructions listed above) */
7841 /* (there could be a REX prefix before the opcode but it is ignored) */
7842 static int
7843 is_nacl_indirect_call_membase_sequence(guint8* code)
7844 {
7845                /* Check for mov opcode, reg-reg addressing mode (mod = 3), */
7846         return code[0] == 0x8b && amd64_modrm_mod(code[1]) == 3 &&
7847                /* and that src reg = dest reg */
7848                amd64_modrm_reg(code[1]) == amd64_modrm_rm(code[1]) &&
7849                /* Check that next inst is mov, uses SIB byte (rm = 4), */
7850                IS_REX(code[2]) &&
7851                code[3] == 0x8b && amd64_modrm_rm(code[4]) == 4 &&
7852                /* and has dst of r11 and base of r15 */
7853                (amd64_modrm_reg(code[4]) + amd64_rex_r(code[2])) == AMD64_R11 &&
7854                (amd64_sib_base(code[5]) + amd64_rex_b(code[2])) == AMD64_R15;
7855 }
7856 #endif /* __native_client_codegen__ */
7857
7858 int
7859 mono_arch_get_this_arg_reg (guint8 *code)
7860 {
7861         return AMD64_ARG_REG1;
7862 }
7863
7864 gpointer
7865 mono_arch_get_this_arg_from_call (mgreg_t *regs, guint8 *code)
7866 {
7867         return (gpointer)regs [mono_arch_get_this_arg_reg (code)];
7868 }
7869
7870 #define MAX_ARCH_DELEGATE_PARAMS 10
7871
7872 static gpointer
7873 get_delegate_invoke_impl (gboolean has_target, guint32 param_count, guint32 *code_len)
7874 {
7875         guint8 *code, *start;
7876         int i;
7877
7878         if (has_target) {
7879                 start = code = mono_global_codeman_reserve (64);
7880
7881                 /* Replace the this argument with the target */
7882                 amd64_mov_reg_reg (code, AMD64_RAX, AMD64_ARG_REG1, 8);
7883                 amd64_mov_reg_membase (code, AMD64_ARG_REG1, AMD64_RAX, G_STRUCT_OFFSET (MonoDelegate, target), 8);
7884                 amd64_jump_membase (code, AMD64_RAX, G_STRUCT_OFFSET (MonoDelegate, method_ptr));
7885
7886                 g_assert ((code - start) < 64);
7887         } else {
7888                 start = code = mono_global_codeman_reserve (64);
7889
7890                 if (param_count == 0) {
7891                         amd64_jump_membase (code, AMD64_ARG_REG1, G_STRUCT_OFFSET (MonoDelegate, method_ptr));
7892                 } else {
7893                         /* We have to shift the arguments left */
7894                         amd64_mov_reg_reg (code, AMD64_RAX, AMD64_ARG_REG1, 8);
7895                         for (i = 0; i < param_count; ++i) {
7896 #ifdef HOST_WIN32
7897                                 if (i < 3)
7898                                         amd64_mov_reg_reg (code, param_regs [i], param_regs [i + 1], 8);
7899                                 else
7900                                         amd64_mov_reg_membase (code, param_regs [i], AMD64_RSP, 0x28, 8);
7901 #else
7902                                 amd64_mov_reg_reg (code, param_regs [i], param_regs [i + 1], 8);
7903 #endif
7904                         }
7905
7906                         amd64_jump_membase (code, AMD64_RAX, G_STRUCT_OFFSET (MonoDelegate, method_ptr));
7907                 }
7908                 g_assert ((code - start) < 64);
7909         }
7910
7911         nacl_global_codeman_validate(&start, 64, &code);
7912
7913         mono_debug_add_delegate_trampoline (start, code - start);
7914
7915         if (code_len)
7916                 *code_len = code - start;
7917
7918
7919         if (mono_jit_map_is_enabled ()) {
7920                 char *buff;
7921                 if (has_target)
7922                         buff = (char*)"delegate_invoke_has_target";
7923                 else
7924                         buff = g_strdup_printf ("delegate_invoke_no_target_%d", param_count);
7925                 mono_emit_jit_tramp (start, code - start, buff);
7926                 if (!has_target)
7927                         g_free (buff);
7928         }
7929
7930         return start;
7931 }
7932
7933 /*
7934  * mono_arch_get_delegate_invoke_impls:
7935  *
7936  *   Return a list of MonoTrampInfo structures for the delegate invoke impl
7937  * trampolines.
7938  */
7939 GSList*
7940 mono_arch_get_delegate_invoke_impls (void)
7941 {
7942         GSList *res = NULL;
7943         guint8 *code;
7944         guint32 code_len;
7945         int i;
7946
7947         code = get_delegate_invoke_impl (TRUE, 0, &code_len);
7948         res = g_slist_prepend (res, mono_tramp_info_create (g_strdup ("delegate_invoke_impl_has_target"), code, code_len, NULL, NULL));
7949
7950         for (i = 0; i < MAX_ARCH_DELEGATE_PARAMS; ++i) {
7951                 code = get_delegate_invoke_impl (FALSE, i, &code_len);
7952                 res = g_slist_prepend (res, mono_tramp_info_create (g_strdup_printf ("delegate_invoke_impl_target_%d", i), code, code_len, NULL, NULL));
7953         }
7954
7955         return res;
7956 }
7957
7958 gpointer
7959 mono_arch_get_delegate_invoke_impl (MonoMethodSignature *sig, gboolean has_target)
7960 {
7961         guint8 *code, *start;
7962         int i;
7963
7964         if (sig->param_count > MAX_ARCH_DELEGATE_PARAMS)
7965                 return NULL;
7966
7967         /* FIXME: Support more cases */
7968         if (MONO_TYPE_ISSTRUCT (sig->ret))
7969                 return NULL;
7970
7971         if (has_target) {
7972                 static guint8* cached = NULL;
7973
7974                 if (cached)
7975                         return cached;
7976
7977                 if (mono_aot_only)
7978                         start = mono_aot_get_trampoline ("delegate_invoke_impl_has_target");
7979                 else
7980                         start = get_delegate_invoke_impl (TRUE, 0, NULL);
7981
7982                 mono_memory_barrier ();
7983
7984                 cached = start;
7985         } else {
7986                 static guint8* cache [MAX_ARCH_DELEGATE_PARAMS + 1] = {NULL};
7987                 for (i = 0; i < sig->param_count; ++i)
7988                         if (!mono_is_regsize_var (sig->params [i]))
7989                                 return NULL;
7990                 if (sig->param_count > 4)
7991                         return NULL;
7992
7993                 code = cache [sig->param_count];
7994                 if (code)
7995                         return code;
7996
7997                 if (mono_aot_only) {
7998                         char *name = g_strdup_printf ("delegate_invoke_impl_target_%d", sig->param_count);
7999                         start = mono_aot_get_trampoline (name);
8000                         g_free (name);
8001                 } else {
8002                         start = get_delegate_invoke_impl (FALSE, sig->param_count, NULL);
8003                 }
8004
8005                 mono_memory_barrier ();
8006
8007                 cache [sig->param_count] = start;
8008         }
8009
8010         return start;
8011 }
8012 void
8013 mono_arch_finish_init (void)
8014 {
8015 #ifdef HOST_WIN32
8016         /*
8017          * We need to init this multiple times, since when we are first called, the key might not
8018          * be initialized yet.
8019          */
8020         appdomain_tls_offset = mono_domain_get_tls_key ();
8021         lmf_tls_offset = mono_get_jit_tls_key ();
8022         lmf_addr_tls_offset = mono_get_jit_tls_key ();
8023
8024         /* Only 64 tls entries can be accessed using inline code */
8025         if (appdomain_tls_offset >= 64)
8026                 appdomain_tls_offset = -1;
8027         if (lmf_tls_offset >= 64)
8028                 lmf_tls_offset = -1;
8029         if (lmf_addr_tls_offset >= 64)
8030                 lmf_addr_tls_offset = -1;
8031 #else
8032 #ifdef MONO_XEN_OPT
8033         optimize_for_xen = access ("/proc/xen", F_OK) == 0;
8034 #endif
8035         appdomain_tls_offset = mono_domain_get_tls_offset ();
8036         lmf_tls_offset = mono_get_lmf_tls_offset ();
8037         lmf_addr_tls_offset = mono_get_lmf_addr_tls_offset ();
8038 #endif
8039 }
8040
8041 void
8042 mono_arch_free_jit_tls_data (MonoJitTlsData *tls)
8043 {
8044 }
8045
8046 #ifdef MONO_ARCH_HAVE_IMT
8047
8048 #if defined(__default_codegen__)
8049 #define CMP_SIZE (6 + 1)
8050 #define CMP_REG_REG_SIZE (4 + 1)
8051 #define BR_SMALL_SIZE 2
8052 #define BR_LARGE_SIZE 6
8053 #define MOV_REG_IMM_SIZE 10
8054 #define MOV_REG_IMM_32BIT_SIZE 6
8055 #define JUMP_REG_SIZE (2 + 1)
8056 #elif defined(__native_client_codegen__)
8057 /* NaCl N-byte instructions can be padded up to N-1 bytes */
8058 #define CMP_SIZE ((6 + 1) * 2 - 1)
8059 #define CMP_REG_REG_SIZE ((4 + 1) * 2 - 1)
8060 #define BR_SMALL_SIZE (2 * 2 - 1)
8061 #define BR_LARGE_SIZE (6 * 2 - 1)
8062 #define MOV_REG_IMM_SIZE (10 * 2 - 1)
8063 #define MOV_REG_IMM_32BIT_SIZE (6 * 2 - 1)
8064 /* Jump reg for NaCl adds a mask (+4) and add (+3) */
8065 #define JUMP_REG_SIZE ((2 + 1 + 4 + 3) * 2 - 1)
8066 /* Jump membase's size is large and unpredictable    */
8067 /* in native client, just pad it out a whole bundle. */
8068 #define JUMP_MEMBASE_SIZE (kNaClAlignment)
8069 #endif
8070
8071 static int
8072 imt_branch_distance (MonoIMTCheckItem **imt_entries, int start, int target)
8073 {
8074         int i, distance = 0;
8075         for (i = start; i < target; ++i)
8076                 distance += imt_entries [i]->chunk_size;
8077         return distance;
8078 }
8079
8080 /*
8081  * LOCKING: called with the domain lock held
8082  */
8083 gpointer
8084 mono_arch_build_imt_thunk (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
8085         gpointer fail_tramp)
8086 {
8087         int i;
8088         int size = 0;
8089         guint8 *code, *start;
8090         gboolean vtable_is_32bit = ((gsize)(vtable) == (gsize)(int)(gsize)(vtable));
8091
8092         for (i = 0; i < count; ++i) {
8093                 MonoIMTCheckItem *item = imt_entries [i];
8094                 if (item->is_equals) {
8095                         if (item->check_target_idx) {
8096                                 if (!item->compare_done) {
8097                                         if (amd64_is_imm32 (item->key))
8098                                                 item->chunk_size += CMP_SIZE;
8099                                         else
8100                                                 item->chunk_size += MOV_REG_IMM_SIZE + CMP_REG_REG_SIZE;
8101                                 }
8102                                 if (item->has_target_code) {
8103                                         item->chunk_size += MOV_REG_IMM_SIZE;
8104                                 } else {
8105                                         if (vtable_is_32bit)
8106                                                 item->chunk_size += MOV_REG_IMM_32BIT_SIZE;
8107                                         else
8108                                                 item->chunk_size += MOV_REG_IMM_SIZE;
8109 #ifdef __native_client_codegen__
8110                                         item->chunk_size += JUMP_MEMBASE_SIZE;
8111 #endif
8112                                 }
8113                                 item->chunk_size += BR_SMALL_SIZE + JUMP_REG_SIZE;
8114                         } else {
8115                                 if (fail_tramp) {
8116                                         item->chunk_size += MOV_REG_IMM_SIZE * 3 + CMP_REG_REG_SIZE +
8117                                                 BR_SMALL_SIZE + JUMP_REG_SIZE * 2;
8118                                 } else {
8119                                         if (vtable_is_32bit)
8120                                                 item->chunk_size += MOV_REG_IMM_32BIT_SIZE;
8121                                         else
8122                                                 item->chunk_size += MOV_REG_IMM_SIZE;
8123                                         item->chunk_size += JUMP_REG_SIZE;
8124                                         /* with assert below:
8125                                          * item->chunk_size += CMP_SIZE + BR_SMALL_SIZE + 1;
8126                                          */
8127 #ifdef __native_client_codegen__
8128                                         item->chunk_size += JUMP_MEMBASE_SIZE;
8129 #endif
8130                                 }
8131                         }
8132                 } else {
8133                         if (amd64_is_imm32 (item->key))
8134                                 item->chunk_size += CMP_SIZE;
8135                         else
8136                                 item->chunk_size += MOV_REG_IMM_SIZE + CMP_REG_REG_SIZE;
8137                         item->chunk_size += BR_LARGE_SIZE;
8138                         imt_entries [item->check_target_idx]->compare_done = TRUE;
8139                 }
8140                 size += item->chunk_size;
8141         }
8142 #if defined(__native_client__) && defined(__native_client_codegen__)
8143         /* In Native Client, we don't re-use thunks, allocate from the */
8144         /* normal code manager paths. */
8145         code = mono_domain_code_reserve (domain, size);
8146 #else
8147         if (fail_tramp)
8148                 code = mono_method_alloc_generic_virtual_thunk (domain, size);
8149         else
8150                 code = mono_domain_code_reserve (domain, size);
8151 #endif
8152         start = code;
8153         for (i = 0; i < count; ++i) {
8154                 MonoIMTCheckItem *item = imt_entries [i];
8155                 item->code_target = code;
8156                 if (item->is_equals) {
8157                         gboolean fail_case = !item->check_target_idx && fail_tramp;
8158
8159                         if (item->check_target_idx || fail_case) {
8160                                 if (!item->compare_done || fail_case) {
8161                                         if (amd64_is_imm32 (item->key))
8162                                                 amd64_alu_reg_imm_size (code, X86_CMP, MONO_ARCH_IMT_REG, (guint32)(gssize)item->key, sizeof(gpointer));
8163                                         else {
8164                                                 amd64_mov_reg_imm (code, MONO_ARCH_IMT_SCRATCH_REG, item->key);
8165                                                 amd64_alu_reg_reg (code, X86_CMP, MONO_ARCH_IMT_REG, MONO_ARCH_IMT_SCRATCH_REG);
8166                                         }
8167                                 }
8168                                 item->jmp_code = code;
8169                                 amd64_branch8 (code, X86_CC_NE, 0, FALSE);
8170                                 if (item->has_target_code) {
8171                                         amd64_mov_reg_imm (code, MONO_ARCH_IMT_SCRATCH_REG, item->value.target_code);
8172                                         amd64_jump_reg (code, MONO_ARCH_IMT_SCRATCH_REG);
8173                                 } else {
8174                                         amd64_mov_reg_imm (code, MONO_ARCH_IMT_SCRATCH_REG, & (vtable->vtable [item->value.vtable_slot]));
8175                                         amd64_jump_membase (code, MONO_ARCH_IMT_SCRATCH_REG, 0);
8176                                 }
8177
8178                                 if (fail_case) {
8179                                         amd64_patch (item->jmp_code, code);
8180                                         amd64_mov_reg_imm (code, MONO_ARCH_IMT_SCRATCH_REG, fail_tramp);
8181                                         amd64_jump_reg (code, MONO_ARCH_IMT_SCRATCH_REG);
8182                                         item->jmp_code = NULL;
8183                                 }
8184                         } else {
8185                                 /* enable the commented code to assert on wrong method */
8186 #if 0
8187                                 if (amd64_is_imm32 (item->key))
8188                                         amd64_alu_reg_imm_size (code, X86_CMP, MONO_ARCH_IMT_REG, (guint32)(gssize)item->key, sizeof(gpointer));
8189                                 else {
8190                                         amd64_mov_reg_imm (code, MONO_ARCH_IMT_SCRATCH_REG, item->key);
8191                                         amd64_alu_reg_reg (code, X86_CMP, MONO_ARCH_IMT_REG, MONO_ARCH_IMT_SCRATCH_REG);
8192                                 }
8193                                 item->jmp_code = code;
8194                                 amd64_branch8 (code, X86_CC_NE, 0, FALSE);
8195                                 /* See the comment below about R10 */
8196                                 amd64_mov_reg_imm (code, MONO_ARCH_IMT_SCRATCH_REG, & (vtable->vtable [item->value.vtable_slot]));
8197                                 amd64_jump_membase (code, MONO_ARCH_IMT_SCRATCH_REG, 0);
8198                                 amd64_patch (item->jmp_code, code);
8199                                 amd64_breakpoint (code);
8200                                 item->jmp_code = NULL;
8201 #else
8202                                 /* We're using R10 (MONO_ARCH_IMT_SCRATCH_REG) here because R11 (MONO_ARCH_IMT_REG)
8203                                    needs to be preserved.  R10 needs
8204                                    to be preserved for calls which
8205                                    require a runtime generic context,
8206                                    but interface calls don't. */
8207                                 amd64_mov_reg_imm (code, MONO_ARCH_IMT_SCRATCH_REG, & (vtable->vtable [item->value.vtable_slot]));
8208                                 amd64_jump_membase (code, MONO_ARCH_IMT_SCRATCH_REG, 0);
8209 #endif
8210                         }
8211                 } else {
8212                         if (amd64_is_imm32 (item->key))
8213                                 amd64_alu_reg_imm_size (code, X86_CMP, MONO_ARCH_IMT_REG, (guint32)(gssize)item->key, sizeof (gpointer));
8214                         else {
8215                                 amd64_mov_reg_imm (code, MONO_ARCH_IMT_SCRATCH_REG, item->key);
8216                                 amd64_alu_reg_reg (code, X86_CMP, MONO_ARCH_IMT_REG, MONO_ARCH_IMT_SCRATCH_REG);
8217                         }
8218                         item->jmp_code = code;
8219                         if (x86_is_imm8 (imt_branch_distance (imt_entries, i, item->check_target_idx)))
8220                                 x86_branch8 (code, X86_CC_GE, 0, FALSE);
8221                         else
8222                                 x86_branch32 (code, X86_CC_GE, 0, FALSE);
8223                 }
8224                 g_assert (code - item->code_target <= item->chunk_size);
8225         }
8226         /* patch the branches to get to the target items */
8227         for (i = 0; i < count; ++i) {
8228                 MonoIMTCheckItem *item = imt_entries [i];
8229                 if (item->jmp_code) {
8230                         if (item->check_target_idx) {
8231                                 amd64_patch (item->jmp_code, imt_entries [item->check_target_idx]->code_target);
8232                         }
8233                 }
8234         }
8235
8236         if (!fail_tramp)
8237                 mono_stats.imt_thunks_size += code - start;
8238         g_assert (code - start <= size);
8239
8240         nacl_domain_code_validate(domain, &start, size, &code);
8241
8242         return start;
8243 }
8244
8245 MonoMethod*
8246 mono_arch_find_imt_method (mgreg_t *regs, guint8 *code)
8247 {
8248         return (MonoMethod*)regs [MONO_ARCH_IMT_REG];
8249 }
8250 #endif
8251
8252 MonoVTable*
8253 mono_arch_find_static_call_vtable (mgreg_t *regs, guint8 *code)
8254 {
8255         return (MonoVTable*) regs [MONO_ARCH_RGCTX_REG];
8256 }
8257
8258 GSList*
8259 mono_arch_get_cie_program (void)
8260 {
8261         GSList *l = NULL;
8262
8263         mono_add_unwind_op_def_cfa (l, (guint8*)NULL, (guint8*)NULL, AMD64_RSP, 8);
8264         mono_add_unwind_op_offset (l, (guint8*)NULL, (guint8*)NULL, AMD64_RIP, -8);
8265
8266         return l;
8267 }
8268
8269 MonoInst*
8270 mono_arch_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args)
8271 {
8272         MonoInst *ins = NULL;
8273         int opcode = 0;
8274
8275         if (cmethod->klass == mono_defaults.math_class) {
8276                 if (strcmp (cmethod->name, "Sin") == 0) {
8277                         opcode = OP_SIN;
8278                 } else if (strcmp (cmethod->name, "Cos") == 0) {
8279                         opcode = OP_COS;
8280                 } else if (strcmp (cmethod->name, "Sqrt") == 0) {
8281                         opcode = OP_SQRT;
8282                 } else if (strcmp (cmethod->name, "Abs") == 0 && fsig->params [0]->type == MONO_TYPE_R8) {
8283                         opcode = OP_ABS;
8284                 }
8285
8286                 if (opcode) {
8287                         MONO_INST_NEW (cfg, ins, opcode);
8288                         ins->type = STACK_R8;
8289                         ins->dreg = mono_alloc_freg (cfg);
8290                         ins->sreg1 = args [0]->dreg;
8291                         MONO_ADD_INS (cfg->cbb, ins);
8292                 }
8293
8294                 opcode = 0;
8295                 if (cfg->opt & MONO_OPT_CMOV) {
8296                         if (strcmp (cmethod->name, "Min") == 0) {
8297                                 if (fsig->params [0]->type == MONO_TYPE_I4)
8298                                         opcode = OP_IMIN;
8299                                 if (fsig->params [0]->type == MONO_TYPE_U4)
8300                                         opcode = OP_IMIN_UN;
8301                                 else if (fsig->params [0]->type == MONO_TYPE_I8)
8302                                         opcode = OP_LMIN;
8303                                 else if (fsig->params [0]->type == MONO_TYPE_U8)
8304                                         opcode = OP_LMIN_UN;
8305                         } else if (strcmp (cmethod->name, "Max") == 0) {
8306                                 if (fsig->params [0]->type == MONO_TYPE_I4)
8307                                         opcode = OP_IMAX;
8308                                 if (fsig->params [0]->type == MONO_TYPE_U4)
8309                                         opcode = OP_IMAX_UN;
8310                                 else if (fsig->params [0]->type == MONO_TYPE_I8)
8311                                         opcode = OP_LMAX;
8312                                 else if (fsig->params [0]->type == MONO_TYPE_U8)
8313                                         opcode = OP_LMAX_UN;
8314                         }
8315                 }
8316
8317                 if (opcode) {
8318                         MONO_INST_NEW (cfg, ins, opcode);
8319                         ins->type = fsig->params [0]->type == MONO_TYPE_I4 ? STACK_I4 : STACK_I8;
8320                         ins->dreg = mono_alloc_ireg (cfg);
8321                         ins->sreg1 = args [0]->dreg;
8322                         ins->sreg2 = args [1]->dreg;
8323                         MONO_ADD_INS (cfg->cbb, ins);
8324                 }
8325
8326 #if 0
8327                 /* OP_FREM is not IEEE compatible */
8328                 else if (strcmp (cmethod->name, "IEEERemainder") == 0) {
8329                         MONO_INST_NEW (cfg, ins, OP_FREM);
8330                         ins->inst_i0 = args [0];
8331                         ins->inst_i1 = args [1];
8332                 }
8333 #endif
8334         }
8335
8336         /*
8337          * Can't implement CompareExchange methods this way since they have
8338          * three arguments.
8339          */
8340
8341         return ins;
8342 }
8343
8344 gboolean
8345 mono_arch_print_tree (MonoInst *tree, int arity)
8346 {
8347         return 0;
8348 }
8349
8350 MonoInst* mono_arch_get_domain_intrinsic (MonoCompile* cfg)
8351 {
8352         MonoInst* ins;
8353
8354         if (appdomain_tls_offset == -1)
8355                 return NULL;
8356
8357         MONO_INST_NEW (cfg, ins, OP_TLS_GET);
8358         ins->inst_offset = appdomain_tls_offset;
8359         return ins;
8360 }
8361
8362 #define _CTX_REG(ctx,fld,i) ((&ctx->fld)[i])
8363
8364 mgreg_t
8365 mono_arch_context_get_int_reg (MonoContext *ctx, int reg)
8366 {
8367         switch (reg) {
8368         case AMD64_RCX: return ctx->rcx;
8369         case AMD64_RDX: return ctx->rdx;
8370         case AMD64_RBX: return ctx->rbx;
8371         case AMD64_RBP: return ctx->rbp;
8372         case AMD64_RSP: return ctx->rsp;
8373         default:
8374                 if (reg < 8)
8375                         return _CTX_REG (ctx, rax, reg);
8376                 else if (reg >= 12)
8377                         return _CTX_REG (ctx, r12, reg - 12);
8378                 else
8379                         g_assert_not_reached ();
8380         }
8381 }
8382
8383 void
8384 mono_arch_context_set_int_reg (MonoContext *ctx, int reg, mgreg_t val)
8385 {
8386         switch (reg) {
8387         case AMD64_RCX:
8388                 ctx->rcx = val;
8389                 break;
8390         case AMD64_RDX:
8391                 ctx->rdx = val;
8392                 break;
8393         case AMD64_RBX:
8394                 ctx->rbx = val;
8395                 break;
8396         case AMD64_RBP:
8397                 ctx->rbp = val;
8398                 break;
8399         case AMD64_RSP:
8400                 ctx->rsp = val;
8401                 break;
8402         default:
8403                 if (reg < 8)
8404                         _CTX_REG (ctx, rax, reg) = val;
8405                 else if (reg >= 12)
8406                         _CTX_REG (ctx, r12, reg - 12) = val;
8407                 else
8408                         g_assert_not_reached ();
8409         }
8410 }
8411
8412 /*MONO_ARCH_HAVE_HANDLER_BLOCK_GUARD*/
8413 gpointer
8414 mono_arch_install_handler_block_guard (MonoJitInfo *ji, MonoJitExceptionInfo *clause, MonoContext *ctx, gpointer new_value)
8415 {
8416         int offset;
8417         gpointer *sp, old_value;
8418         char *bp;
8419         const unsigned char *handler;
8420
8421         /*Decode the first instruction to figure out where did we store the spvar*/
8422         /*Our jit MUST generate the following:
8423          mov    %rsp, ?(%rbp)
8424
8425          Which is encoded as: REX.W 0x89 mod_rm
8426          mod_rm (rsp, rbp, imm) which can be: (imm will never be zero)
8427                 mod (reg + imm8):  01 reg(rsp): 100 rm(rbp): 101 -> 01100101 (0x65)
8428                 mod (reg + imm32): 10 reg(rsp): 100 rm(rbp): 101 -> 10100101 (0xA5)
8429
8430         FIXME can we generate frameless methods on this case?
8431
8432         */
8433         handler = clause->handler_start;
8434
8435         /*REX.W*/
8436         if (*handler != 0x48)
8437                 return NULL;
8438         ++handler;
8439
8440         /*mov r, r/m */
8441         if (*handler != 0x89)
8442                 return NULL;
8443         ++handler;
8444
8445         if (*handler == 0x65)
8446                 offset = *(signed char*)(handler + 1);
8447         else if (*handler == 0xA5)
8448                 offset = *(int*)(handler + 1);
8449         else
8450                 return NULL;
8451
8452         /*Load the spvar*/
8453         bp = MONO_CONTEXT_GET_BP (ctx);
8454         sp = *(gpointer*)(bp + offset);
8455
8456         old_value = *sp;
8457         if (old_value < ji->code_start || (char*)old_value > ((char*)ji->code_start + ji->code_size))
8458                 return old_value;
8459
8460         *sp = new_value;
8461
8462         return old_value;
8463 }
8464
8465 /*
8466  * mono_arch_emit_load_aotconst:
8467  *
8468  *   Emit code to load the contents of the GOT slot identified by TRAMP_TYPE and
8469  * TARGET from the mscorlib GOT in full-aot code.
8470  * On AMD64, the result is placed into R11.
8471  */
8472 guint8*
8473 mono_arch_emit_load_aotconst (guint8 *start, guint8 *code, MonoJumpInfo **ji, int tramp_type, gconstpointer target)
8474 {
8475         *ji = mono_patch_info_list_prepend (*ji, code - start, tramp_type, target);
8476         amd64_mov_reg_membase (code, AMD64_R11, AMD64_RIP, 0, 8);
8477
8478         return code;
8479 }
8480
8481 /*
8482  * mono_arch_get_trampolines:
8483  *
8484  *   Return a list of MonoTrampInfo structures describing arch specific trampolines
8485  * for AOT.
8486  */
8487 GSList *
8488 mono_arch_get_trampolines (gboolean aot)
8489 {
8490         return mono_amd64_get_exception_trampolines (aot);
8491 }
8492
8493 /* Soft Debug support */
8494 #ifdef MONO_ARCH_SOFT_DEBUG_SUPPORTED
8495
8496 /*
8497  * mono_arch_set_breakpoint:
8498  *
8499  *   Set a breakpoint at the native code corresponding to JI at NATIVE_OFFSET.
8500  * The location should contain code emitted by OP_SEQ_POINT.
8501  */
8502 void
8503 mono_arch_set_breakpoint (MonoJitInfo *ji, guint8 *ip)
8504 {
8505         guint8 *code = ip;
8506         guint8 *orig_code = code;
8507
8508         if (ji->from_aot) {
8509                 guint32 native_offset = ip - (guint8*)ji->code_start;
8510                 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
8511
8512                 g_assert (info->bp_addrs [native_offset] == 0);
8513                 info->bp_addrs [native_offset] = bp_trigger_page;
8514         } else {
8515                 /*
8516                  * In production, we will use int3 (has to fix the size in the md
8517                  * file). But that could confuse gdb, so during development, we emit a SIGSEGV
8518                  * instead.
8519                  */
8520                 g_assert (code [0] == 0x90);
8521                 if (breakpoint_size == 8) {
8522                         amd64_mov_reg_mem (code, AMD64_R11, (guint64)bp_trigger_page, 4);
8523                 } else {
8524                         amd64_mov_reg_imm_size (code, AMD64_R11, (guint64)bp_trigger_page, 8);
8525                         amd64_mov_reg_membase (code, AMD64_R11, AMD64_R11, 0, 4);
8526                 }
8527
8528                 g_assert (code - orig_code == breakpoint_size);
8529         }
8530 }
8531
8532 /*
8533  * mono_arch_clear_breakpoint:
8534  *
8535  *   Clear the breakpoint at IP.
8536  */
8537 void
8538 mono_arch_clear_breakpoint (MonoJitInfo *ji, guint8 *ip)
8539 {
8540         guint8 *code = ip;
8541         int i;
8542
8543         if (ji->from_aot) {
8544                 guint32 native_offset = ip - (guint8*)ji->code_start;
8545                 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
8546
8547                 g_assert (info->bp_addrs [native_offset] == 0);
8548                 info->bp_addrs [native_offset] = info;
8549         } else {
8550                 for (i = 0; i < breakpoint_size; ++i)
8551                         x86_nop (code);
8552         }
8553 }
8554
8555 gboolean
8556 mono_arch_is_breakpoint_event (void *info, void *sigctx)
8557 {
8558 #ifdef HOST_WIN32
8559         EXCEPTION_RECORD* einfo = (EXCEPTION_RECORD*)info;
8560         return FALSE;
8561 #else
8562         siginfo_t* sinfo = (siginfo_t*) info;
8563         /* Sometimes the address is off by 4 */
8564         if (sinfo->si_addr >= bp_trigger_page && (guint8*)sinfo->si_addr <= (guint8*)bp_trigger_page + 128)
8565                 return TRUE;
8566         else
8567                 return FALSE;
8568 #endif
8569 }
8570
8571 /*
8572  * mono_arch_skip_breakpoint:
8573  *
8574  *   Modify CTX so the ip is placed after the breakpoint instruction, so when
8575  * we resume, the instruction is not executed again.
8576  */
8577 void
8578 mono_arch_skip_breakpoint (MonoContext *ctx, MonoJitInfo *ji)
8579 {
8580         if (ji->from_aot) {
8581                 /* amd64_mov_reg_membase (code, AMD64_R11, AMD64_R11, 0, 8) */
8582                 MONO_CONTEXT_SET_IP (ctx, (guint8*)MONO_CONTEXT_GET_IP (ctx) + 3);
8583         } else {
8584                 MONO_CONTEXT_SET_IP (ctx, (guint8*)MONO_CONTEXT_GET_IP (ctx) + breakpoint_fault_size);
8585         }
8586 }
8587
8588 /*
8589  * mono_arch_start_single_stepping:
8590  *
8591  *   Start single stepping.
8592  */
8593 void
8594 mono_arch_start_single_stepping (void)
8595 {
8596         mono_mprotect (ss_trigger_page, mono_pagesize (), 0);
8597 }
8598
8599 /*
8600  * mono_arch_stop_single_stepping:
8601  *
8602  *   Stop single stepping.
8603  */
8604 void
8605 mono_arch_stop_single_stepping (void)
8606 {
8607         mono_mprotect (ss_trigger_page, mono_pagesize (), MONO_MMAP_READ);
8608 }
8609
8610 /*
8611  * mono_arch_is_single_step_event:
8612  *
8613  *   Return whenever the machine state in SIGCTX corresponds to a single
8614  * step event.
8615  */
8616 gboolean
8617 mono_arch_is_single_step_event (void *info, void *sigctx)
8618 {
8619 #ifdef HOST_WIN32
8620         EXCEPTION_RECORD* einfo = (EXCEPTION_RECORD*)info;
8621         return FALSE;
8622 #else
8623         siginfo_t* sinfo = (siginfo_t*) info;
8624         /* Sometimes the address is off by 4 */
8625         if (sinfo->si_addr >= ss_trigger_page && (guint8*)sinfo->si_addr <= (guint8*)ss_trigger_page + 128)
8626                 return TRUE;
8627         else
8628                 return FALSE;
8629 #endif
8630 }
8631
8632 /*
8633  * mono_arch_skip_single_step:
8634  *
8635  *   Modify CTX so the ip is placed after the single step trigger instruction,
8636  * we resume, the instruction is not executed again.
8637  */
8638 void
8639 mono_arch_skip_single_step (MonoContext *ctx)
8640 {
8641         MONO_CONTEXT_SET_IP (ctx, (guint8*)MONO_CONTEXT_GET_IP (ctx) + single_step_fault_size);
8642 }
8643
8644 /*
8645  * mono_arch_create_seq_point_info:
8646  *
8647  *   Return a pointer to a data structure which is used by the sequence
8648  * point implementation in AOTed code.
8649  */
8650 gpointer
8651 mono_arch_get_seq_point_info (MonoDomain *domain, guint8 *code)
8652 {
8653         SeqPointInfo *info;
8654         MonoJitInfo *ji;
8655         int i;
8656
8657         // FIXME: Add a free function
8658
8659         mono_domain_lock (domain);
8660         info = g_hash_table_lookup (domain_jit_info (domain)->arch_seq_points,
8661                                                                 code);
8662         mono_domain_unlock (domain);
8663
8664         if (!info) {
8665                 ji = mono_jit_info_table_find (domain, (char*)code);
8666                 g_assert (ji);
8667
8668                 // FIXME: Optimize the size
8669                 info = g_malloc0 (sizeof (SeqPointInfo) + (ji->code_size * sizeof (gpointer)));
8670
8671                 info->ss_trigger_page = ss_trigger_page;
8672                 info->bp_trigger_page = bp_trigger_page;
8673                 /* Initialize to a valid address */
8674                 for (i = 0; i < ji->code_size; ++i)
8675                         info->bp_addrs [i] = info;
8676
8677                 mono_domain_lock (domain);
8678                 g_hash_table_insert (domain_jit_info (domain)->arch_seq_points,
8679                                                          code, info);
8680                 mono_domain_unlock (domain);
8681         }
8682
8683         return info;
8684 }
8685
8686 #endif