mono/mini/mini-amd64.c

   1 /*
   2  * mini-amd64.c: AMD64 backend for the Mono code generator
   3  *
   4  * Based on mini-x86.c.
   5  *
   6  * Authors:
   7  *   Paolo Molaro (lupus@ximian.com)
   8  *   Dietmar Maurer (dietmar@ximian.com)
   9  *   Patrik Torstensson
  10  *   Zoltan Varga (vargaz@gmail.com)
  11  *
  12  * (C) 2003 Ximian, Inc.
  13  */
  14 #include "mini.h"
  15 #include <string.h>
  16 #include <math.h>
  17 #ifdef HAVE_UNISTD_H
  18 #include <unistd.h>
  19 #endif
  20
  21 #include <mono/metadata/appdomain.h>
  22 #include <mono/metadata/debug-helpers.h>
  23 #include <mono/metadata/threads.h>
  24 #include <mono/metadata/profiler-private.h>
  25 #include <mono/metadata/mono-debug.h>
  26 #include <mono/utils/mono-math.h>
  27 #include <mono/utils/mono-mmap.h>
  28
  29 #include "trace.h"
  30 #include "ir-emit.h"
  31 #include "mini-amd64.h"
  32 #include "cpu-amd64.h"
  33 #include "debugger-agent.h"
  34
  35 /*
  36  * Can't define this in mini-amd64.h cause that would turn on the generic code in
  37  * method-to-ir.c.
  38  */
  39 #define MONO_ARCH_IMT_REG AMD64_R11
  40
  41 static gint lmf_tls_offset = -1;
  42 static gint lmf_addr_tls_offset = -1;
  43 static gint appdomain_tls_offset = -1;
  44
  45 #ifdef MONO_XEN_OPT
  46 static gboolean optimize_for_xen = TRUE;
  47 #else
  48 #define optimize_for_xen 0
  49 #endif
  50
  51 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
  52
  53 #define IS_IMM32(val) ((((guint64)val) >> 32) == 0)
  54
  55 #define IS_REX(inst) (((inst) >= 0x40) && ((inst) <= 0x4f))
  56
  57 #ifdef HOST_WIN32
  58 /* Under windows, the calling convention is never stdcall */
  59 #define CALLCONV_IS_STDCALL(call_conv) (FALSE)
  60 #else
  61 #define CALLCONV_IS_STDCALL(call_conv) ((call_conv) == MONO_CALL_STDCALL)
  62 #endif
  63
  64 /* This mutex protects architecture specific caches */
  65 #define mono_mini_arch_lock() EnterCriticalSection (&mini_arch_mutex)
  66 #define mono_mini_arch_unlock() LeaveCriticalSection (&mini_arch_mutex)
  67 static CRITICAL_SECTION mini_arch_mutex;
  68
  69 MonoBreakpointInfo
  70 mono_breakpoint_info [MONO_BREAKPOINT_ARRAY_SIZE];
  71
  72 /*
  73  * The code generated for sequence points reads from this location, which is
  74  * made read-only when single stepping is enabled.
  75  */
  76 static gpointer ss_trigger_page;
  77
  78 /* Enabled breakpoints read from this trigger page */
  79 static gpointer bp_trigger_page;
  80
  81 /* The size of the breakpoint sequence */
  82 static int breakpoint_size;
  83
  84 /* The size of the breakpoint instruction causing the actual fault */
  85 static int breakpoint_fault_size;
  86
  87 /* The size of the single step instruction causing the actual fault */
  88 static int single_step_fault_size;
  89
  90 #ifdef HOST_WIN32
  91 /* On Win64 always reserve first 32 bytes for first four arguments */
  92 #define ARGS_OFFSET 48
  93 #else
  94 #define ARGS_OFFSET 16
  95 #endif
  96 #define GP_SCRATCH_REG AMD64_R11
  97
  98 /*
  99  * AMD64 register usage:
 100  * - callee saved registers are used for global register allocation
 101  * - %r11 is used for materializing 64 bit constants in opcodes
 102  * - the rest is used for local allocation
 103  */
 104
 105 /*
 106  * Floating point comparison results:
 107  *                  ZF PF CF
 108  * A > B            0  0  0
 109  * A < B            0  0  1
 110  * A = B            1  0  0
 111  * A > B            0  0  0
 112  * UNORDERED        1  1  1
 113  */
 114
 115 const char*
 116 mono_arch_regname (int reg)
 117 {
 118         switch (reg) {
 119         case AMD64_RAX: return "%rax";
 120         case AMD64_RBX: return "%rbx";
 121         case AMD64_RCX: return "%rcx";
 122         case AMD64_RDX: return "%rdx";
 123         case AMD64_RSP: return "%rsp";
 124         case AMD64_RBP: return "%rbp";
 125         case AMD64_RDI: return "%rdi";
 126         case AMD64_RSI: return "%rsi";
 127         case AMD64_R8: return "%r8";
 128         case AMD64_R9: return "%r9";
 129         case AMD64_R10: return "%r10";
 130         case AMD64_R11: return "%r11";
 131         case AMD64_R12: return "%r12";
 132         case AMD64_R13: return "%r13";
 133         case AMD64_R14: return "%r14";
 134         case AMD64_R15: return "%r15";
 135         }
 136         return "unknown";
 137 }
 138
 139 static const char * packed_xmmregs [] = {
 140         "p:xmm0", "p:xmm1", "p:xmm2", "p:xmm3", "p:xmm4", "p:xmm5", "p:xmm6", "p:xmm7", "p:xmm8",
 141         "p:xmm9", "p:xmm10", "p:xmm11", "p:xmm12", "p:xmm13", "p:xmm14", "p:xmm15"
 142 };
 143
 144 static const char * single_xmmregs [] = {
 145         "s:xmm0", "s:xmm1", "s:xmm2", "s:xmm3", "s:xmm4", "s:xmm5", "s:xmm6", "s:xmm7", "s:xmm8",
 146         "s:xmm9", "s:xmm10", "s:xmm11", "s:xmm12", "s:xmm13", "s:xmm14", "s:xmm15"
 147 };
 148
 149 const char*
 150 mono_arch_fregname (int reg)
 151 {
 152         if (reg < AMD64_XMM_NREG)
 153                 return single_xmmregs [reg];
 154         else
 155                 return "unknown";
 156 }
 157
 158 const char *
 159 mono_arch_xregname (int reg)
 160 {
 161         if (reg < AMD64_XMM_NREG)
 162                 return packed_xmmregs [reg];
 163         else
 164                 return "unknown";
 165 }
 166
 167 G_GNUC_UNUSED static void
 168 break_count (void)
 169 {
 170 }
 171
 172 G_GNUC_UNUSED static gboolean
 173 debug_count (void)
 174 {
 175         static int count = 0;
 176         count ++;
 177
 178         if (!getenv ("COUNT"))
 179                 return TRUE;
 180
 181         if (count == atoi (getenv ("COUNT"))) {
 182                 break_count ();
 183         }
 184
 185         if (count > atoi (getenv ("COUNT"))) {
 186                 return FALSE;
 187         }
 188
 189         return TRUE;
 190 }
 191
 192 static gboolean
 193 debug_omit_fp (void)
 194 {
 195 #if 0
 196         return debug_count ();
 197 #else
 198         return TRUE;
 199 #endif
 200 }
 201
 202 static inline gboolean
 203 amd64_is_near_call (guint8 *code)
 204 {
 205         /* Skip REX */
 206         if ((code [0] >= 0x40) && (code [0] <= 0x4f))
 207                 code += 1;
 208
 209         return code [0] == 0xe8;
 210 }
 211
 212 static inline void
 213 amd64_patch (unsigned char* code, gpointer target)
 214 {
 215         guint8 rex = 0;
 216
 217         /* Skip REX */
 218         if ((code [0] >= 0x40) && (code [0] <= 0x4f)) {
 219                 rex = code [0];
 220                 code += 1;
 221         }
 222
 223         if ((code [0] & 0xf8) == 0xb8) {
 224                 /* amd64_set_reg_template */
 225                 *(guint64*)(code + 1) = (guint64)target;
 226         }
 227         else if ((code [0] == 0x8b) && rex && x86_modrm_mod (code [1]) == 0 && x86_modrm_rm (code [1]) == 5) {
 228                 /* mov 0(%rip), %dreg */
 229                 *(guint32*)(code + 2) = (guint32)(guint64)target - 7;
 230         }
 231         else if ((code [0] == 0xff) && (code [1] == 0x15)) {
 232                 /* call *<OFFSET>(%rip) */
 233                 *(guint32*)(code + 2) = ((guint32)(guint64)target) - 7;
 234         }
 235         else if ((code [0] == 0xe8)) {
 236                 /* call <DISP> */
 237                 gint64 disp = (guint8*)target - (guint8*)code;
 238                 g_assert (amd64_is_imm32 (disp));
 239                 x86_patch (code, (unsigned char*)target);
 240         }
 241         else
 242                 x86_patch (code, (unsigned char*)target);
 243 }
 244
 245 void
 246 mono_amd64_patch (unsigned char* code, gpointer target)
 247 {
 248         amd64_patch (code, target);
 249 }
 250
 251 typedef enum {
 252         ArgInIReg,
 253         ArgInFloatSSEReg,
 254         ArgInDoubleSSEReg,
 255         ArgOnStack,
 256         ArgValuetypeInReg,
 257         ArgValuetypeAddrInIReg,
 258         ArgNone /* only in pair_storage */
 259 } ArgStorage;
 260
 261 typedef struct {
 262         gint16 offset;
 263         gint8  reg;
 264         ArgStorage storage;
 265
 266         /* Only if storage == ArgValuetypeInReg */
 267         ArgStorage pair_storage [2];
 268         gint8 pair_regs [2];
 269 } ArgInfo;
 270
 271 typedef struct {
 272         int nargs;
 273         guint32 stack_usage;
 274         guint32 reg_usage;
 275         guint32 freg_usage;
 276         gboolean need_stack_align;
 277         gboolean vtype_retaddr;
 278         ArgInfo ret;
 279         ArgInfo sig_cookie;
 280         ArgInfo args [1];
 281 } CallInfo;
 282
 283 #define DEBUG(a) if (cfg->verbose_level > 1) a
 284
 285 #ifdef HOST_WIN32
 286 #define PARAM_REGS 4
 287
 288 static AMD64_Reg_No param_regs [] = { AMD64_RCX, AMD64_RDX, AMD64_R8, AMD64_R9 };
 289
 290 static AMD64_Reg_No return_regs [] = { AMD64_RAX, AMD64_RDX };
 291 #else
 292 #define PARAM_REGS 6
 293
 294 static AMD64_Reg_No param_regs [] = { AMD64_RDI, AMD64_RSI, AMD64_RDX, AMD64_RCX, AMD64_R8, AMD64_R9 };
 295
 296  static AMD64_Reg_No return_regs [] = { AMD64_RAX, AMD64_RDX };
 297 #endif
 298
 299 static void inline
 300 add_general (guint32 *gr, guint32 *stack_size, ArgInfo *ainfo)
 301 {
 302     ainfo->offset = *stack_size;
 303
 304     if (*gr >= PARAM_REGS) {
 305                 ainfo->storage = ArgOnStack;
 306                 (*stack_size) += sizeof (gpointer);
 307     }
 308     else {
 309                 ainfo->storage = ArgInIReg;
 310                 ainfo->reg = param_regs [*gr];
 311                 (*gr) ++;
 312     }
 313 }
 314
 315 #ifdef HOST_WIN32
 316 #define FLOAT_PARAM_REGS 4
 317 #else
 318 #define FLOAT_PARAM_REGS 8
 319 #endif
 320
 321 static void inline
 322 add_float (guint32 *gr, guint32 *stack_size, ArgInfo *ainfo, gboolean is_double)
 323 {
 324     ainfo->offset = *stack_size;
 325
 326     if (*gr >= FLOAT_PARAM_REGS) {
 327                 ainfo->storage = ArgOnStack;
 328                 (*stack_size) += sizeof (gpointer);
 329     }
 330     else {
 331                 /* A double register */
 332                 if (is_double)
 333                         ainfo->storage = ArgInDoubleSSEReg;
 334                 else
 335                         ainfo->storage = ArgInFloatSSEReg;
 336                 ainfo->reg = *gr;
 337                 (*gr) += 1;
 338     }
 339 }
 340
 341 typedef enum ArgumentClass {
 342         ARG_CLASS_NO_CLASS,
 343         ARG_CLASS_MEMORY,
 344         ARG_CLASS_INTEGER,
 345         ARG_CLASS_SSE
 346 } ArgumentClass;
 347
 348 static ArgumentClass
 349 merge_argument_class_from_type (MonoType *type, ArgumentClass class1)
 350 {
 351         ArgumentClass class2 = ARG_CLASS_NO_CLASS;
 352         MonoType *ptype;
 353
 354         ptype = mini_type_get_underlying_type (NULL, type);
 355         switch (ptype->type) {
 356         case MONO_TYPE_BOOLEAN:
 357         case MONO_TYPE_CHAR:
 358         case MONO_TYPE_I1:
 359         case MONO_TYPE_U1:
 360         case MONO_TYPE_I2:
 361         case MONO_TYPE_U2:
 362         case MONO_TYPE_I4:
 363         case MONO_TYPE_U4:
 364         case MONO_TYPE_I:
 365         case MONO_TYPE_U:
 366         case MONO_TYPE_STRING:
 367         case MONO_TYPE_OBJECT:
 368         case MONO_TYPE_CLASS:
 369         case MONO_TYPE_SZARRAY:
 370         case MONO_TYPE_PTR:
 371         case MONO_TYPE_FNPTR:
 372         case MONO_TYPE_ARRAY:
 373         case MONO_TYPE_I8:
 374         case MONO_TYPE_U8:
 375                 class2 = ARG_CLASS_INTEGER;
 376                 break;
 377         case MONO_TYPE_R4:
 378         case MONO_TYPE_R8:
 379 #ifdef HOST_WIN32
 380                 class2 = ARG_CLASS_INTEGER;
 381 #else
 382                 class2 = ARG_CLASS_SSE;
 383 #endif
 384                 break;
 385
 386         case MONO_TYPE_TYPEDBYREF:
 387                 g_assert_not_reached ();
 388
 389         case MONO_TYPE_GENERICINST:
 390                 if (!mono_type_generic_inst_is_valuetype (ptype)) {
 391                         class2 = ARG_CLASS_INTEGER;
 392                         break;
 393                 }
 394                 /* fall through */
 395         case MONO_TYPE_VALUETYPE: {
 396                 MonoMarshalType *info = mono_marshal_load_type_info (ptype->data.klass);
 397                 int i;
 398
 399                 for (i = 0; i < info->num_fields; ++i) {
 400                         class2 = class1;
 401                         class2 = merge_argument_class_from_type (info->fields [i].field->type, class2);
 402                 }
 403                 break;
 404         }
 405         default:
 406                 g_assert_not_reached ();
 407         }
 408
 409         /* Merge */
 410         if (class1 == class2)
 411                 ;
 412         else if (class1 == ARG_CLASS_NO_CLASS)
 413                 class1 = class2;
 414         else if ((class1 == ARG_CLASS_MEMORY) || (class2 == ARG_CLASS_MEMORY))
 415                 class1 = ARG_CLASS_MEMORY;
 416         else if ((class1 == ARG_CLASS_INTEGER) || (class2 == ARG_CLASS_INTEGER))
 417                 class1 = ARG_CLASS_INTEGER;
 418         else
 419                 class1 = ARG_CLASS_SSE;
 420
 421         return class1;
 422 }
 423
 424 static void
 425 add_valuetype (MonoGenericSharingContext *gsctx, MonoMethodSignature *sig, ArgInfo *ainfo, MonoType *type,
 426                            gboolean is_return,
 427                            guint32 *gr, guint32 *fr, guint32 *stack_size)
 428 {
 429         guint32 size, quad, nquads, i;
 430         ArgumentClass args [2];
 431         MonoMarshalType *info = NULL;
 432         MonoClass *klass;
 433         MonoGenericSharingContext tmp_gsctx;
 434         gboolean pass_on_stack = FALSE;
 435
 436         /*
 437          * The gsctx currently contains no data, it is only used for checking whenever
 438          * open types are allowed, some callers like mono_arch_get_argument_info ()
 439          * don't pass it to us, so work around that.
 440          */
 441         if (!gsctx)
 442                 gsctx = &tmp_gsctx;
 443
 444         klass = mono_class_from_mono_type (type);
 445         size = mini_type_stack_size_full (gsctx, &klass->byval_arg, NULL, sig->pinvoke);
 446 #ifndef HOST_WIN32
 447         if (!sig->pinvoke && !disable_vtypes_in_regs && ((is_return && (size == 8)) || (!is_return && (size <= 16)))) {
 448                 /* We pass and return vtypes of size 8 in a register */
 449         } else if (!sig->pinvoke || (size == 0) || (size > 16)) {
 450                 pass_on_stack = TRUE;
 451         }
 452 #else
 453         if (!sig->pinvoke) {
 454                 pass_on_stack = TRUE;
 455         }
 456 #endif
 457
 458         if (pass_on_stack) {
 459                 /* Allways pass in memory */
 460                 ainfo->offset = *stack_size;
 461                 *stack_size += ALIGN_TO (size, 8);
 462                 ainfo->storage = ArgOnStack;
 463
 464                 return;
 465         }
 466
 467         /* FIXME: Handle structs smaller than 8 bytes */
 468         //if ((size % 8) != 0)
 469         //      NOT_IMPLEMENTED;
 470
 471         if (size > 8)
 472                 nquads = 2;
 473         else
 474                 nquads = 1;
 475
 476         if (!sig->pinvoke) {
 477                 /* Always pass in 1 or 2 integer registers */
 478                 args [0] = ARG_CLASS_INTEGER;
 479                 args [1] = ARG_CLASS_INTEGER;
 480                 /* Only the simplest cases are supported */
 481                 if (is_return && nquads != 1) {
 482                         args [0] = ARG_CLASS_MEMORY;
 483                         args [1] = ARG_CLASS_MEMORY;
 484                 }
 485         } else {
 486                 /*
 487                  * Implement the algorithm from section 3.2.3 of the X86_64 ABI.
 488                  * The X87 and SSEUP stuff is left out since there are no such types in
 489                  * the CLR.
 490                  */
 491                 info = mono_marshal_load_type_info (klass);
 492                 g_assert (info);
 493
 494 #ifndef HOST_WIN32
 495                 if (info->native_size > 16) {
 496                         ainfo->offset = *stack_size;
 497                         *stack_size += ALIGN_TO (info->native_size, 8);
 498                         ainfo->storage = ArgOnStack;
 499
 500                         return;
 501                 }
 502 #else
 503                 switch (info->native_size) {
 504                 case 1: case 2: case 4: case 8:
 505                         break;
 506                 default:
 507                         if (is_return) {
 508                                 ainfo->storage = ArgOnStack;
 509                                 ainfo->offset = *stack_size;
 510                                 *stack_size += ALIGN_TO (info->native_size, 8);
 511                         }
 512                         else {
 513                                 ainfo->storage = ArgValuetypeAddrInIReg;
 514
 515                                 if (*gr < PARAM_REGS) {
 516                                         ainfo->pair_storage [0] = ArgInIReg;
 517                                         ainfo->pair_regs [0] = param_regs [*gr];
 518                                         (*gr) ++;
 519                                 }
 520                                 else {
 521                                         ainfo->pair_storage [0] = ArgOnStack;
 522                                         ainfo->offset = *stack_size;
 523                                         *stack_size += 8;
 524                                 }
 525                         }
 526
 527                         return;
 528                 }
 529 #endif
 530
 531                 args [0] = ARG_CLASS_NO_CLASS;
 532                 args [1] = ARG_CLASS_NO_CLASS;
 533                 for (quad = 0; quad < nquads; ++quad) {
 534                         int size;
 535                         guint32 align;
 536                         ArgumentClass class1;
 537
 538                         if (info->num_fields == 0)
 539                                 class1 = ARG_CLASS_MEMORY;
 540                         else
 541                                 class1 = ARG_CLASS_NO_CLASS;
 542                         for (i = 0; i < info->num_fields; ++i) {
 543                                 size = mono_marshal_type_size (info->fields [i].field->type,
 544                                                                                            info->fields [i].mspec,
 545                                                                                            &align, TRUE, klass->unicode);
 546                                 if ((info->fields [i].offset < 8) && (info->fields [i].offset + size) > 8) {
 547                                         /* Unaligned field */
 548                                         NOT_IMPLEMENTED;
 549                                 }
 550
 551                                 /* Skip fields in other quad */
 552                                 if ((quad == 0) && (info->fields [i].offset >= 8))
 553                                         continue;
 554                                 if ((quad == 1) && (info->fields [i].offset < 8))
 555                                         continue;
 556
 557                                 class1 = merge_argument_class_from_type (info->fields [i].field->type, class1);
 558                         }
 559                         g_assert (class1 != ARG_CLASS_NO_CLASS);
 560                         args [quad] = class1;
 561                 }
 562         }
 563
 564         /* Post merger cleanup */
 565         if ((args [0] == ARG_CLASS_MEMORY) || (args [1] == ARG_CLASS_MEMORY))
 566                 args [0] = args [1] = ARG_CLASS_MEMORY;
 567
 568         /* Allocate registers */
 569         {
 570                 int orig_gr = *gr;
 571                 int orig_fr = *fr;
 572
 573                 ainfo->storage = ArgValuetypeInReg;
 574                 ainfo->pair_storage [0] = ainfo->pair_storage [1] = ArgNone;
 575                 for (quad = 0; quad < nquads; ++quad) {
 576                         switch (args [quad]) {
 577                         case ARG_CLASS_INTEGER:
 578                                 if (*gr >= PARAM_REGS)
 579                                         args [quad] = ARG_CLASS_MEMORY;
 580                                 else {
 581                                         ainfo->pair_storage [quad] = ArgInIReg;
 582                                         if (is_return)
 583                                                 ainfo->pair_regs [quad] = return_regs [*gr];
 584                                         else
 585                                                 ainfo->pair_regs [quad] = param_regs [*gr];
 586                                         (*gr) ++;
 587                                 }
 588                                 break;
 589                         case ARG_CLASS_SSE:
 590                                 if (*fr >= FLOAT_PARAM_REGS)
 591                                         args [quad] = ARG_CLASS_MEMORY;
 592                                 else {
 593                                         ainfo->pair_storage [quad] = ArgInDoubleSSEReg;
 594                                         ainfo->pair_regs [quad] = *fr;
 595                                         (*fr) ++;
 596                                 }
 597                                 break;
 598                         case ARG_CLASS_MEMORY:
 599                                 break;
 600                         default:
 601                                 g_assert_not_reached ();
 602                         }
 603                 }
 604
 605                 if ((args [0] == ARG_CLASS_MEMORY) || (args [1] == ARG_CLASS_MEMORY)) {
 606                         /* Revert possible register assignments */
 607                         *gr = orig_gr;
 608                         *fr = orig_fr;
 609
 610                         ainfo->offset = *stack_size;
 611                         if (sig->pinvoke)
 612                                 *stack_size += ALIGN_TO (info->native_size, 8);
 613                         else
 614                                 *stack_size += nquads * sizeof (gpointer);
 615                         ainfo->storage = ArgOnStack;
 616                 }
 617         }
 618 }
 619
 620 /*
 621  * get_call_info:
 622  *
 623  *  Obtain information about a call according to the calling convention.
 624  * For AMD64, see the "System V ABI, x86-64 Architecture Processor Supplement
 625  * Draft Version 0.23" document for more information.
 626  */
 627 static CallInfo*
 628 get_call_info (MonoGenericSharingContext *gsctx, MonoMemPool *mp, MonoMethodSignature *sig, gboolean is_pinvoke)
 629 {
 630         guint32 i, gr, fr;
 631         MonoType *ret_type;
 632         int n = sig->hasthis + sig->param_count;
 633         guint32 stack_size = 0;
 634         CallInfo *cinfo;
 635
 636         if (mp)
 637                 cinfo = mono_mempool_alloc0 (mp, sizeof (CallInfo) + (sizeof (ArgInfo) * n));
 638         else
 639                 cinfo = g_malloc0 (sizeof (CallInfo) + (sizeof (ArgInfo) * n));
 640
 641         cinfo->nargs = n;
 642
 643         gr = 0;
 644         fr = 0;
 645
 646         /* return value */
 647         {
 648                 ret_type = mini_type_get_underlying_type (gsctx, sig->ret);
 649                 switch (ret_type->type) {
 650                 case MONO_TYPE_BOOLEAN:
 651                 case MONO_TYPE_I1:
 652                 case MONO_TYPE_U1:
 653                 case MONO_TYPE_I2:
 654                 case MONO_TYPE_U2:
 655                 case MONO_TYPE_CHAR:
 656                 case MONO_TYPE_I4:
 657                 case MONO_TYPE_U4:
 658                 case MONO_TYPE_I:
 659                 case MONO_TYPE_U:
 660                 case MONO_TYPE_PTR:
 661                 case MONO_TYPE_FNPTR:
 662                 case MONO_TYPE_CLASS:
 663                 case MONO_TYPE_OBJECT:
 664                 case MONO_TYPE_SZARRAY:
 665                 case MONO_TYPE_ARRAY:
 666                 case MONO_TYPE_STRING:
 667                         cinfo->ret.storage = ArgInIReg;
 668                         cinfo->ret.reg = AMD64_RAX;
 669                         break;
 670                 case MONO_TYPE_U8:
 671                 case MONO_TYPE_I8:
 672                         cinfo->ret.storage = ArgInIReg;
 673                         cinfo->ret.reg = AMD64_RAX;
 674                         break;
 675                 case MONO_TYPE_R4:
 676                         cinfo->ret.storage = ArgInFloatSSEReg;
 677                         cinfo->ret.reg = AMD64_XMM0;
 678                         break;
 679                 case MONO_TYPE_R8:
 680                         cinfo->ret.storage = ArgInDoubleSSEReg;
 681                         cinfo->ret.reg = AMD64_XMM0;
 682                         break;
 683                 case MONO_TYPE_GENERICINST:
 684                         if (!mono_type_generic_inst_is_valuetype (ret_type)) {
 685                                 cinfo->ret.storage = ArgInIReg;
 686                                 cinfo->ret.reg = AMD64_RAX;
 687                                 break;
 688                         }
 689                         /* fall through */
 690                 case MONO_TYPE_VALUETYPE: {
 691                         guint32 tmp_gr = 0, tmp_fr = 0, tmp_stacksize = 0;
 692
 693                         add_valuetype (gsctx, sig, &cinfo->ret, sig->ret, TRUE, &tmp_gr, &tmp_fr, &tmp_stacksize);
 694                         if (cinfo->ret.storage == ArgOnStack) {
 695                                 cinfo->vtype_retaddr = TRUE;
 696                                 /* The caller passes the address where the value is stored */
 697                                 add_general (&gr, &stack_size, &cinfo->ret);
 698                         }
 699                         break;
 700                 }
 701                 case MONO_TYPE_TYPEDBYREF:
 702                         /* Same as a valuetype with size 24 */
 703                         add_general (&gr, &stack_size, &cinfo->ret);
 704                         ;
 705                         break;
 706                 case MONO_TYPE_VOID:
 707                         break;
 708                 default:
 709                         g_error ("Can't handle as return value 0x%x", sig->ret->type);
 710                 }
 711         }
 712
 713         /* this */
 714         if (sig->hasthis)
 715                 add_general (&gr, &stack_size, cinfo->args + 0);
 716
 717         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n == 0)) {
 718                 gr = PARAM_REGS;
 719                 fr = FLOAT_PARAM_REGS;
 720
 721                 /* Emit the signature cookie just before the implicit arguments */
 722                 add_general (&gr, &stack_size, &cinfo->sig_cookie);
 723         }
 724
 725         for (i = 0; i < sig->param_count; ++i) {
 726                 ArgInfo *ainfo = &cinfo->args [sig->hasthis + i];
 727                 MonoType *ptype;
 728
 729 #ifdef HOST_WIN32
 730                 /* The float param registers and other param registers must be the same index on Windows x64.*/
 731                 if (gr > fr)
 732                         fr = gr;
 733                 else if (fr > gr)
 734                         gr = fr;
 735 #endif
 736
 737                 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
 738                         /* We allways pass the sig cookie on the stack for simplicity */
 739                         /*
 740                          * Prevent implicit arguments + the sig cookie from being passed
 741                          * in registers.
 742                          */
 743                         gr = PARAM_REGS;
 744                         fr = FLOAT_PARAM_REGS;
 745
 746                         /* Emit the signature cookie just before the implicit arguments */
 747                         add_general (&gr, &stack_size, &cinfo->sig_cookie);
 748                 }
 749
 750                 ptype = mini_type_get_underlying_type (gsctx, sig->params [i]);
 751                 switch (ptype->type) {
 752                 case MONO_TYPE_BOOLEAN:
 753                 case MONO_TYPE_I1:
 754                 case MONO_TYPE_U1:
 755                         add_general (&gr, &stack_size, ainfo);
 756                         break;
 757                 case MONO_TYPE_I2:
 758                 case MONO_TYPE_U2:
 759                 case MONO_TYPE_CHAR:
 760                         add_general (&gr, &stack_size, ainfo);
 761                         break;
 762                 case MONO_TYPE_I4:
 763                 case MONO_TYPE_U4:
 764                         add_general (&gr, &stack_size, ainfo);
 765                         break;
 766                 case MONO_TYPE_I:
 767                 case MONO_TYPE_U:
 768                 case MONO_TYPE_PTR:
 769                 case MONO_TYPE_FNPTR:
 770                 case MONO_TYPE_CLASS:
 771                 case MONO_TYPE_OBJECT:
 772                 case MONO_TYPE_STRING:
 773                 case MONO_TYPE_SZARRAY:
 774                 case MONO_TYPE_ARRAY:
 775                         add_general (&gr, &stack_size, ainfo);
 776                         break;
 777                 case MONO_TYPE_GENERICINST:
 778                         if (!mono_type_generic_inst_is_valuetype (ptype)) {
 779                                 add_general (&gr, &stack_size, ainfo);
 780                                 break;
 781                         }
 782                         /* fall through */
 783                 case MONO_TYPE_VALUETYPE:
 784                         add_valuetype (gsctx, sig, ainfo, sig->params [i], FALSE, &gr, &fr, &stack_size);
 785                         break;
 786                 case MONO_TYPE_TYPEDBYREF:
 787 #ifdef HOST_WIN32
 788                         add_valuetype (gsctx, sig, ainfo, sig->params [i], FALSE, &gr, &fr, &stack_size);
 789 #else
 790                         stack_size += sizeof (MonoTypedRef);
 791                         ainfo->storage = ArgOnStack;
 792 #endif
 793                         break;
 794                 case MONO_TYPE_U8:
 795                 case MONO_TYPE_I8:
 796                         add_general (&gr, &stack_size, ainfo);
 797                         break;
 798                 case MONO_TYPE_R4:
 799                         add_float (&fr, &stack_size, ainfo, FALSE);
 800                         break;
 801                 case MONO_TYPE_R8:
 802                         add_float (&fr, &stack_size, ainfo, TRUE);
 803                         break;
 804                 default:
 805                         g_assert_not_reached ();
 806                 }
 807         }
 808
 809         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n > 0) && (sig->sentinelpos == sig->param_count)) {
 810                 gr = PARAM_REGS;
 811                 fr = FLOAT_PARAM_REGS;
 812
 813                 /* Emit the signature cookie just before the implicit arguments */
 814                 add_general (&gr, &stack_size, &cinfo->sig_cookie);
 815         }
 816
 817 #ifdef HOST_WIN32
 818         // There always is 32 bytes reserved on the stack when calling on Winx64
 819         stack_size += 0x20;
 820 #endif
 821
 822         if (stack_size & 0x8) {
 823                 /* The AMD64 ABI requires each stack frame to be 16 byte aligned */
 824                 cinfo->need_stack_align = TRUE;
 825                 stack_size += 8;
 826         }
 827
 828         cinfo->stack_usage = stack_size;
 829         cinfo->reg_usage = gr;
 830         cinfo->freg_usage = fr;
 831         return cinfo;
 832 }
 833
 834 /*
 835  * mono_arch_get_argument_info:
 836  * @csig:  a method signature
 837  * @param_count: the number of parameters to consider
 838  * @arg_info: an array to store the result infos
 839  *
 840  * Gathers information on parameters such as size, alignment and
 841  * padding. arg_info should be large enought to hold param_count + 1 entries.
 842  *
 843  * Returns the size of the argument area on the stack.
 844  */
 845 int
 846 mono_arch_get_argument_info (MonoMethodSignature *csig, int param_count, MonoJitArgumentInfo *arg_info)
 847 {
 848         int k;
 849         CallInfo *cinfo = get_call_info (NULL, NULL, csig, FALSE);
 850         guint32 args_size = cinfo->stack_usage;
 851
 852         /* The arguments are saved to a stack area in mono_arch_instrument_prolog */
 853         if (csig->hasthis) {
 854                 arg_info [0].offset = 0;
 855         }
 856
 857         for (k = 0; k < param_count; k++) {
 858                 arg_info [k + 1].offset = ((k + csig->hasthis) * 8);
 859                 /* FIXME: */
 860                 arg_info [k + 1].size = 0;
 861         }
 862
 863         g_free (cinfo);
 864
 865         return args_size;
 866 }
 867
 868 static int
 869 cpuid (int id, int* p_eax, int* p_ebx, int* p_ecx, int* p_edx)
 870 {
 871 #ifndef _MSC_VER
 872         __asm__ __volatile__ ("cpuid"
 873                 : "=a" (*p_eax), "=b" (*p_ebx), "=c" (*p_ecx), "=d" (*p_edx)
 874                 : "a" (id));
 875 #else
 876         int info[4];
 877         __cpuid(info, id);
 878         *p_eax = info[0];
 879         *p_ebx = info[1];
 880         *p_ecx = info[2];
 881         *p_edx = info[3];
 882 #endif
 883         return 1;
 884 }
 885
 886 /*
 887  * Initialize the cpu to execute managed code.
 888  */
 889 void
 890 mono_arch_cpu_init (void)
 891 {
 892 #ifndef _MSC_VER
 893         guint16 fpcw;
 894
 895         /* spec compliance requires running with double precision */
 896         __asm__  __volatile__ ("fnstcw %0\n": "=m" (fpcw));
 897         fpcw &= ~X86_FPCW_PRECC_MASK;
 898         fpcw |= X86_FPCW_PREC_DOUBLE;
 899         __asm__  __volatile__ ("fldcw %0\n": : "m" (fpcw));
 900         __asm__  __volatile__ ("fnstcw %0\n": "=m" (fpcw));
 901 #else
 902         /* TODO: This is crashing on Win64 right now.
 903         * _control87 (_PC_53, MCW_PC);
 904         */
 905 #endif
 906 }
 907
 908 /*
 909  * Initialize architecture specific code.
 910  */
 911 void
 912 mono_arch_init (void)
 913 {
 914         int flags;
 915
 916         InitializeCriticalSection (&mini_arch_mutex);
 917
 918 #ifdef MONO_ARCH_NOMAP32BIT
 919         flags = MONO_MMAP_READ;
 920         /* amd64_mov_reg_imm () + amd64_mov_reg_membase () */
 921         breakpoint_size = 13;
 922         breakpoint_fault_size = 3;
 923         /* amd64_alu_membase_imm_size (code, X86_CMP, AMD64_R11, 0, 0, 4); */
 924         single_step_fault_size = 5;
 925 #else
 926         flags = MONO_MMAP_READ|MONO_MMAP_32BIT;
 927         /* amd64_mov_reg_mem () */
 928         breakpoint_size = 8;
 929         breakpoint_fault_size = 8;
 930         single_step_fault_size = 8;
 931 #endif
 932
 933         ss_trigger_page = mono_valloc (NULL, mono_pagesize (), flags);
 934         bp_trigger_page = mono_valloc (NULL, mono_pagesize (), flags);
 935         mono_mprotect (bp_trigger_page, mono_pagesize (), 0);
 936 }
 937
 938 /*
 939  * Cleanup architecture specific code.
 940  */
 941 void
 942 mono_arch_cleanup (void)
 943 {
 944         DeleteCriticalSection (&mini_arch_mutex);
 945 }
 946
 947 /*
 948  * This function returns the optimizations supported on this cpu.
 949  */
 950 guint32
 951 mono_arch_cpu_optimizazions (guint32 *exclude_mask)
 952 {
 953         int eax, ebx, ecx, edx;
 954         guint32 opts = 0;
 955
 956         /* FIXME: AMD64 */
 957
 958         *exclude_mask = 0;
 959         /* Feature Flags function, flags returned in EDX. */
 960         if (cpuid (1, &eax, &ebx, &ecx, &edx)) {
 961                 if (edx & (1 << 15)) {
 962                         opts |= MONO_OPT_CMOV;
 963                         if (edx & 1)
 964                                 opts |= MONO_OPT_FCMOV;
 965                         else
 966                                 *exclude_mask |= MONO_OPT_FCMOV;
 967                 } else
 968                         *exclude_mask |= MONO_OPT_CMOV;
 969         }
 970
 971         return opts;
 972 }
 973
 974 /*
 975  * This function test for all SSE functions supported.
 976  *
 977  * Returns a bitmask corresponding to all supported versions.
 978  *
 979  */
 980 guint32
 981 mono_arch_cpu_enumerate_simd_versions (void)
 982 {
 983         int eax, ebx, ecx, edx;
 984         guint32 sse_opts = 0;
 985
 986         if (cpuid (1, &eax, &ebx, &ecx, &edx)) {
 987                 if (edx & (1 << 25))
 988                         sse_opts |= 1 << SIMD_VERSION_SSE1;
 989                 if (edx & (1 << 26))
 990                         sse_opts |= 1 << SIMD_VERSION_SSE2;
 991                 if (ecx & (1 << 0))
 992                         sse_opts |= 1 << SIMD_VERSION_SSE3;
 993                 if (ecx & (1 << 9))
 994                         sse_opts |= 1 << SIMD_VERSION_SSSE3;
 995                 if (ecx & (1 << 19))
 996                         sse_opts |= 1 << SIMD_VERSION_SSE41;
 997                 if (ecx & (1 << 20))
 998                         sse_opts |= 1 << SIMD_VERSION_SSE42;
 999         }
1000
1001         /* Yes, all this needs to be done to check for sse4a.
1002            See: "Amd: CPUID Specification"
1003          */
1004         if (cpuid (0x80000000, &eax, &ebx, &ecx, &edx)) {
1005                 /* eax greater or equal than 0x80000001, ebx = 'htuA', ecx = DMAc', edx = 'itne'*/
1006                 if ((((unsigned int) eax) >= 0x80000001) && (ebx == 0x68747541) && (ecx == 0x444D4163) && (edx == 0x69746E65)) {
1007                         cpuid (0x80000001, &eax, &ebx, &ecx, &edx);
1008                         if (ecx & (1 << 6))
1009                                 sse_opts |= 1 << SIMD_VERSION_SSE4a;
1010                 }
1011         }
1012
1013         return sse_opts;
1014 }
1015
1016 GList *
1017 mono_arch_get_allocatable_int_vars (MonoCompile *cfg)
1018 {
1019         GList *vars = NULL;
1020         int i;
1021
1022         for (i = 0; i < cfg->num_varinfo; i++) {
1023                 MonoInst *ins = cfg->varinfo [i];
1024                 MonoMethodVar *vmv = MONO_VARINFO (cfg, i);
1025
1026                 /* unused vars */
1027                 if (vmv->range.first_use.abs_pos >= vmv->range.last_use.abs_pos)
1028                         continue;
1029
1030                 if ((ins->flags & (MONO_INST_IS_DEAD|MONO_INST_VOLATILE|MONO_INST_INDIRECT)) ||
1031                     (ins->opcode != OP_LOCAL && ins->opcode != OP_ARG))
1032                         continue;
1033
1034                 if (mono_is_regsize_var (ins->inst_vtype)) {
1035                         g_assert (MONO_VARINFO (cfg, i)->reg == -1);
1036                         g_assert (i == vmv->idx);
1037                         vars = g_list_prepend (vars, vmv);
1038                 }
1039         }
1040
1041         vars = mono_varlist_sort (cfg, vars, 0);
1042
1043         return vars;
1044 }
1045
1046 /**
1047  * mono_arch_compute_omit_fp:
1048  *
1049  *   Determine whenever the frame pointer can be eliminated.
1050  */
1051 static void
1052 mono_arch_compute_omit_fp (MonoCompile *cfg)
1053 {
1054         MonoMethodSignature *sig;
1055         MonoMethodHeader *header;
1056         int i, locals_size;
1057         CallInfo *cinfo;
1058
1059         if (cfg->arch.omit_fp_computed)
1060                 return;
1061
1062         header = mono_method_get_header (cfg->method);
1063
1064         sig = mono_method_signature (cfg->method);
1065
1066         if (!cfg->arch.cinfo)
1067                 cfg->arch.cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig, FALSE);
1068         cinfo = cfg->arch.cinfo;
1069
1070         /*
1071          * FIXME: Remove some of the restrictions.
1072          */
1073         cfg->arch.omit_fp = TRUE;
1074         cfg->arch.omit_fp_computed = TRUE;
1075
1076         if (cfg->disable_omit_fp)
1077                 cfg->arch.omit_fp = FALSE;
1078
1079         if (!debug_omit_fp ())
1080                 cfg->arch.omit_fp = FALSE;
1081         /*
1082         if (cfg->method->save_lmf)
1083                 cfg->arch.omit_fp = FALSE;
1084         */
1085         if (cfg->flags & MONO_CFG_HAS_ALLOCA)
1086                 cfg->arch.omit_fp = FALSE;
1087         if (header->num_clauses)
1088                 cfg->arch.omit_fp = FALSE;
1089         if (cfg->param_area)
1090                 cfg->arch.omit_fp = FALSE;
1091         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG))
1092                 cfg->arch.omit_fp = FALSE;
1093         if ((mono_jit_trace_calls != NULL && mono_trace_eval (cfg->method)) ||
1094                 (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE))
1095                 cfg->arch.omit_fp = FALSE;
1096         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
1097                 ArgInfo *ainfo = &cinfo->args [i];
1098
1099                 if (ainfo->storage == ArgOnStack) {
1100                         /*
1101                          * The stack offset can only be determined when the frame
1102                          * size is known.
1103                          */
1104                         cfg->arch.omit_fp = FALSE;
1105                 }
1106         }
1107
1108         locals_size = 0;
1109         for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
1110                 MonoInst *ins = cfg->varinfo [i];
1111                 int ialign;
1112
1113                 locals_size += mono_type_size (ins->inst_vtype, &ialign);
1114         }
1115 }
1116
1117 GList *
1118 mono_arch_get_global_int_regs (MonoCompile *cfg)
1119 {
1120         GList *regs = NULL;
1121
1122         mono_arch_compute_omit_fp (cfg);
1123
1124         if (cfg->globalra) {
1125                 if (cfg->arch.omit_fp)
1126                         regs = g_list_prepend (regs, (gpointer)AMD64_RBP);
1127
1128                 regs = g_list_prepend (regs, (gpointer)AMD64_RBX);
1129                 regs = g_list_prepend (regs, (gpointer)AMD64_R12);
1130                 regs = g_list_prepend (regs, (gpointer)AMD64_R13);
1131                 regs = g_list_prepend (regs, (gpointer)AMD64_R14);
1132                 regs = g_list_prepend (regs, (gpointer)AMD64_R15);
1133
1134                 regs = g_list_prepend (regs, (gpointer)AMD64_R10);
1135                 regs = g_list_prepend (regs, (gpointer)AMD64_R9);
1136                 regs = g_list_prepend (regs, (gpointer)AMD64_R8);
1137                 regs = g_list_prepend (regs, (gpointer)AMD64_RDI);
1138                 regs = g_list_prepend (regs, (gpointer)AMD64_RSI);
1139                 regs = g_list_prepend (regs, (gpointer)AMD64_RDX);
1140                 regs = g_list_prepend (regs, (gpointer)AMD64_RCX);
1141                 regs = g_list_prepend (regs, (gpointer)AMD64_RAX);
1142         } else {
1143                 if (cfg->arch.omit_fp)
1144                         regs = g_list_prepend (regs, (gpointer)AMD64_RBP);
1145
1146                 /* We use the callee saved registers for global allocation */
1147                 regs = g_list_prepend (regs, (gpointer)AMD64_RBX);
1148                 regs = g_list_prepend (regs, (gpointer)AMD64_R12);
1149                 regs = g_list_prepend (regs, (gpointer)AMD64_R13);
1150                 regs = g_list_prepend (regs, (gpointer)AMD64_R14);
1151                 regs = g_list_prepend (regs, (gpointer)AMD64_R15);
1152 #ifdef HOST_WIN32
1153                 regs = g_list_prepend (regs, (gpointer)AMD64_RDI);
1154                 regs = g_list_prepend (regs, (gpointer)AMD64_RSI);
1155 #endif
1156         }
1157
1158         return regs;
1159 }
1160
1161 GList*
1162 mono_arch_get_global_fp_regs (MonoCompile *cfg)
1163 {
1164         GList *regs = NULL;
1165         int i;
1166
1167         /* All XMM registers */
1168         for (i = 0; i < 16; ++i)
1169                 regs = g_list_prepend (regs, GINT_TO_POINTER (i));
1170
1171         return regs;
1172 }
1173
1174 GList*
1175 mono_arch_get_iregs_clobbered_by_call (MonoCallInst *call)
1176 {
1177         static GList *r = NULL;
1178
1179         if (r == NULL) {
1180                 GList *regs = NULL;
1181
1182                 regs = g_list_prepend (regs, (gpointer)AMD64_RBP);
1183                 regs = g_list_prepend (regs, (gpointer)AMD64_RBX);
1184                 regs = g_list_prepend (regs, (gpointer)AMD64_R12);
1185                 regs = g_list_prepend (regs, (gpointer)AMD64_R13);
1186                 regs = g_list_prepend (regs, (gpointer)AMD64_R14);
1187                 regs = g_list_prepend (regs, (gpointer)AMD64_R15);
1188
1189                 regs = g_list_prepend (regs, (gpointer)AMD64_R10);
1190                 regs = g_list_prepend (regs, (gpointer)AMD64_R9);
1191                 regs = g_list_prepend (regs, (gpointer)AMD64_R8);
1192                 regs = g_list_prepend (regs, (gpointer)AMD64_RDI);
1193                 regs = g_list_prepend (regs, (gpointer)AMD64_RSI);
1194                 regs = g_list_prepend (regs, (gpointer)AMD64_RDX);
1195                 regs = g_list_prepend (regs, (gpointer)AMD64_RCX);
1196                 regs = g_list_prepend (regs, (gpointer)AMD64_RAX);
1197
1198                 InterlockedCompareExchangePointer ((gpointer*)&r, regs, NULL);
1199         }
1200
1201         return r;
1202 }
1203
1204 GList*
1205 mono_arch_get_fregs_clobbered_by_call (MonoCallInst *call)
1206 {
1207         int i;
1208         static GList *r = NULL;
1209
1210         if (r == NULL) {
1211                 GList *regs = NULL;
1212
1213                 for (i = 0; i < AMD64_XMM_NREG; ++i)
1214                         regs = g_list_prepend (regs, GINT_TO_POINTER (MONO_MAX_IREGS + i));
1215
1216                 InterlockedCompareExchangePointer ((gpointer*)&r, regs, NULL);
1217         }
1218
1219         return r;
1220 }
1221
1222 /*
1223  * mono_arch_regalloc_cost:
1224  *
1225  *  Return the cost, in number of memory references, of the action of
1226  * allocating the variable VMV into a register during global register
1227  * allocation.
1228  */
1229 guint32
1230 mono_arch_regalloc_cost (MonoCompile *cfg, MonoMethodVar *vmv)
1231 {
1232         MonoInst *ins = cfg->varinfo [vmv->idx];
1233
1234         if (cfg->method->save_lmf)
1235                 /* The register is already saved */
1236                 /* substract 1 for the invisible store in the prolog */
1237                 return (ins->opcode == OP_ARG) ? 0 : 1;
1238         else
1239                 /* push+pop */
1240                 return (ins->opcode == OP_ARG) ? 1 : 2;
1241 }
1242
1243 /*
1244  * mono_arch_fill_argument_info:
1245  *
1246  *   Populate cfg->args, cfg->ret and cfg->vret_addr with information about the arguments
1247  * of the method.
1248  */
1249 void
1250 mono_arch_fill_argument_info (MonoCompile *cfg)
1251 {
1252         MonoMethodSignature *sig;
1253         MonoMethodHeader *header;
1254         MonoInst *ins;
1255         int i;
1256         CallInfo *cinfo;
1257
1258         header = mono_method_get_header (cfg->method);
1259
1260         sig = mono_method_signature (cfg->method);
1261
1262         cinfo = cfg->arch.cinfo;
1263
1264         /*
1265          * Contrary to mono_arch_allocate_vars (), the information should describe
1266          * where the arguments are at the beginning of the method, not where they can be
1267          * accessed during the execution of the method. The later makes no sense for the
1268          * global register allocator, since a variable can be in more than one location.
1269          */
1270         if (sig->ret->type != MONO_TYPE_VOID) {
1271                 switch (cinfo->ret.storage) {
1272                 case ArgInIReg:
1273                 case ArgInFloatSSEReg:
1274                 case ArgInDoubleSSEReg:
1275                         if ((MONO_TYPE_ISSTRUCT (sig->ret) && !mono_class_from_mono_type (sig->ret)->enumtype) || (sig->ret->type == MONO_TYPE_TYPEDBYREF)) {
1276                                 cfg->vret_addr->opcode = OP_REGVAR;
1277                                 cfg->vret_addr->inst_c0 = cinfo->ret.reg;
1278                         }
1279                         else {
1280                                 cfg->ret->opcode = OP_REGVAR;
1281                                 cfg->ret->inst_c0 = cinfo->ret.reg;
1282                         }
1283                         break;
1284                 case ArgValuetypeInReg:
1285                         cfg->ret->opcode = OP_REGOFFSET;
1286                         cfg->ret->inst_basereg = -1;
1287                         cfg->ret->inst_offset = -1;
1288                         break;
1289                 default:
1290                         g_assert_not_reached ();
1291                 }
1292         }
1293
1294         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
1295                 ArgInfo *ainfo = &cinfo->args [i];
1296                 MonoType *arg_type;
1297
1298                 ins = cfg->args [i];
1299
1300                 if (sig->hasthis && (i == 0))
1301                         arg_type = &mono_defaults.object_class->byval_arg;
1302                 else
1303                         arg_type = sig->params [i - sig->hasthis];
1304
1305                 switch (ainfo->storage) {
1306                 case ArgInIReg:
1307                 case ArgInFloatSSEReg:
1308                 case ArgInDoubleSSEReg:
1309                         ins->opcode = OP_REGVAR;
1310                         ins->inst_c0 = ainfo->reg;
1311                         break;
1312                 case ArgOnStack:
1313                         ins->opcode = OP_REGOFFSET;
1314                         ins->inst_basereg = -1;
1315                         ins->inst_offset = -1;
1316                         break;
1317                 case ArgValuetypeInReg:
1318                         /* Dummy */
1319                         ins->opcode = OP_NOP;
1320                         break;
1321                 default:
1322                         g_assert_not_reached ();
1323                 }
1324         }
1325 }
1326
1327 void
1328 mono_arch_allocate_vars (MonoCompile *cfg)
1329 {
1330         MonoMethodSignature *sig;
1331         MonoMethodHeader *header;
1332         MonoInst *ins;
1333         int i, offset;
1334         guint32 locals_stack_size, locals_stack_align;
1335         gint32 *offsets;
1336         CallInfo *cinfo;
1337
1338         header = mono_method_get_header (cfg->method);
1339
1340         sig = mono_method_signature (cfg->method);
1341
1342         cinfo = cfg->arch.cinfo;
1343
1344         mono_arch_compute_omit_fp (cfg);
1345
1346         /*
1347          * We use the ABI calling conventions for managed code as well.
1348          * Exception: valuetypes are only sometimes passed or returned in registers.
1349          */
1350
1351         /*
1352          * The stack looks like this:
1353          * <incoming arguments passed on the stack>
1354          * <return value>
1355          * <lmf/caller saved registers>
1356          * <locals>
1357          * <spill area>
1358          * <localloc area>  -> grows dynamically
1359          * <params area>
1360          */
1361
1362         if (cfg->arch.omit_fp) {
1363                 cfg->flags |= MONO_CFG_HAS_SPILLUP;
1364                 cfg->frame_reg = AMD64_RSP;
1365                 offset = 0;
1366         } else {
1367                 /* Locals are allocated backwards from %fp */
1368                 cfg->frame_reg = AMD64_RBP;
1369                 offset = 0;
1370         }
1371
1372         if (cfg->method->save_lmf) {
1373                 /* Reserve stack space for saving LMF */
1374                 if (cfg->arch.omit_fp) {
1375                         cfg->arch.lmf_offset = offset;
1376                         offset += sizeof (MonoLMF);
1377                 }
1378                 else {
1379                         offset += sizeof (MonoLMF);
1380                         cfg->arch.lmf_offset = -offset;
1381                 }
1382         } else {
1383                 if (cfg->arch.omit_fp)
1384                         cfg->arch.reg_save_area_offset = offset;
1385                 /* Reserve space for caller saved registers */
1386                 for (i = 0; i < AMD64_NREG; ++i)
1387                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
1388                                 offset += sizeof (gpointer);
1389                         }
1390         }
1391
1392         if (sig->ret->type != MONO_TYPE_VOID) {
1393                 switch (cinfo->ret.storage) {
1394                 case ArgInIReg:
1395                 case ArgInFloatSSEReg:
1396                 case ArgInDoubleSSEReg:
1397                         if ((MONO_TYPE_ISSTRUCT (sig->ret) && !mono_class_from_mono_type (sig->ret)->enumtype) || (sig->ret->type == MONO_TYPE_TYPEDBYREF)) {
1398                                 if (cfg->globalra) {
1399                                         cfg->vret_addr->opcode = OP_REGVAR;
1400                                         cfg->vret_addr->inst_c0 = cinfo->ret.reg;
1401                                 } else {
1402                                         /* The register is volatile */
1403                                         cfg->vret_addr->opcode = OP_REGOFFSET;
1404                                         cfg->vret_addr->inst_basereg = cfg->frame_reg;
1405                                         if (cfg->arch.omit_fp) {
1406                                                 cfg->vret_addr->inst_offset = offset;
1407                                                 offset += 8;
1408                                         } else {
1409                                                 offset += 8;
1410                                                 cfg->vret_addr->inst_offset = -offset;
1411                                         }
1412                                         if (G_UNLIKELY (cfg->verbose_level > 1)) {
1413                                                 printf ("vret_addr =");
1414                                                 mono_print_ins (cfg->vret_addr);
1415                                         }
1416                                 }
1417                         }
1418                         else {
1419                                 cfg->ret->opcode = OP_REGVAR;
1420                                 cfg->ret->inst_c0 = cinfo->ret.reg;
1421                         }
1422                         break;
1423                 case ArgValuetypeInReg:
1424                         /* Allocate a local to hold the result, the epilog will copy it to the correct place */
1425                         cfg->ret->opcode = OP_REGOFFSET;
1426                         cfg->ret->inst_basereg = cfg->frame_reg;
1427                         if (cfg->arch.omit_fp) {
1428                                 cfg->ret->inst_offset = offset;
1429                                 offset += 16;
1430                         } else {
1431                                 offset += 16;
1432                                 cfg->ret->inst_offset = - offset;
1433                         }
1434                         break;
1435                 default:
1436                         g_assert_not_reached ();
1437                 }
1438                 if (!cfg->globalra)
1439                         cfg->ret->dreg = cfg->ret->inst_c0;
1440         }
1441
1442         /* Allocate locals */
1443         if (!cfg->globalra) {
1444                 offsets = mono_allocate_stack_slots_full (cfg, cfg->arch.omit_fp ? FALSE: TRUE, &locals_stack_size, &locals_stack_align);
1445                 if (locals_stack_size > MONO_ARCH_MAX_FRAME_SIZE) {
1446                         char *mname = mono_method_full_name (cfg->method, TRUE);
1447                         cfg->exception_type = MONO_EXCEPTION_INVALID_PROGRAM;
1448                         cfg->exception_message = g_strdup_printf ("Method %s stack is too big.", mname);
1449                         g_free (mname);
1450                         return;
1451                 }
1452
1453                 if (locals_stack_align) {
1454                         offset += (locals_stack_align - 1);
1455                         offset &= ~(locals_stack_align - 1);
1456                 }
1457                 if (cfg->arch.omit_fp) {
1458                         cfg->locals_min_stack_offset = offset;
1459                         cfg->locals_max_stack_offset = offset + locals_stack_size;
1460                 } else {
1461                         cfg->locals_min_stack_offset = - (offset + locals_stack_size);
1462                         cfg->locals_max_stack_offset = - offset;
1463                 }
1464
1465                 for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
1466                         if (offsets [i] != -1) {
1467                                 MonoInst *ins = cfg->varinfo [i];
1468                                 ins->opcode = OP_REGOFFSET;
1469                                 ins->inst_basereg = cfg->frame_reg;
1470                                 if (cfg->arch.omit_fp)
1471                                         ins->inst_offset = (offset + offsets [i]);
1472                                 else
1473                                         ins->inst_offset = - (offset + offsets [i]);
1474                                 //printf ("allocated local %d to ", i); mono_print_tree_nl (ins);
1475                         }
1476                 }
1477                 offset += locals_stack_size;
1478         }
1479
1480         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG)) {
1481                 g_assert (!cfg->arch.omit_fp);
1482                 g_assert (cinfo->sig_cookie.storage == ArgOnStack);
1483                 cfg->sig_cookie = cinfo->sig_cookie.offset + ARGS_OFFSET;
1484         }
1485
1486         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
1487                 ins = cfg->args [i];
1488                 if (ins->opcode != OP_REGVAR) {
1489                         ArgInfo *ainfo = &cinfo->args [i];
1490                         gboolean inreg = TRUE;
1491                         MonoType *arg_type;
1492
1493                         if (sig->hasthis && (i == 0))
1494                                 arg_type = &mono_defaults.object_class->byval_arg;
1495                         else
1496                                 arg_type = sig->params [i - sig->hasthis];
1497
1498                         if (cfg->globalra) {
1499                                 /* The new allocator needs info about the original locations of the arguments */
1500                                 switch (ainfo->storage) {
1501                                 case ArgInIReg:
1502                                 case ArgInFloatSSEReg:
1503                                 case ArgInDoubleSSEReg:
1504                                         ins->opcode = OP_REGVAR;
1505                                         ins->inst_c0 = ainfo->reg;
1506                                         break;
1507                                 case ArgOnStack:
1508                                         g_assert (!cfg->arch.omit_fp);
1509                                         ins->opcode = OP_REGOFFSET;
1510                                         ins->inst_basereg = cfg->frame_reg;
1511                                         ins->inst_offset = ainfo->offset + ARGS_OFFSET;
1512                                         break;
1513                                 case ArgValuetypeInReg:
1514                                         ins->opcode = OP_REGOFFSET;
1515                                         ins->inst_basereg = cfg->frame_reg;
1516                                         /* These arguments are saved to the stack in the prolog */
1517                                         offset = ALIGN_TO (offset, sizeof (gpointer));
1518                                         if (cfg->arch.omit_fp) {
1519                                                 ins->inst_offset = offset;
1520                                                 offset += (ainfo->storage == ArgValuetypeInReg) ? 2 * sizeof (gpointer) : sizeof (gpointer);
1521                                         } else {
1522                                                 offset += (ainfo->storage == ArgValuetypeInReg) ? 2 * sizeof (gpointer) : sizeof (gpointer);
1523                                                 ins->inst_offset = - offset;
1524                                         }
1525                                         break;
1526                                 default:
1527                                         g_assert_not_reached ();
1528                                 }
1529
1530                                 continue;
1531                         }
1532
1533                         /* FIXME: Allocate volatile arguments to registers */
1534                         if (ins->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT))
1535                                 inreg = FALSE;
1536
1537                         /*
1538                          * Under AMD64, all registers used to pass arguments to functions
1539                          * are volatile across calls.
1540                          * FIXME: Optimize this.
1541                          */
1542                         if ((ainfo->storage == ArgInIReg) || (ainfo->storage == ArgInFloatSSEReg) || (ainfo->storage == ArgInDoubleSSEReg) || (ainfo->storage == ArgValuetypeInReg))
1543                                 inreg = FALSE;
1544
1545                         ins->opcode = OP_REGOFFSET;
1546
1547                         switch (ainfo->storage) {
1548                         case ArgInIReg:
1549                         case ArgInFloatSSEReg:
1550                         case ArgInDoubleSSEReg:
1551                                 if (inreg) {
1552                                         ins->opcode = OP_REGVAR;
1553                                         ins->dreg = ainfo->reg;
1554                                 }
1555                                 break;
1556                         case ArgOnStack:
1557                                 g_assert (!cfg->arch.omit_fp);
1558                                 ins->opcode = OP_REGOFFSET;
1559                                 ins->inst_basereg = cfg->frame_reg;
1560                                 ins->inst_offset = ainfo->offset + ARGS_OFFSET;
1561                                 break;
1562                         case ArgValuetypeInReg:
1563                                 break;
1564                         case ArgValuetypeAddrInIReg: {
1565                                 MonoInst *indir;
1566                                 g_assert (!cfg->arch.omit_fp);
1567
1568                                 MONO_INST_NEW (cfg, indir, 0);
1569                                 indir->opcode = OP_REGOFFSET;
1570                                 if (ainfo->pair_storage [0] == ArgInIReg) {
1571                                         indir->inst_basereg = cfg->frame_reg;
1572                                         offset = ALIGN_TO (offset, sizeof (gpointer));
1573                                         offset += (sizeof (gpointer));
1574                                         indir->inst_offset = - offset;
1575                                 }
1576                                 else {
1577                                         indir->inst_basereg = cfg->frame_reg;
1578                                         indir->inst_offset = ainfo->offset + ARGS_OFFSET;
1579                                 }
1580
1581                                 ins->opcode = OP_VTARG_ADDR;
1582                                 ins->inst_left = indir;
1583
1584                                 break;
1585                         }
1586                         default:
1587                                 NOT_IMPLEMENTED;
1588                         }
1589
1590                         if (!inreg && (ainfo->storage != ArgOnStack) && (ainfo->storage != ArgValuetypeAddrInIReg)) {
1591                                 ins->opcode = OP_REGOFFSET;
1592                                 ins->inst_basereg = cfg->frame_reg;
1593                                 /* These arguments are saved to the stack in the prolog */
1594                                 offset = ALIGN_TO (offset, sizeof (gpointer));
1595                                 if (cfg->arch.omit_fp) {
1596                                         ins->inst_offset = offset;
1597                                         offset += (ainfo->storage == ArgValuetypeInReg) ? 2 * sizeof (gpointer) : sizeof (gpointer);
1598                                         // Arguments are yet supported by the stack map creation code
1599                                         //cfg->locals_max_stack_offset = MAX (cfg->locals_max_stack_offset, offset);
1600                                 } else {
1601                                         offset += (ainfo->storage == ArgValuetypeInReg) ? 2 * sizeof (gpointer) : sizeof (gpointer);
1602                                         ins->inst_offset = - offset;
1603                                         //cfg->locals_min_stack_offset = MIN (cfg->locals_min_stack_offset, offset);
1604                                 }
1605                         }
1606                 }
1607         }
1608
1609         cfg->stack_offset = offset;
1610 }
1611
1612 void
1613 mono_arch_create_vars (MonoCompile *cfg)
1614 {
1615         MonoMethodSignature *sig;
1616         CallInfo *cinfo;
1617
1618         sig = mono_method_signature (cfg->method);
1619
1620         if (!cfg->arch.cinfo)
1621                 cfg->arch.cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig, FALSE);
1622         cinfo = cfg->arch.cinfo;
1623
1624         if (cinfo->ret.storage == ArgValuetypeInReg)
1625                 cfg->ret_var_is_local = TRUE;
1626
1627         if ((cinfo->ret.storage != ArgValuetypeInReg) && MONO_TYPE_ISSTRUCT (sig->ret)) {
1628                 cfg->vret_addr = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_ARG);
1629                 if (G_UNLIKELY (cfg->verbose_level > 1)) {
1630                         printf ("vret_addr = ");
1631                         mono_print_ins (cfg->vret_addr);
1632                 }
1633         }
1634
1635         if (cfg->gen_seq_points) {
1636                 MonoInst *ins;
1637
1638             ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1639                 ins->flags |= MONO_INST_VOLATILE;
1640                 cfg->arch.ss_trigger_page_var = ins;
1641         }
1642
1643 #ifdef MONO_AMD64_NO_PUSHES
1644         /*
1645          * When this is set, we pass arguments on the stack by moves, and by allocating
1646          * a bigger stack frame, instead of pushes.
1647          * Pushes complicate exception handling because the arguments on the stack have
1648          * to be popped each time a frame is unwound. They also make fp elimination
1649          * impossible.
1650          * FIXME: This doesn't work inside filter/finally clauses, since those execute
1651          * on a new frame which doesn't include a param area.
1652          */
1653         cfg->arch.no_pushes = TRUE;
1654 #endif
1655 }
1656
1657 static void
1658 add_outarg_reg (MonoCompile *cfg, MonoCallInst *call, ArgStorage storage, int reg, MonoInst *tree)
1659 {
1660         MonoInst *ins;
1661
1662         switch (storage) {
1663         case ArgInIReg:
1664                 MONO_INST_NEW (cfg, ins, OP_MOVE);
1665                 ins->dreg = mono_alloc_ireg (cfg);
1666                 ins->sreg1 = tree->dreg;
1667                 MONO_ADD_INS (cfg->cbb, ins);
1668                 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, FALSE);
1669                 break;
1670         case ArgInFloatSSEReg:
1671                 MONO_INST_NEW (cfg, ins, OP_AMD64_SET_XMMREG_R4);
1672                 ins->dreg = mono_alloc_freg (cfg);
1673                 ins->sreg1 = tree->dreg;
1674                 MONO_ADD_INS (cfg->cbb, ins);
1675
1676                 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
1677                 break;
1678         case ArgInDoubleSSEReg:
1679                 MONO_INST_NEW (cfg, ins, OP_FMOVE);
1680                 ins->dreg = mono_alloc_freg (cfg);
1681                 ins->sreg1 = tree->dreg;
1682                 MONO_ADD_INS (cfg->cbb, ins);
1683
1684                 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
1685
1686                 break;
1687         default:
1688                 g_assert_not_reached ();
1689         }
1690 }
1691
1692 static int
1693 arg_storage_to_load_membase (ArgStorage storage)
1694 {
1695         switch (storage) {
1696         case ArgInIReg:
1697                 return OP_LOAD_MEMBASE;
1698         case ArgInDoubleSSEReg:
1699                 return OP_LOADR8_MEMBASE;
1700         case ArgInFloatSSEReg:
1701                 return OP_LOADR4_MEMBASE;
1702         default:
1703                 g_assert_not_reached ();
1704         }
1705
1706         return -1;
1707 }
1708
1709 static void
1710 emit_sig_cookie (MonoCompile *cfg, MonoCallInst *call, CallInfo *cinfo)
1711 {
1712         MonoInst *arg;
1713         MonoMethodSignature *tmp_sig;
1714         MonoInst *sig_arg;
1715
1716         if (call->tail_call)
1717                 NOT_IMPLEMENTED;
1718
1719         /* FIXME: Add support for signature tokens to AOT */
1720         cfg->disable_aot = TRUE;
1721
1722         g_assert (cinfo->sig_cookie.storage == ArgOnStack);
1723
1724         /*
1725          * mono_ArgIterator_Setup assumes the signature cookie is
1726          * passed first and all the arguments which were before it are
1727          * passed on the stack after the signature. So compensate by
1728          * passing a different signature.
1729          */
1730         tmp_sig = mono_metadata_signature_dup (call->signature);
1731         tmp_sig->param_count -= call->signature->sentinelpos;
1732         tmp_sig->sentinelpos = 0;
1733         memcpy (tmp_sig->params, call->signature->params + call->signature->sentinelpos, tmp_sig->param_count * sizeof (MonoType*));
1734
1735         MONO_INST_NEW (cfg, sig_arg, OP_ICONST);
1736         sig_arg->dreg = mono_alloc_ireg (cfg);
1737         sig_arg->inst_p0 = tmp_sig;
1738         MONO_ADD_INS (cfg->cbb, sig_arg);
1739
1740         if (cfg->arch.no_pushes) {
1741                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, AMD64_RSP, cinfo->sig_cookie.offset, sig_arg->dreg);
1742         } else {
1743                 MONO_INST_NEW (cfg, arg, OP_X86_PUSH);
1744                 arg->sreg1 = sig_arg->dreg;
1745                 MONO_ADD_INS (cfg->cbb, arg);
1746         }
1747 }
1748
1749 static inline LLVMArgStorage
1750 arg_storage_to_llvm_arg_storage (MonoCompile *cfg, ArgStorage storage)
1751 {
1752         switch (storage) {
1753         case ArgInIReg:
1754                 return LLVMArgInIReg;
1755         case ArgNone:
1756                 return LLVMArgNone;
1757         default:
1758                 g_assert_not_reached ();
1759                 return LLVMArgNone;
1760         }
1761 }
1762
1763 #ifdef ENABLE_LLVM
1764 LLVMCallInfo*
1765 mono_arch_get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
1766 {
1767         int i, n;
1768         CallInfo *cinfo;
1769         ArgInfo *ainfo;
1770         int j;
1771         LLVMCallInfo *linfo;
1772
1773         n = sig->param_count + sig->hasthis;
1774
1775         cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig, sig->pinvoke);
1776
1777         linfo = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
1778
1779         /*
1780          * LLVM always uses the native ABI while we use our own ABI, the
1781          * only difference is the handling of vtypes:
1782          * - we only pass/receive them in registers in some cases, and only
1783          *   in 1 or 2 integer registers.
1784          */
1785         if (cinfo->ret.storage == ArgValuetypeInReg) {
1786                 if (sig->pinvoke) {
1787                         cfg->exception_message = g_strdup ("pinvoke + vtypes");
1788                         cfg->disable_llvm = TRUE;
1789                         return linfo;
1790                 }
1791
1792                 linfo->ret.storage = LLVMArgVtypeInReg;
1793                 for (j = 0; j < 2; ++j)
1794                         linfo->ret.pair_storage [j] = arg_storage_to_llvm_arg_storage (cfg, cinfo->ret.pair_storage [j]);
1795         }
1796
1797         if (MONO_TYPE_ISSTRUCT (sig->ret) && cinfo->ret.storage == ArgInIReg) {
1798                 /* Vtype returned using a hidden argument */
1799                 linfo->ret.storage = LLVMArgVtypeRetAddr;
1800         }
1801
1802         for (i = 0; i < n; ++i) {
1803                 ainfo = cinfo->args + i;
1804
1805                 linfo->args [i].storage = LLVMArgNone;
1806
1807                 switch (ainfo->storage) {
1808                 case ArgInIReg:
1809                         linfo->args [i].storage = LLVMArgInIReg;
1810                         break;
1811                 case ArgInDoubleSSEReg:
1812                 case ArgInFloatSSEReg:
1813                         linfo->args [i].storage = LLVMArgInFPReg;
1814                         break;
1815                 case ArgOnStack:
1816                         if ((i >= sig->hasthis) && (MONO_TYPE_ISSTRUCT(sig->params [i - sig->hasthis]))) {
1817                                 linfo->args [i].storage = LLVMArgVtypeByVal;
1818                         } else {
1819                                 linfo->args [i].storage = LLVMArgInIReg;
1820                                 if (!sig->params [i - sig->hasthis]->byref) {
1821                                         if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R4) {
1822                                                 linfo->args [i].storage = LLVMArgInFPReg;
1823                                         } else if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R8) {
1824                                                 linfo->args [i].storage = LLVMArgInFPReg;
1825                                         }
1826                                 }
1827                         }
1828                         break;
1829                 case ArgValuetypeInReg:
1830                         if (sig->pinvoke) {
1831                                 cfg->exception_message = g_strdup ("pinvoke + vtypes");
1832                                 cfg->disable_llvm = TRUE;
1833                                 return linfo;
1834                         }
1835
1836                         linfo->args [i].storage = LLVMArgVtypeInReg;
1837                         for (j = 0; j < 2; ++j)
1838                                 linfo->args [i].pair_storage [j] = arg_storage_to_llvm_arg_storage (cfg, ainfo->pair_storage [j]);
1839                         break;
1840                 default:
1841                         cfg->exception_message = g_strdup ("ainfo->storage");
1842                         cfg->disable_llvm = TRUE;
1843                         break;
1844                 }
1845         }
1846
1847         return linfo;
1848 }
1849 #endif
1850
1851 void
1852 mono_arch_emit_call (MonoCompile *cfg, MonoCallInst *call)
1853 {
1854         MonoInst *arg, *in;
1855         MonoMethodSignature *sig;
1856         int i, n, stack_size;
1857         CallInfo *cinfo;
1858         ArgInfo *ainfo;
1859
1860         stack_size = 0;
1861
1862         sig = call->signature;
1863         n = sig->param_count + sig->hasthis;
1864
1865         cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig, sig->pinvoke);
1866
1867         if (COMPILE_LLVM (cfg)) {
1868                 /* We shouldn't be called in the llvm case */
1869                 cfg->disable_llvm = TRUE;
1870                 return;
1871         }
1872
1873         if (cinfo->need_stack_align) {
1874                 if (!cfg->arch.no_pushes)
1875                         MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 8);
1876         }
1877
1878         /*
1879          * Emit all arguments which are passed on the stack to prevent register
1880          * allocation problems.
1881          */
1882         if (cfg->arch.no_pushes) {
1883                 for (i = 0; i < n; ++i) {
1884                         MonoType *t;
1885                         ainfo = cinfo->args + i;
1886
1887                         in = call->args [i];
1888
1889                         if (sig->hasthis && i == 0)
1890                                 t = &mono_defaults.object_class->byval_arg;
1891                         else
1892                                 t = sig->params [i - sig->hasthis];
1893
1894                         if (ainfo->storage == ArgOnStack && !MONO_TYPE_ISSTRUCT (t) && !call->tail_call) {
1895                                 if (!t->byref) {
1896                                         if (t->type == MONO_TYPE_R4)
1897                                                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER4_MEMBASE_REG, AMD64_RSP, ainfo->offset, in->dreg);
1898                                         else if (t->type == MONO_TYPE_R8)
1899                                                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER8_MEMBASE_REG, AMD64_RSP, ainfo->offset, in->dreg);
1900                                         else
1901                                                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, AMD64_RSP, ainfo->offset, in->dreg);
1902                                 } else {
1903                                         MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, AMD64_RSP, ainfo->offset, in->dreg);
1904                                 }
1905                         }
1906                 }
1907         }
1908
1909         /*
1910          * Emit all parameters passed in registers in non-reverse order for better readability
1911          * and to help the optimization in emit_prolog ().
1912          */
1913         for (i = 0; i < n; ++i) {
1914                 ainfo = cinfo->args + i;
1915
1916                 in = call->args [i];
1917
1918                 if (ainfo->storage == ArgInIReg)
1919                         add_outarg_reg (cfg, call, ainfo->storage, ainfo->reg, in);
1920         }
1921
1922         for (i = n - 1; i >= 0; --i) {
1923                 ainfo = cinfo->args + i;
1924
1925                 in = call->args [i];
1926
1927                 switch (ainfo->storage) {
1928                 case ArgInIReg:
1929                         /* Already done */
1930                         break;
1931                 case ArgInFloatSSEReg:
1932                 case ArgInDoubleSSEReg:
1933                         add_outarg_reg (cfg, call, ainfo->storage, ainfo->reg, in);
1934                         break;
1935                 case ArgOnStack:
1936                 case ArgValuetypeInReg:
1937                 case ArgValuetypeAddrInIReg:
1938                         if (ainfo->storage == ArgOnStack && call->tail_call) {
1939                                 MonoInst *call_inst = (MonoInst*)call;
1940                                 cfg->args [i]->flags |= MONO_INST_VOLATILE;
1941                                 EMIT_NEW_ARGSTORE (cfg, call_inst, i, in);
1942                         } else if ((i >= sig->hasthis) && (MONO_TYPE_ISSTRUCT(sig->params [i - sig->hasthis]))) {
1943                                 guint32 align;
1944                                 guint32 size;
1945
1946                                 if (sig->params [i - sig->hasthis]->type == MONO_TYPE_TYPEDBYREF) {
1947                                         size = sizeof (MonoTypedRef);
1948                                         align = sizeof (gpointer);
1949                                 }
1950                                 else {
1951                                         if (sig->pinvoke)
1952                                                 size = mono_type_native_stack_size (&in->klass->byval_arg, &align);
1953                                         else {
1954                                                 /*
1955                                                  * Other backends use mono_type_stack_size (), but that
1956                                                  * aligns the size to 8, which is larger than the size of
1957                                                  * the source, leading to reads of invalid memory if the
1958                                                  * source is at the end of address space.
1959                                                  */
1960                                                 size = mono_class_value_size (in->klass, &align);
1961                                         }
1962                                 }
1963                                 g_assert (in->klass);
1964
1965                                 if (size > 0) {
1966                                         MONO_INST_NEW (cfg, arg, OP_OUTARG_VT);
1967                                         arg->sreg1 = in->dreg;
1968                                         arg->klass = in->klass;
1969                                         arg->backend.size = size;
1970                                         arg->inst_p0 = call;
1971                                         arg->inst_p1 = mono_mempool_alloc (cfg->mempool, sizeof (ArgInfo));
1972                                         memcpy (arg->inst_p1, ainfo, sizeof (ArgInfo));
1973
1974                                         MONO_ADD_INS (cfg->cbb, arg);
1975                                 }
1976                         } else {
1977                                 if (cfg->arch.no_pushes) {
1978                                         /* Already done */
1979                                 } else {
1980                                         MONO_INST_NEW (cfg, arg, OP_X86_PUSH);
1981                                         arg->sreg1 = in->dreg;
1982                                         if (!sig->params [i - sig->hasthis]->byref) {
1983                                                 if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R4) {
1984                                                         MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 8);
1985                                                         arg->opcode = OP_STORER4_MEMBASE_REG;
1986                                                         arg->inst_destbasereg = X86_ESP;
1987                                                         arg->inst_offset = 0;
1988                                                 } else if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R8) {
1989                                                         MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 8);
1990                                                         arg->opcode = OP_STORER8_MEMBASE_REG;
1991                                                         arg->inst_destbasereg = X86_ESP;
1992                                                         arg->inst_offset = 0;
1993                                                 }
1994                                         }
1995                                         MONO_ADD_INS (cfg->cbb, arg);
1996                                 }
1997                         }
1998                         break;
1999                 default:
2000                         g_assert_not_reached ();
2001                 }
2002
2003                 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos))
2004                         /* Emit the signature cookie just before the implicit arguments */
2005                         emit_sig_cookie (cfg, call, cinfo);
2006         }
2007
2008         /* Handle the case where there are no implicit arguments */
2009         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n == sig->sentinelpos))
2010                 emit_sig_cookie (cfg, call, cinfo);
2011
2012         if (sig->ret && MONO_TYPE_ISSTRUCT (sig->ret)) {
2013                 MonoInst *vtarg;
2014
2015                 if (cinfo->ret.storage == ArgValuetypeInReg) {
2016                         if (cinfo->ret.pair_storage [0] == ArgInIReg && cinfo->ret.pair_storage [1] == ArgNone) {
2017                                 /*
2018                                  * Tell the JIT to use a more efficient calling convention: call using
2019                                  * OP_CALL, compute the result location after the call, and save the
2020                                  * result there.
2021                                  */
2022                                 call->vret_in_reg = TRUE;
2023                                 /*
2024                                  * Nullify the instruction computing the vret addr to enable
2025                                  * future optimizations.
2026                                  */
2027                                 if (call->vret_var)
2028                                         NULLIFY_INS (call->vret_var);
2029                         } else {
2030                                 if (call->tail_call)
2031                                         NOT_IMPLEMENTED;
2032                                 /*
2033                                  * The valuetype is in RAX:RDX after the call, need to be copied to
2034                                  * the stack. Push the address here, so the call instruction can
2035                                  * access it.
2036                                  */
2037                                 if (!cfg->arch.vret_addr_loc) {
2038                                         cfg->arch.vret_addr_loc = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
2039                                         /* Prevent it from being register allocated or optimized away */
2040                                         ((MonoInst*)cfg->arch.vret_addr_loc)->flags |= MONO_INST_VOLATILE;
2041                                 }
2042
2043                                 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, ((MonoInst*)cfg->arch.vret_addr_loc)->dreg, call->vret_var->dreg);
2044                         }
2045                 }
2046                 else {
2047                         MONO_INST_NEW (cfg, vtarg, OP_MOVE);
2048                         vtarg->sreg1 = call->vret_var->dreg;
2049                         vtarg->dreg = mono_alloc_preg (cfg);
2050                         MONO_ADD_INS (cfg->cbb, vtarg);
2051
2052                         mono_call_inst_add_outarg_reg (cfg, call, vtarg->dreg, cinfo->ret.reg, FALSE);
2053                 }
2054         }
2055
2056 #ifdef HOST_WIN32
2057         if (call->inst.opcode != OP_JMP && OP_TAILCALL != call->inst.opcode) {
2058                 MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 0x20);
2059         }
2060 #endif
2061
2062         if (cfg->method->save_lmf) {
2063                 MONO_INST_NEW (cfg, arg, OP_AMD64_SAVE_SP_TO_LMF);
2064                 MONO_ADD_INS (cfg->cbb, arg);
2065         }
2066
2067         call->stack_usage = cinfo->stack_usage;
2068 }
2069
2070 void
2071 mono_arch_emit_outarg_vt (MonoCompile *cfg, MonoInst *ins, MonoInst *src)
2072 {
2073         MonoInst *arg;
2074         MonoCallInst *call = (MonoCallInst*)ins->inst_p0;
2075         ArgInfo *ainfo = (ArgInfo*)ins->inst_p1;
2076         int size = ins->backend.size;
2077
2078         if (ainfo->storage == ArgValuetypeInReg) {
2079                 MonoInst *load;
2080                 int part;
2081
2082                 for (part = 0; part < 2; ++part) {
2083                         if (ainfo->pair_storage [part] == ArgNone)
2084                                 continue;
2085
2086                         MONO_INST_NEW (cfg, load, arg_storage_to_load_membase (ainfo->pair_storage [part]));
2087                         load->inst_basereg = src->dreg;
2088                         load->inst_offset = part * sizeof (gpointer);
2089
2090                         switch (ainfo->pair_storage [part]) {
2091                         case ArgInIReg:
2092                                 load->dreg = mono_alloc_ireg (cfg);
2093                                 break;
2094                         case ArgInDoubleSSEReg:
2095                         case ArgInFloatSSEReg:
2096                                 load->dreg = mono_alloc_freg (cfg);
2097                                 break;
2098                         default:
2099                                 g_assert_not_reached ();
2100                         }
2101                         MONO_ADD_INS (cfg->cbb, load);
2102
2103                         add_outarg_reg (cfg, call, ainfo->pair_storage [part], ainfo->pair_regs [part], load);
2104                 }
2105         } else if (ainfo->storage == ArgValuetypeAddrInIReg) {
2106                 MonoInst *vtaddr, *load;
2107                 vtaddr = mono_compile_create_var (cfg, &ins->klass->byval_arg, OP_LOCAL);
2108
2109                 g_assert (!cfg->arch.no_pushes);
2110
2111                 MONO_INST_NEW (cfg, load, OP_LDADDR);
2112                 load->inst_p0 = vtaddr;
2113                 vtaddr->flags |= MONO_INST_INDIRECT;
2114                 load->type = STACK_MP;
2115                 load->klass = vtaddr->klass;
2116                 load->dreg = mono_alloc_ireg (cfg);
2117                 MONO_ADD_INS (cfg->cbb, load);
2118                 mini_emit_memcpy (cfg, load->dreg, 0, src->dreg, 0, size, 4);
2119
2120                 if (ainfo->pair_storage [0] == ArgInIReg) {
2121                         MONO_INST_NEW (cfg, arg, OP_X86_LEA_MEMBASE);
2122                         arg->dreg = mono_alloc_ireg (cfg);
2123                         arg->sreg1 = load->dreg;
2124                         arg->inst_imm = 0;
2125                         MONO_ADD_INS (cfg->cbb, arg);
2126                         mono_call_inst_add_outarg_reg (cfg, call, arg->dreg, ainfo->pair_regs [0], FALSE);
2127                 } else {
2128                         MONO_INST_NEW (cfg, arg, OP_X86_PUSH);
2129                         arg->sreg1 = load->dreg;
2130                         MONO_ADD_INS (cfg->cbb, arg);
2131                 }
2132         } else {
2133                 if (size == 8) {
2134                         if (cfg->arch.no_pushes) {
2135                                 int dreg = mono_alloc_ireg (cfg);
2136
2137                                 MONO_EMIT_NEW_LOAD_MEMBASE (cfg, dreg, src->dreg, 0);
2138                                 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, AMD64_RSP, ainfo->offset, dreg);
2139                         } else {
2140                                 /* Can't use this for < 8 since it does an 8 byte memory load */
2141                                 MONO_INST_NEW (cfg, arg, OP_X86_PUSH_MEMBASE);
2142                                 arg->inst_basereg = src->dreg;
2143                                 arg->inst_offset = 0;
2144                                 MONO_ADD_INS (cfg->cbb, arg);
2145                         }
2146                 } else if (size <= 40) {
2147                         if (cfg->arch.no_pushes) {
2148                                 mini_emit_memcpy (cfg, AMD64_RSP, ainfo->offset, src->dreg, 0, size, 4);
2149                         } else {
2150                                 MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, ALIGN_TO (size, 8));
2151                                 mini_emit_memcpy (cfg, X86_ESP, 0, src->dreg, 0, size, 4);
2152                         }
2153                 } else {
2154                         if (cfg->arch.no_pushes) {
2155                                 // FIXME: Code growth
2156                                 mini_emit_memcpy (cfg, AMD64_RSP, ainfo->offset, src->dreg, 0, size, 4);
2157                         } else {
2158                                 MONO_INST_NEW (cfg, arg, OP_X86_PUSH_OBJ);
2159                                 arg->inst_basereg = src->dreg;
2160                                 arg->inst_offset = 0;
2161                                 arg->inst_imm = size;
2162                                 MONO_ADD_INS (cfg->cbb, arg);
2163                         }
2164                 }
2165         }
2166 }
2167
2168 void
2169 mono_arch_emit_setret (MonoCompile *cfg, MonoMethod *method, MonoInst *val)
2170 {
2171         MonoType *ret = mini_type_get_underlying_type (NULL, mono_method_signature (method)->ret);
2172
2173         if (ret->type == MONO_TYPE_R4) {
2174                 if (COMPILE_LLVM (cfg))
2175                         MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2176                 else
2177                         MONO_EMIT_NEW_UNALU (cfg, OP_AMD64_SET_XMMREG_R4, cfg->ret->dreg, val->dreg);
2178                 return;
2179         } else if (ret->type == MONO_TYPE_R8) {
2180                 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2181                 return;
2182         }
2183
2184         MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, cfg->ret->dreg, val->dreg);
2185 }
2186
2187 #define EMIT_COND_BRANCH(ins,cond,sign) \
2188         if (ins->inst_true_bb->native_offset) { \
2189                 x86_branch (code, cond, cfg->native_code + ins->inst_true_bb->native_offset, sign); \
2190         } else { \
2191                 mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_BB, ins->inst_true_bb); \
2192                 if ((cfg->opt & MONO_OPT_BRANCH) && \
2193             x86_is_imm8 (ins->inst_true_bb->max_offset - offset)) \
2194                         x86_branch8 (code, cond, 0, sign); \
2195                 else \
2196                         x86_branch32 (code, cond, 0, sign); \
2197 }
2198
2199 typedef struct {
2200         MonoMethodSignature *sig;
2201         CallInfo *cinfo;
2202 } ArchDynCallInfo;
2203
2204 typedef struct {
2205         mgreg_t regs [PARAM_REGS];
2206         mgreg_t res;
2207         guint8 *ret;
2208 } DynCallArgs;
2209
2210 static gboolean
2211 dyn_call_supported (MonoMethodSignature *sig, CallInfo *cinfo)
2212 {
2213         int i;
2214
2215 #ifdef HOST_WIN32
2216         return FALSE;
2217 #endif
2218
2219         switch (cinfo->ret.storage) {
2220         case ArgNone:
2221         case ArgInIReg:
2222                 break;
2223         case ArgValuetypeInReg: {
2224                 ArgInfo *ainfo = &cinfo->ret;
2225
2226                 if (ainfo->pair_storage [0] != ArgNone && ainfo->pair_storage [0] != ArgInIReg)
2227                         return FALSE;
2228                 if (ainfo->pair_storage [1] != ArgNone && ainfo->pair_storage [1] != ArgInIReg)
2229                         return FALSE;
2230                 break;
2231         }
2232         default:
2233                 return FALSE;
2234         }
2235
2236         for (i = 0; i < cinfo->nargs; ++i) {
2237                 ArgInfo *ainfo = &cinfo->args [i];
2238                 switch (ainfo->storage) {
2239                 case ArgInIReg:
2240                         break;
2241                 case ArgValuetypeInReg:
2242                         if (ainfo->pair_storage [0] != ArgNone && ainfo->pair_storage [0] != ArgInIReg)
2243                                 return FALSE;
2244                         if (ainfo->pair_storage [1] != ArgNone && ainfo->pair_storage [1] != ArgInIReg)
2245                                 return FALSE;
2246                         break;
2247                 default:
2248                         return FALSE;
2249                 }
2250         }
2251
2252         return TRUE;
2253 }
2254
2255 /*
2256  * mono_arch_dyn_call_prepare:
2257  *
2258  *   Return a pointer to an arch-specific structure which contains information
2259  * needed by mono_arch_get_dyn_call_args (). Return NULL if OP_DYN_CALL is not
2260  * supported for SIG.
2261  * This function is equivalent to ffi_prep_cif in libffi.
2262  */
2263 MonoDynCallInfo*
2264 mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
2265 {
2266         ArchDynCallInfo *info;
2267         CallInfo *cinfo;
2268
2269         cinfo = get_call_info (NULL, NULL, sig, FALSE);
2270
2271         if (!dyn_call_supported (sig, cinfo)) {
2272                 g_free (cinfo);
2273                 return NULL;
2274         }
2275
2276         info = g_new0 (ArchDynCallInfo, 1);
2277         // FIXME: Preprocess the info to speed up get_dyn_call_args ().
2278         info->sig = sig;
2279         info->cinfo = cinfo;
2280
2281         return (MonoDynCallInfo*)info;
2282 }
2283
2284 /*
2285  * mono_arch_dyn_call_free:
2286  *
2287  *   Free a MonoDynCallInfo structure.
2288  */
2289 void
2290 mono_arch_dyn_call_free (MonoDynCallInfo *info)
2291 {
2292         ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
2293
2294         g_free (ainfo->cinfo);
2295         g_free (ainfo);
2296 }
2297
2298 /*
2299  * mono_arch_get_start_dyn_call:
2300  *
2301  *   Convert the arguments ARGS to a format which can be passed to OP_DYN_CALL, and
2302  * store the result into BUF.
2303  * ARGS should be an array of pointers pointing to the arguments.
2304  * RET should point to a memory buffer large enought to hold the result of the
2305  * call.
2306  * This function should be as fast as possible, any work which does not depend
2307  * on the actual values of the arguments should be done in
2308  * mono_arch_dyn_call_prepare ().
2309  * start_dyn_call + OP_DYN_CALL + finish_dyn_call is equivalent to ffi_call in
2310  * libffi.
2311  */
2312 void
2313 mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, guint8 *buf, int buf_len)
2314 {
2315         ArchDynCallInfo *dinfo = (ArchDynCallInfo*)info;
2316         DynCallArgs *p = (DynCallArgs*)buf;
2317         int arg_index, greg, i;
2318         MonoMethodSignature *sig = dinfo->sig;
2319
2320         g_assert (buf_len >= sizeof (DynCallArgs));
2321
2322         p->res = 0;
2323         p->ret = ret;
2324
2325         arg_index = 0;
2326         greg = 0;
2327
2328         if (dinfo->cinfo->vtype_retaddr)
2329                 p->regs [greg ++] = (mgreg_t)ret;
2330
2331         if (sig->hasthis) {
2332                 p->regs [greg ++] = (mgreg_t)*(args [arg_index ++]);
2333         }
2334
2335         for (i = 0; i < sig->param_count; i++) {
2336                 MonoType *t = mono_type_get_underlying_type (sig->params [i]);
2337                 gpointer *arg = args [arg_index ++];
2338
2339                 if (t->byref) {
2340                         p->regs [greg ++] = (mgreg_t)*(arg);
2341                         continue;
2342                 }
2343
2344                 switch (t->type) {
2345                 case MONO_TYPE_STRING:
2346                 case MONO_TYPE_CLASS:
2347                 case MONO_TYPE_ARRAY:
2348                 case MONO_TYPE_SZARRAY:
2349                 case MONO_TYPE_OBJECT:
2350                 case MONO_TYPE_PTR:
2351                 case MONO_TYPE_I:
2352                 case MONO_TYPE_U:
2353                 case MONO_TYPE_I8:
2354                 case MONO_TYPE_U8:
2355                         g_assert (dinfo->cinfo->args [i + sig->hasthis].reg == param_regs [greg]);
2356                         p->regs [greg ++] = (mgreg_t)*(arg);
2357                         break;
2358                 case MONO_TYPE_BOOLEAN:
2359                 case MONO_TYPE_U1:
2360                         p->regs [greg ++] = *(guint8*)(arg);
2361                         break;
2362                 case MONO_TYPE_I1:
2363                         p->regs [greg ++] = *(gint8*)(arg);
2364                         break;
2365                 case MONO_TYPE_I2:
2366                         p->regs [greg ++] = *(gint16*)(arg);
2367                         break;
2368                 case MONO_TYPE_U2:
2369                 case MONO_TYPE_CHAR:
2370                         p->regs [greg ++] = *(guint16*)(arg);
2371                         break;
2372                 case MONO_TYPE_I4:
2373                         p->regs [greg ++] = *(gint32*)(arg);
2374                         break;
2375                 case MONO_TYPE_U4:
2376                         p->regs [greg ++] = *(guint32*)(arg);
2377                         break;
2378                 case MONO_TYPE_GENERICINST:
2379                     if (MONO_TYPE_IS_REFERENCE (t)) {
2380                                 p->regs [greg ++] = (mgreg_t)*(arg);
2381                                 break;
2382                         } else {
2383                                 /* Fall through */
2384                         }
2385                 case MONO_TYPE_VALUETYPE: {
2386                         ArgInfo *ainfo = &dinfo->cinfo->args [i + sig->hasthis];
2387
2388                         g_assert (ainfo->storage == ArgValuetypeInReg);
2389                         if (ainfo->pair_storage [0] != ArgNone) {
2390                                 g_assert (ainfo->pair_storage [0] == ArgInIReg);
2391                                 p->regs [greg ++] = ((mgreg_t*)(arg))[0];
2392                         }
2393                         if (ainfo->pair_storage [1] != ArgNone) {
2394                                 g_assert (ainfo->pair_storage [1] == ArgInIReg);
2395                                 p->regs [greg ++] = ((mgreg_t*)(arg))[1];
2396                         }
2397                         break;
2398                 }
2399                 default:
2400                         g_assert_not_reached ();
2401                 }
2402         }
2403
2404         g_assert (greg <= PARAM_REGS);
2405 }
2406
2407 /*
2408  * mono_arch_finish_dyn_call:
2409  *
2410  *   Store the result of a dyn call into the return value buffer passed to
2411  * start_dyn_call ().
2412  * This function should be as fast as possible, any work which does not depend
2413  * on the actual values of the arguments should be done in
2414  * mono_arch_dyn_call_prepare ().
2415  */
2416 void
2417 mono_arch_finish_dyn_call (MonoDynCallInfo *info, guint8 *buf)
2418 {
2419         ArchDynCallInfo *dinfo = (ArchDynCallInfo*)info;
2420         MonoMethodSignature *sig = dinfo->sig;
2421         guint8 *ret = ((DynCallArgs*)buf)->ret;
2422         mgreg_t res = ((DynCallArgs*)buf)->res;
2423
2424         switch (mono_type_get_underlying_type (sig->ret)->type) {
2425         case MONO_TYPE_VOID:
2426                 *(gpointer*)ret = NULL;
2427                 break;
2428         case MONO_TYPE_STRING:
2429         case MONO_TYPE_CLASS:
2430         case MONO_TYPE_ARRAY:
2431         case MONO_TYPE_SZARRAY:
2432         case MONO_TYPE_OBJECT:
2433         case MONO_TYPE_I:
2434         case MONO_TYPE_U:
2435         case MONO_TYPE_PTR:
2436                 *(gpointer*)ret = (gpointer)res;
2437                 break;
2438         case MONO_TYPE_I1:
2439                 *(gint8*)ret = res;
2440                 break;
2441         case MONO_TYPE_U1:
2442         case MONO_TYPE_BOOLEAN:
2443                 *(guint8*)ret = res;
2444                 break;
2445         case MONO_TYPE_I2:
2446                 *(gint16*)ret = res;
2447                 break;
2448         case MONO_TYPE_U2:
2449         case MONO_TYPE_CHAR:
2450                 *(guint16*)ret = res;
2451                 break;
2452         case MONO_TYPE_I4:
2453                 *(gint32*)ret = res;
2454                 break;
2455         case MONO_TYPE_U4:
2456                 *(guint32*)ret = res;
2457                 break;
2458         case MONO_TYPE_I8:
2459                 *(gint64*)ret = res;
2460                 break;
2461         case MONO_TYPE_U8:
2462                 *(guint64*)ret = res;
2463                 break;
2464         case MONO_TYPE_GENERICINST:
2465                 if (MONO_TYPE_IS_REFERENCE (sig->ret)) {
2466                         *(gpointer*)ret = (gpointer)res;
2467                         break;
2468                 } else {
2469                         /* Fall through */
2470                 }
2471         case MONO_TYPE_VALUETYPE:
2472                 if (dinfo->cinfo->vtype_retaddr) {
2473                         /* Nothing to do */
2474                 } else {
2475                         ArgInfo *ainfo = &dinfo->cinfo->ret;
2476
2477                         g_assert (ainfo->storage == ArgValuetypeInReg);
2478
2479                         if (ainfo->pair_storage [0] != ArgNone) {
2480                                 g_assert (ainfo->pair_storage [0] == ArgInIReg);
2481                                 ((mgreg_t*)ret)[0] = res;
2482                         }
2483
2484                         g_assert (ainfo->pair_storage [1] == ArgNone);
2485                 }
2486                 break;
2487         default:
2488                 g_assert_not_reached ();
2489         }
2490 }
2491
2492 /* emit an exception if condition is fail */
2493 #define EMIT_COND_SYSTEM_EXCEPTION(cond,signed,exc_name)            \
2494         do {                                                        \
2495                 MonoInst *tins = mono_branch_optimize_exception_target (cfg, bb, exc_name); \
2496                 if (tins == NULL) {                                                                             \
2497                         mono_add_patch_info (cfg, code - cfg->native_code,   \
2498                                         MONO_PATCH_INFO_EXC, exc_name);  \
2499                         x86_branch32 (code, cond, 0, signed);               \
2500                 } else {        \
2501                         EMIT_COND_BRANCH (tins, cond, signed);  \
2502                 }                       \
2503         } while (0);
2504
2505 #define EMIT_FPCOMPARE(code) do { \
2506         amd64_fcompp (code); \
2507         amd64_fnstsw (code); \
2508 } while (0);
2509
2510 #define EMIT_SSE2_FPFUNC(code, op, dreg, sreg1) do { \
2511     amd64_movsd_membase_reg (code, AMD64_RSP, -8, (sreg1)); \
2512         amd64_fld_membase (code, AMD64_RSP, -8, TRUE); \
2513         amd64_ ##op (code); \
2514         amd64_fst_membase (code, AMD64_RSP, -8, TRUE, TRUE); \
2515         amd64_movsd_reg_membase (code, (dreg), AMD64_RSP, -8); \
2516 } while (0);
2517
2518 static guint8*
2519 emit_call_body (MonoCompile *cfg, guint8 *code, guint32 patch_type, gconstpointer data)
2520 {
2521         gboolean no_patch = FALSE;
2522
2523         /*
2524          * FIXME: Add support for thunks
2525          */
2526         {
2527                 gboolean near_call = FALSE;
2528
2529                 /*
2530                  * Indirect calls are expensive so try to make a near call if possible.
2531                  * The caller memory is allocated by the code manager so it is
2532                  * guaranteed to be at a 32 bit offset.
2533                  */
2534
2535                 if (patch_type != MONO_PATCH_INFO_ABS) {
2536                         /* The target is in memory allocated using the code manager */
2537                         near_call = TRUE;
2538
2539                         if ((patch_type == MONO_PATCH_INFO_METHOD) || (patch_type == MONO_PATCH_INFO_METHOD_JUMP)) {
2540                                 if (((MonoMethod*)data)->klass->image->aot_module)
2541                                         /* The callee might be an AOT method */
2542                                         near_call = FALSE;
2543                                 if (((MonoMethod*)data)->dynamic)
2544                                         /* The target is in malloc-ed memory */
2545                                         near_call = FALSE;
2546                         }
2547
2548                         if (patch_type == MONO_PATCH_INFO_INTERNAL_METHOD) {
2549                                 /*
2550                                  * The call might go directly to a native function without
2551                                  * the wrapper.
2552                                  */
2553                                 MonoJitICallInfo *mi = mono_find_jit_icall_by_name (data);
2554                                 if (mi) {
2555                                         gconstpointer target = mono_icall_get_wrapper (mi);
2556                                         if ((((guint64)target) >> 32) != 0)
2557                                                 near_call = FALSE;
2558                                 }
2559                         }
2560                 }
2561                 else {
2562                         if (cfg->abs_patches && g_hash_table_lookup (cfg->abs_patches, data)) {
2563                                 /*
2564                                  * This is not really an optimization, but required because the
2565                                  * generic class init trampolines use R11 to pass the vtable.
2566                                  */
2567                                 near_call = TRUE;
2568                         } else {
2569                                 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (data);
2570                                 if (info) {
2571                                         if ((cfg->method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) &&
2572                                                 strstr (cfg->method->name, info->name)) {
2573                                                 /* A call to the wrapped function */
2574                                                 if ((((guint64)data) >> 32) == 0)
2575                                                         near_call = TRUE;
2576                                                 no_patch = TRUE;
2577                                         }
2578                                         else if (info->func == info->wrapper) {
2579                                                 /* No wrapper */
2580                                                 if ((((guint64)info->func) >> 32) == 0)
2581                                                         near_call = TRUE;
2582                                         }
2583                                         else {
2584                                                 /* See the comment in mono_codegen () */
2585                                                 if ((info->name [0] != 'v') || (strstr (info->name, "ves_array_new_va_") == NULL && strstr (info->name, "ves_array_element_address_") == NULL))
2586                                                         near_call = TRUE;
2587                                         }
2588                                 }
2589                                 else if ((((guint64)data) >> 32) == 0) {
2590                                         near_call = TRUE;
2591                                         no_patch = TRUE;
2592                                 }
2593                         }
2594                 }
2595
2596                 if (cfg->method->dynamic)
2597                         /* These methods are allocated using malloc */
2598                         near_call = FALSE;
2599
2600 #ifdef MONO_ARCH_NOMAP32BIT
2601                 near_call = FALSE;
2602 #endif
2603
2604                 /* The 64bit XEN kernel does not honour the MAP_32BIT flag. (#522894) */
2605                 if (optimize_for_xen)
2606                         near_call = FALSE;
2607
2608                 if (cfg->compile_aot) {
2609                         near_call = TRUE;
2610                         no_patch = TRUE;
2611                 }
2612
2613                 if (near_call) {
2614                         /*
2615                          * Align the call displacement to an address divisible by 4 so it does
2616                          * not span cache lines. This is required for code patching to work on SMP
2617                          * systems.
2618                          */
2619                         if (!no_patch && ((guint32)(code + 1 - cfg->native_code) % 4) != 0)
2620                                 amd64_padding (code, 4 - ((guint32)(code + 1 - cfg->native_code) % 4));
2621                         mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
2622                         amd64_call_code (code, 0);
2623                 }
2624                 else {
2625                         mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
2626                         amd64_set_reg_template (code, GP_SCRATCH_REG);
2627                         amd64_call_reg (code, GP_SCRATCH_REG);
2628                 }
2629         }
2630
2631         return code;
2632 }
2633
2634 static inline guint8*
2635 emit_call (MonoCompile *cfg, guint8 *code, guint32 patch_type, gconstpointer data, gboolean win64_adjust_stack)
2636 {
2637 #ifdef HOST_WIN32
2638         if (win64_adjust_stack)
2639                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 32);
2640 #endif
2641         code = emit_call_body (cfg, code, patch_type, data);
2642 #ifdef HOST_WIN32
2643         if (win64_adjust_stack)
2644                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 32);
2645 #endif
2646
2647         return code;
2648 }
2649
2650 static inline int
2651 store_membase_imm_to_store_membase_reg (int opcode)
2652 {
2653         switch (opcode) {
2654         case OP_STORE_MEMBASE_IMM:
2655                 return OP_STORE_MEMBASE_REG;
2656         case OP_STOREI4_MEMBASE_IMM:
2657                 return OP_STOREI4_MEMBASE_REG;
2658         case OP_STOREI8_MEMBASE_IMM:
2659                 return OP_STOREI8_MEMBASE_REG;
2660         }
2661
2662         return -1;
2663 }
2664
2665 #define INST_IGNORES_CFLAGS(opcode) (!(((opcode) == OP_ADC) || ((opcode) == OP_ADC_IMM) || ((opcode) == OP_IADC) || ((opcode) == OP_IADC_IMM) || ((opcode) == OP_SBB) || ((opcode) == OP_SBB_IMM) || ((opcode) == OP_ISBB) || ((opcode) == OP_ISBB_IMM)))
2666
2667 /*
2668  * mono_arch_peephole_pass_1:
2669  *
2670  *   Perform peephole opts which should/can be performed before local regalloc
2671  */
2672 void
2673 mono_arch_peephole_pass_1 (MonoCompile *cfg, MonoBasicBlock *bb)
2674 {
2675         MonoInst *ins, *n;
2676
2677         MONO_BB_FOR_EACH_INS_SAFE (bb, n, ins) {
2678                 MonoInst *last_ins = ins->prev;
2679
2680                 switch (ins->opcode) {
2681                 case OP_ADD_IMM:
2682                 case OP_IADD_IMM:
2683                 case OP_LADD_IMM:
2684                         if ((ins->sreg1 < MONO_MAX_IREGS) && (ins->dreg >= MONO_MAX_IREGS) && (ins->inst_imm > 0)) {
2685                                 /*
2686                                  * X86_LEA is like ADD, but doesn't have the
2687                                  * sreg1==dreg restriction. inst_imm > 0 is needed since LEA sign-extends
2688                                  * its operand to 64 bit.
2689                                  */
2690                                 ins->opcode = OP_X86_LEA_MEMBASE;
2691                                 ins->inst_basereg = ins->sreg1;
2692                         }
2693                         break;
2694                 case OP_LXOR:
2695                 case OP_IXOR:
2696                         if ((ins->sreg1 == ins->sreg2) && (ins->sreg1 == ins->dreg)) {
2697                                 MonoInst *ins2;
2698
2699                                 /*
2700                                  * Replace STORE_MEMBASE_IMM 0 with STORE_MEMBASE_REG since
2701                                  * the latter has length 2-3 instead of 6 (reverse constant
2702                                  * propagation). These instruction sequences are very common
2703                                  * in the initlocals bblock.
2704                                  */
2705                                 for (ins2 = ins->next; ins2; ins2 = ins2->next) {
2706                                         if (((ins2->opcode == OP_STORE_MEMBASE_IMM) || (ins2->opcode == OP_STOREI4_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_IMM) || (ins2->opcode == OP_STORE_MEMBASE_IMM)) && (ins2->inst_imm == 0)) {
2707                                                 ins2->opcode = store_membase_imm_to_store_membase_reg (ins2->opcode);
2708                                                 ins2->sreg1 = ins->dreg;
2709                                         } else if ((ins2->opcode == OP_STOREI1_MEMBASE_IMM) || (ins2->opcode == OP_STOREI2_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_REG) || (ins2->opcode == OP_STORE_MEMBASE_REG)) {
2710                                                 /* Continue */
2711                                         } else if (((ins2->opcode == OP_ICONST) || (ins2->opcode == OP_I8CONST)) && (ins2->dreg == ins->dreg) && (ins2->inst_c0 == 0)) {
2712                                                 NULLIFY_INS (ins2);
2713                                                 /* Continue */
2714                                         } else {
2715                                                 break;
2716                                         }
2717                                 }
2718                         }
2719                         break;
2720                 case OP_COMPARE_IMM:
2721                 case OP_LCOMPARE_IMM:
2722                         /* OP_COMPARE_IMM (reg, 0)
2723                          * -->
2724                          * OP_AMD64_TEST_NULL (reg)
2725                          */
2726                         if (!ins->inst_imm)
2727                                 ins->opcode = OP_AMD64_TEST_NULL;
2728                         break;
2729                 case OP_ICOMPARE_IMM:
2730                         if (!ins->inst_imm)
2731                                 ins->opcode = OP_X86_TEST_NULL;
2732                         break;
2733                 case OP_AMD64_ICOMPARE_MEMBASE_IMM:
2734                         /*
2735                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
2736                          * OP_X86_COMPARE_MEMBASE_IMM offset(basereg), imm
2737                          * -->
2738                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
2739                          * OP_COMPARE_IMM reg, imm
2740                          *
2741                          * Note: if imm = 0 then OP_COMPARE_IMM replaced with OP_X86_TEST_NULL
2742                          */
2743                         if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_REG) &&
2744                             ins->inst_basereg == last_ins->inst_destbasereg &&
2745                             ins->inst_offset == last_ins->inst_offset) {
2746                                         ins->opcode = OP_ICOMPARE_IMM;
2747                                         ins->sreg1 = last_ins->sreg1;
2748
2749                                         /* check if we can remove cmp reg,0 with test null */
2750                                         if (!ins->inst_imm)
2751                                                 ins->opcode = OP_X86_TEST_NULL;
2752                                 }
2753
2754                         break;
2755                 }
2756
2757                 mono_peephole_ins (bb, ins);
2758         }
2759 }
2760
2761 void
2762 mono_arch_peephole_pass_2 (MonoCompile *cfg, MonoBasicBlock *bb)
2763 {
2764         MonoInst *ins, *n;
2765
2766         MONO_BB_FOR_EACH_INS_SAFE (bb, n, ins) {
2767                 switch (ins->opcode) {
2768                 case OP_ICONST:
2769                 case OP_I8CONST: {
2770                         /* reg = 0 -> XOR (reg, reg) */
2771                         /* XOR sets cflags on x86, so we cant do it always */
2772                         if (ins->inst_c0 == 0 && (!ins->next || (ins->next && INST_IGNORES_CFLAGS (ins->next->opcode)))) {
2773                                 ins->opcode = OP_LXOR;
2774                                 ins->sreg1 = ins->dreg;
2775                                 ins->sreg2 = ins->dreg;
2776                                 /* Fall through */
2777                         } else {
2778                                 break;
2779                         }
2780                 }
2781                 case OP_LXOR:
2782                         /*
2783                          * Use IXOR to avoid a rex prefix if possible. The cpu will sign extend the
2784                          * 0 result into 64 bits.
2785                          */
2786                         if ((ins->sreg1 == ins->sreg2) && (ins->sreg1 == ins->dreg)) {
2787                                 ins->opcode = OP_IXOR;
2788                         }
2789                         /* Fall through */
2790                 case OP_IXOR:
2791                         if ((ins->sreg1 == ins->sreg2) && (ins->sreg1 == ins->dreg)) {
2792                                 MonoInst *ins2;
2793
2794                                 /*
2795                                  * Replace STORE_MEMBASE_IMM 0 with STORE_MEMBASE_REG since
2796                                  * the latter has length 2-3 instead of 6 (reverse constant
2797                                  * propagation). These instruction sequences are very common
2798                                  * in the initlocals bblock.
2799                                  */
2800                                 for (ins2 = ins->next; ins2; ins2 = ins2->next) {
2801                                         if (((ins2->opcode == OP_STORE_MEMBASE_IMM) || (ins2->opcode == OP_STOREI4_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_IMM) || (ins2->opcode == OP_STORE_MEMBASE_IMM)) && (ins2->inst_imm == 0)) {
2802                                                 ins2->opcode = store_membase_imm_to_store_membase_reg (ins2->opcode);
2803                                                 ins2->sreg1 = ins->dreg;
2804                                         } else if ((ins2->opcode == OP_STOREI1_MEMBASE_IMM) || (ins2->opcode == OP_STOREI2_MEMBASE_IMM) || (ins2->opcode == OP_STOREI4_MEMBASE_REG) || (ins2->opcode == OP_STOREI8_MEMBASE_REG) || (ins2->opcode == OP_STORE_MEMBASE_REG) || (ins2->opcode == OP_LIVERANGE_START)) {
2805                                                 /* Continue */
2806                                         } else if (((ins2->opcode == OP_ICONST) || (ins2->opcode == OP_I8CONST)) && (ins2->dreg == ins->dreg) && (ins2->inst_c0 == 0)) {
2807                                                 NULLIFY_INS (ins2);
2808                                                 /* Continue */
2809                                         } else {
2810                                                 break;
2811                                         }
2812                                 }
2813                         }
2814                         break;
2815                 case OP_IADD_IMM:
2816                         if ((ins->inst_imm == 1) && (ins->dreg == ins->sreg1))
2817                                 ins->opcode = OP_X86_INC_REG;
2818                         break;
2819                 case OP_ISUB_IMM:
2820                         if ((ins->inst_imm == 1) && (ins->dreg == ins->sreg1))
2821                                 ins->opcode = OP_X86_DEC_REG;
2822                         break;
2823                 }
2824
2825                 mono_peephole_ins (bb, ins);
2826         }
2827 }
2828
2829 #define NEW_INS(cfg,ins,dest,op) do {   \
2830                 MONO_INST_NEW ((cfg), (dest), (op)); \
2831         (dest)->cil_code = (ins)->cil_code; \
2832         mono_bblock_insert_before_ins (bb, ins, (dest)); \
2833         } while (0)
2834
2835 /*
2836  * mono_arch_lowering_pass:
2837  *
2838  *  Converts complex opcodes into simpler ones so that each IR instruction
2839  * corresponds to one machine instruction.
2840  */
2841 void
2842 mono_arch_lowering_pass (MonoCompile *cfg, MonoBasicBlock *bb)
2843 {
2844         MonoInst *ins, *n, *temp;
2845
2846         /*
2847          * FIXME: Need to add more instructions, but the current machine
2848          * description can't model some parts of the composite instructions like
2849          * cdq.
2850          */
2851         MONO_BB_FOR_EACH_INS_SAFE (bb, n, ins) {
2852                 switch (ins->opcode) {
2853                 case OP_DIV_IMM:
2854                 case OP_REM_IMM:
2855                 case OP_IDIV_IMM:
2856                 case OP_IDIV_UN_IMM:
2857                 case OP_IREM_UN_IMM:
2858                         mono_decompose_op_imm (cfg, bb, ins);
2859                         break;
2860                 case OP_IREM_IMM:
2861                         /* Keep the opcode if we can implement it efficiently */
2862                         if (!((ins->inst_imm > 0) && (mono_is_power_of_two (ins->inst_imm) != -1)))
2863                                 mono_decompose_op_imm (cfg, bb, ins);
2864                         break;
2865                 case OP_COMPARE_IMM:
2866                 case OP_LCOMPARE_IMM:
2867                         if (!amd64_is_imm32 (ins->inst_imm)) {
2868                                 NEW_INS (cfg, ins, temp, OP_I8CONST);
2869                                 temp->inst_c0 = ins->inst_imm;
2870                                 temp->dreg = mono_alloc_ireg (cfg);
2871                                 ins->opcode = OP_COMPARE;
2872                                 ins->sreg2 = temp->dreg;
2873                         }
2874                         break;
2875                 case OP_LOAD_MEMBASE:
2876                 case OP_LOADI8_MEMBASE:
2877                         if (!amd64_is_imm32 (ins->inst_offset)) {
2878                                 NEW_INS (cfg, ins, temp, OP_I8CONST);
2879                                 temp->inst_c0 = ins->inst_offset;
2880                                 temp->dreg = mono_alloc_ireg (cfg);
2881                                 ins->opcode = OP_AMD64_LOADI8_MEMINDEX;
2882                                 ins->inst_indexreg = temp->dreg;
2883                         }
2884                         break;
2885                 case OP_STORE_MEMBASE_IMM:
2886                 case OP_STOREI8_MEMBASE_IMM:
2887                         if (!amd64_is_imm32 (ins->inst_imm)) {
2888                                 NEW_INS (cfg, ins, temp, OP_I8CONST);
2889                                 temp->inst_c0 = ins->inst_imm;
2890                                 temp->dreg = mono_alloc_ireg (cfg);
2891                                 ins->opcode = OP_STOREI8_MEMBASE_REG;
2892                                 ins->sreg1 = temp->dreg;
2893                         }
2894                         break;
2895 #ifdef MONO_ARCH_SIMD_INTRINSICS
2896                 case OP_EXPAND_I1: {
2897                                 int temp_reg1 = mono_alloc_ireg (cfg);
2898                                 int temp_reg2 = mono_alloc_ireg (cfg);
2899                                 int original_reg = ins->sreg1;
2900
2901                                 NEW_INS (cfg, ins, temp, OP_ICONV_TO_U1);
2902                                 temp->sreg1 = original_reg;
2903                                 temp->dreg = temp_reg1;
2904
2905                                 NEW_INS (cfg, ins, temp, OP_SHL_IMM);
2906                                 temp->sreg1 = temp_reg1;
2907                                 temp->dreg = temp_reg2;
2908                                 temp->inst_imm = 8;
2909
2910                                 NEW_INS (cfg, ins, temp, OP_LOR);
2911                                 temp->sreg1 = temp->dreg = temp_reg2;
2912                                 temp->sreg2 = temp_reg1;
2913
2914                                 ins->opcode = OP_EXPAND_I2;
2915                                 ins->sreg1 = temp_reg2;
2916                         }
2917                         break;
2918 #endif
2919                 default:
2920                         break;
2921                 }
2922         }
2923
2924         bb->max_vreg = cfg->next_vreg;
2925 }
2926
2927 static const int
2928 branch_cc_table [] = {
2929         X86_CC_EQ, X86_CC_GE, X86_CC_GT, X86_CC_LE, X86_CC_LT,
2930         X86_CC_NE, X86_CC_GE, X86_CC_GT, X86_CC_LE, X86_CC_LT,
2931         X86_CC_O, X86_CC_NO, X86_CC_C, X86_CC_NC
2932 };
2933
2934 /* Maps CMP_... constants to X86_CC_... constants */
2935 static const int
2936 cc_table [] = {
2937         X86_CC_EQ, X86_CC_NE, X86_CC_LE, X86_CC_GE, X86_CC_LT, X86_CC_GT,
2938         X86_CC_LE, X86_CC_GE, X86_CC_LT, X86_CC_GT
2939 };
2940
2941 static const int
2942 cc_signed_table [] = {
2943         TRUE, TRUE, TRUE, TRUE, TRUE, TRUE,
2944         FALSE, FALSE, FALSE, FALSE
2945 };
2946
2947 /*#include "cprop.c"*/
2948
2949 static unsigned char*
2950 emit_float_to_int (MonoCompile *cfg, guchar *code, int dreg, int sreg, int size, gboolean is_signed)
2951 {
2952         amd64_sse_cvttsd2si_reg_reg (code, dreg, sreg);
2953
2954         if (size == 1)
2955                 amd64_widen_reg (code, dreg, dreg, is_signed, FALSE);
2956         else if (size == 2)
2957                 amd64_widen_reg (code, dreg, dreg, is_signed, TRUE);
2958         return code;
2959 }
2960
2961 static unsigned char*
2962 mono_emit_stack_alloc (MonoCompile *cfg, guchar *code, MonoInst* tree)
2963 {
2964         int sreg = tree->sreg1;
2965         int need_touch = FALSE;
2966
2967 #if defined(HOST_WIN32) || defined(MONO_ARCH_SIGSEGV_ON_ALTSTACK)
2968         if (!tree->flags & MONO_INST_INIT)
2969                 need_touch = TRUE;
2970 #endif
2971
2972         if (need_touch) {
2973                 guint8* br[5];
2974
2975                 /*
2976                  * Under Windows:
2977                  * If requested stack size is larger than one page,
2978                  * perform stack-touch operation
2979                  */
2980                 /*
2981                  * Generate stack probe code.
2982                  * Under Windows, it is necessary to allocate one page at a time,
2983                  * "touching" stack after each successful sub-allocation. This is
2984                  * because of the way stack growth is implemented - there is a
2985                  * guard page before the lowest stack page that is currently commited.
2986                  * Stack normally grows sequentially so OS traps access to the
2987                  * guard page and commits more pages when needed.
2988                  */
2989                 amd64_test_reg_imm (code, sreg, ~0xFFF);
2990                 br[0] = code; x86_branch8 (code, X86_CC_Z, 0, FALSE);
2991
2992                 br[2] = code; /* loop */
2993                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 0x1000);
2994                 amd64_test_membase_reg (code, AMD64_RSP, 0, AMD64_RSP);
2995                 amd64_alu_reg_imm (code, X86_SUB, sreg, 0x1000);
2996                 amd64_alu_reg_imm (code, X86_CMP, sreg, 0x1000);
2997                 br[3] = code; x86_branch8 (code, X86_CC_AE, 0, FALSE);
2998                 amd64_patch (br[3], br[2]);
2999                 amd64_test_reg_reg (code, sreg, sreg);
3000                 br[4] = code; x86_branch8 (code, X86_CC_Z, 0, FALSE);
3001                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, sreg);
3002
3003                 br[1] = code; x86_jump8 (code, 0);
3004
3005                 amd64_patch (br[0], code);
3006                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, sreg);
3007                 amd64_patch (br[1], code);
3008                 amd64_patch (br[4], code);
3009         }
3010         else
3011                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, tree->sreg1);
3012
3013         if (tree->flags & MONO_INST_INIT) {
3014                 int offset = 0;
3015                 if (tree->dreg != AMD64_RAX && sreg != AMD64_RAX) {
3016                         amd64_push_reg (code, AMD64_RAX);
3017                         offset += 8;
3018                 }
3019                 if (tree->dreg != AMD64_RCX && sreg != AMD64_RCX) {
3020                         amd64_push_reg (code, AMD64_RCX);
3021                         offset += 8;
3022                 }
3023                 if (tree->dreg != AMD64_RDI && sreg != AMD64_RDI) {
3024                         amd64_push_reg (code, AMD64_RDI);
3025                         offset += 8;
3026                 }
3027
3028                 amd64_shift_reg_imm (code, X86_SHR, sreg, 3);
3029                 if (sreg != AMD64_RCX)
3030                         amd64_mov_reg_reg (code, AMD64_RCX, sreg, 8);
3031                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
3032
3033                 amd64_lea_membase (code, AMD64_RDI, AMD64_RSP, offset);
3034                 if (cfg->param_area && cfg->arch.no_pushes)
3035                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RDI, cfg->param_area);
3036                 amd64_cld (code);
3037                 amd64_prefix (code, X86_REP_PREFIX);
3038                 amd64_stosl (code);
3039
3040                 if (tree->dreg != AMD64_RDI && sreg != AMD64_RDI)
3041                         amd64_pop_reg (code, AMD64_RDI);
3042                 if (tree->dreg != AMD64_RCX && sreg != AMD64_RCX)
3043                         amd64_pop_reg (code, AMD64_RCX);
3044                 if (tree->dreg != AMD64_RAX && sreg != AMD64_RAX)
3045                         amd64_pop_reg (code, AMD64_RAX);
3046         }
3047         return code;
3048 }
3049
3050 static guint8*
3051 emit_move_return_value (MonoCompile *cfg, MonoInst *ins, guint8 *code)
3052 {
3053         CallInfo *cinfo;
3054         guint32 quad;
3055
3056         /* Move return value to the target register */
3057         /* FIXME: do this in the local reg allocator */
3058         switch (ins->opcode) {
3059         case OP_CALL:
3060         case OP_CALL_REG:
3061         case OP_CALL_MEMBASE:
3062         case OP_LCALL:
3063         case OP_LCALL_REG:
3064         case OP_LCALL_MEMBASE:
3065                 g_assert (ins->dreg == AMD64_RAX);
3066                 break;
3067         case OP_FCALL:
3068         case OP_FCALL_REG:
3069         case OP_FCALL_MEMBASE:
3070                 if (((MonoCallInst*)ins)->signature->ret->type == MONO_TYPE_R4) {
3071                         amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, AMD64_XMM0);
3072                 }
3073                 else {
3074                         if (ins->dreg != AMD64_XMM0)
3075                                 amd64_sse_movsd_reg_reg (code, ins->dreg, AMD64_XMM0);
3076                 }
3077                 break;
3078         case OP_VCALL:
3079         case OP_VCALL_REG:
3080         case OP_VCALL_MEMBASE:
3081         case OP_VCALL2:
3082         case OP_VCALL2_REG:
3083         case OP_VCALL2_MEMBASE:
3084                 cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, ((MonoCallInst*)ins)->signature, FALSE);
3085                 if (cinfo->ret.storage == ArgValuetypeInReg) {
3086                         MonoInst *loc = cfg->arch.vret_addr_loc;
3087
3088                         /* Load the destination address */
3089                         g_assert (loc->opcode == OP_REGOFFSET);
3090                         amd64_mov_reg_membase (code, AMD64_RCX, loc->inst_basereg, loc->inst_offset, 8);
3091
3092                         for (quad = 0; quad < 2; quad ++) {
3093                                 switch (cinfo->ret.pair_storage [quad]) {
3094                                 case ArgInIReg:
3095                                         amd64_mov_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad], 8);
3096                                         break;
3097                                 case ArgInFloatSSEReg:
3098                                         amd64_movss_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad]);
3099                                         break;
3100                                 case ArgInDoubleSSEReg:
3101                                         amd64_movsd_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad]);
3102                                         break;
3103                                 case ArgNone:
3104                                         break;
3105                                 default:
3106                                         NOT_IMPLEMENTED;
3107                                 }
3108                         }
3109                 }
3110                 break;
3111         }
3112
3113         return code;
3114 }
3115
3116 /*
3117  * mono_amd64_emit_tls_get:
3118  * @code: buffer to store code to
3119  * @dreg: hard register where to place the result
3120  * @tls_offset: offset info
3121  *
3122  * mono_amd64_emit_tls_get emits in @code the native code that puts in
3123  * the dreg register the item in the thread local storage identified
3124  * by tls_offset.
3125  *
3126  * Returns: a pointer to the end of the stored code
3127  */
3128 guint8*
3129 mono_amd64_emit_tls_get (guint8* code, int dreg, int tls_offset)
3130 {
3131 #ifdef HOST_WIN32
3132         g_assert (tls_offset < 64);
3133         x86_prefix (code, X86_GS_PREFIX);
3134         amd64_mov_reg_mem (code, dreg, (tls_offset * 8) + 0x1480, 8);
3135 #else
3136         if (optimize_for_xen) {
3137                 x86_prefix (code, X86_FS_PREFIX);
3138                 amd64_mov_reg_mem (code, dreg, 0, 8);
3139                 amd64_mov_reg_membase (code, dreg, dreg, tls_offset, 8);
3140         } else {
3141                 x86_prefix (code, X86_FS_PREFIX);
3142                 amd64_mov_reg_mem (code, dreg, tls_offset, 8);
3143         }
3144 #endif
3145         return code;
3146 }
3147
3148 #define REAL_PRINT_REG(text,reg) \
3149 mono_assert (reg >= 0); \
3150 amd64_push_reg (code, AMD64_RAX); \
3151 amd64_push_reg (code, AMD64_RDX); \
3152 amd64_push_reg (code, AMD64_RCX); \
3153 amd64_push_reg (code, reg); \
3154 amd64_push_imm (code, reg); \
3155 amd64_push_imm (code, text " %d %p\n"); \
3156 amd64_mov_reg_imm (code, AMD64_RAX, printf); \
3157 amd64_call_reg (code, AMD64_RAX); \
3158 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 3*4); \
3159 amd64_pop_reg (code, AMD64_RCX); \
3160 amd64_pop_reg (code, AMD64_RDX); \
3161 amd64_pop_reg (code, AMD64_RAX);
3162
3163 /* benchmark and set based on cpu */
3164 #define LOOP_ALIGNMENT 8
3165 #define bb_is_loop_start(bb) ((bb)->loop_body_start && (bb)->nesting)
3166
3167 #ifndef DISABLE_JIT
3168
3169 void
3170 mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
3171 {
3172         MonoInst *ins;
3173         MonoCallInst *call;
3174         guint offset;
3175         guint8 *code = cfg->native_code + cfg->code_len;
3176         MonoInst *last_ins = NULL;
3177         guint last_offset = 0;
3178         int max_len;
3179
3180         /* Fix max_offset estimate for each successor bb */
3181         if (cfg->opt & MONO_OPT_BRANCH) {
3182                 int current_offset = cfg->code_len;
3183                 MonoBasicBlock *current_bb;
3184                 for (current_bb = bb; current_bb != NULL; current_bb = current_bb->next_bb) {
3185                         current_bb->max_offset = current_offset;
3186                         current_offset += current_bb->max_length;
3187                 }
3188         }
3189
3190         if (cfg->opt & MONO_OPT_LOOP) {
3191                 int pad, align = LOOP_ALIGNMENT;
3192                 /* set alignment depending on cpu */
3193                 if (bb_is_loop_start (bb) && (pad = (cfg->code_len & (align - 1)))) {
3194                         pad = align - pad;
3195                         /*g_print ("adding %d pad at %x to loop in %s\n", pad, cfg->code_len, cfg->method->name);*/
3196                         amd64_padding (code, pad);
3197                         cfg->code_len += pad;
3198                         bb->native_offset = cfg->code_len;
3199                 }
3200         }
3201
3202         if (cfg->verbose_level > 2)
3203                 g_print ("Basic block %d starting at offset 0x%x\n", bb->block_num, bb->native_offset);
3204
3205         if (cfg->prof_options & MONO_PROFILE_COVERAGE) {
3206                 MonoProfileCoverageInfo *cov = cfg->coverage_info;
3207                 g_assert (!cfg->compile_aot);
3208
3209                 cov->data [bb->dfn].cil_code = bb->cil_code;
3210                 amd64_mov_reg_imm (code, AMD64_R11, (guint64)&cov->data [bb->dfn].count);
3211                 /* this is not thread save, but good enough */
3212                 amd64_inc_membase (code, AMD64_R11, 0);
3213         }
3214
3215         offset = code - cfg->native_code;
3216
3217         mono_debug_open_block (cfg, bb, offset);
3218
3219     if (mono_break_at_bb_method && mono_method_desc_full_match (mono_break_at_bb_method, cfg->method) && bb->block_num == mono_break_at_bb_bb_num)
3220                 x86_breakpoint (code);
3221
3222         MONO_BB_FOR_EACH_INS (bb, ins) {
3223                 offset = code - cfg->native_code;
3224
3225                 max_len = ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
3226
3227                 if (G_UNLIKELY (offset > (cfg->code_size - max_len - 16))) {
3228                         cfg->code_size *= 2;
3229                         cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
3230                         code = cfg->native_code + offset;
3231                         mono_jit_stats.code_reallocs++;
3232                 }
3233
3234                 if (cfg->debug_info)
3235                         mono_debug_record_line_number (cfg, ins, offset);
3236
3237                 switch (ins->opcode) {
3238                 case OP_BIGMUL:
3239                         amd64_mul_reg (code, ins->sreg2, TRUE);
3240                         break;
3241                 case OP_BIGMUL_UN:
3242                         amd64_mul_reg (code, ins->sreg2, FALSE);
3243                         break;
3244                 case OP_X86_SETEQ_MEMBASE:
3245                         amd64_set_membase (code, X86_CC_EQ, ins->inst_basereg, ins->inst_offset, TRUE);
3246                         break;
3247                 case OP_STOREI1_MEMBASE_IMM:
3248                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 1);
3249                         break;
3250                 case OP_STOREI2_MEMBASE_IMM:
3251                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 2);
3252                         break;
3253                 case OP_STOREI4_MEMBASE_IMM:
3254                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 4);
3255                         break;
3256                 case OP_STOREI1_MEMBASE_REG:
3257                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 1);
3258                         break;
3259                 case OP_STOREI2_MEMBASE_REG:
3260                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 2);
3261                         break;
3262                 case OP_STORE_MEMBASE_REG:
3263                 case OP_STOREI8_MEMBASE_REG:
3264                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 8);
3265                         break;
3266                 case OP_STOREI4_MEMBASE_REG:
3267                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 4);
3268                         break;
3269                 case OP_STORE_MEMBASE_IMM:
3270                 case OP_STOREI8_MEMBASE_IMM:
3271                         g_assert (amd64_is_imm32 (ins->inst_imm));
3272                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 8);
3273                         break;
3274                 case OP_LOAD_MEM:
3275                 case OP_LOADI8_MEM:
3276                         // FIXME: Decompose this earlier
3277                         if (amd64_is_imm32 (ins->inst_imm))
3278                                 amd64_mov_reg_mem (code, ins->dreg, ins->inst_imm, sizeof (gpointer));
3279                         else {
3280                                 amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
3281                                 amd64_mov_reg_membase (code, ins->dreg, ins->dreg, 0, 8);
3282                         }
3283                         break;
3284                 case OP_LOADI4_MEM:
3285                         amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
3286                         amd64_movsxd_reg_membase (code, ins->dreg, ins->dreg, 0);
3287                         break;
3288                 case OP_LOADU4_MEM:
3289                         // FIXME: Decompose this earlier
3290                         if (amd64_is_imm32 (ins->inst_imm))
3291                                 amd64_mov_reg_mem (code, ins->dreg, ins->inst_imm, 4);
3292                         else {
3293                                 amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
3294                                 amd64_mov_reg_membase (code, ins->dreg, ins->dreg, 0, 4);
3295                         }
3296                         break;
3297                 case OP_LOADU1_MEM:
3298                         amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
3299                         amd64_widen_membase (code, ins->dreg, ins->dreg, 0, FALSE, FALSE);
3300                         break;
3301                 case OP_LOADU2_MEM:
3302                         amd64_mov_reg_imm (code, ins->dreg, ins->inst_imm);
3303                         amd64_widen_membase (code, ins->dreg, ins->dreg, 0, FALSE, TRUE);
3304                         break;
3305                 case OP_LOAD_MEMBASE:
3306                 case OP_LOADI8_MEMBASE:
3307                         g_assert (amd64_is_imm32 (ins->inst_offset));
3308                         amd64_mov_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, sizeof (gpointer));
3309                         break;
3310                 case OP_LOADI4_MEMBASE:
3311                         amd64_movsxd_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
3312                         break;
3313                 case OP_LOADU4_MEMBASE:
3314                         amd64_mov_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, 4);
3315                         break;
3316                 case OP_LOADU1_MEMBASE:
3317                         /* The cpu zero extends the result into 64 bits */
3318                         amd64_widen_membase_size (code, ins->dreg, ins->inst_basereg, ins->inst_offset, FALSE, FALSE, 4);
3319                         break;
3320                 case OP_LOADI1_MEMBASE:
3321                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, TRUE, FALSE);
3322                         break;
3323                 case OP_LOADU2_MEMBASE:
3324                         /* The cpu zero extends the result into 64 bits */
3325                         amd64_widen_membase_size (code, ins->dreg, ins->inst_basereg, ins->inst_offset, FALSE, TRUE, 4);
3326                         break;
3327                 case OP_LOADI2_MEMBASE:
3328                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, TRUE, TRUE);
3329                         break;
3330                 case OP_AMD64_LOADI8_MEMINDEX:
3331                         amd64_mov_reg_memindex_size (code, ins->dreg, ins->inst_basereg, 0, ins->inst_indexreg, 0, 8);
3332                         break;
3333                 case OP_LCONV_TO_I1:
3334                 case OP_ICONV_TO_I1:
3335                 case OP_SEXT_I1:
3336                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, FALSE);
3337                         break;
3338                 case OP_LCONV_TO_I2:
3339                 case OP_ICONV_TO_I2:
3340                 case OP_SEXT_I2:
3341                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, TRUE);
3342                         break;
3343                 case OP_LCONV_TO_U1:
3344                 case OP_ICONV_TO_U1:
3345                         amd64_widen_reg (code, ins->dreg, ins->sreg1, FALSE, FALSE);
3346                         break;
3347                 case OP_LCONV_TO_U2:
3348                 case OP_ICONV_TO_U2:
3349                         amd64_widen_reg (code, ins->dreg, ins->sreg1, FALSE, TRUE);
3350                         break;
3351                 case OP_ZEXT_I4:
3352                         /* Clean out the upper word */
3353                         amd64_mov_reg_reg_size (code, ins->dreg, ins->sreg1, 4);
3354                         break;
3355                 case OP_SEXT_I4:
3356                         amd64_movsxd_reg_reg (code, ins->dreg, ins->sreg1);
3357                         break;
3358                 case OP_COMPARE:
3359                 case OP_LCOMPARE:
3360                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
3361                         break;
3362                 case OP_COMPARE_IMM:
3363                 case OP_LCOMPARE_IMM:
3364                         g_assert (amd64_is_imm32 (ins->inst_imm));
3365                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, ins->inst_imm);
3366                         break;
3367                 case OP_X86_COMPARE_REG_MEMBASE:
3368                         amd64_alu_reg_membase (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset);
3369                         break;
3370                 case OP_X86_TEST_NULL:
3371                         amd64_test_reg_reg_size (code, ins->sreg1, ins->sreg1, 4);
3372                         break;
3373                 case OP_AMD64_TEST_NULL:
3374                         amd64_test_reg_reg (code, ins->sreg1, ins->sreg1);
3375                         break;
3376
3377                 case OP_X86_ADD_REG_MEMBASE:
3378                         amd64_alu_reg_membase_size (code, X86_ADD, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3379                         break;
3380                 case OP_X86_SUB_REG_MEMBASE:
3381                         amd64_alu_reg_membase_size (code, X86_SUB, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3382                         break;
3383                 case OP_X86_AND_REG_MEMBASE:
3384                         amd64_alu_reg_membase_size (code, X86_AND, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3385                         break;
3386                 case OP_X86_OR_REG_MEMBASE:
3387                         amd64_alu_reg_membase_size (code, X86_OR, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3388                         break;
3389                 case OP_X86_XOR_REG_MEMBASE:
3390                         amd64_alu_reg_membase_size (code, X86_XOR, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3391                         break;
3392
3393                 case OP_X86_ADD_MEMBASE_IMM:
3394                         /* FIXME: Make a 64 version too */
3395                         amd64_alu_membase_imm_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3396                         break;
3397                 case OP_X86_SUB_MEMBASE_IMM:
3398                         g_assert (amd64_is_imm32 (ins->inst_imm));
3399                         amd64_alu_membase_imm_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3400                         break;
3401                 case OP_X86_AND_MEMBASE_IMM:
3402                         g_assert (amd64_is_imm32 (ins->inst_imm));
3403                         amd64_alu_membase_imm_size (code, X86_AND, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3404                         break;
3405                 case OP_X86_OR_MEMBASE_IMM:
3406                         g_assert (amd64_is_imm32 (ins->inst_imm));
3407                         amd64_alu_membase_imm_size (code, X86_OR, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3408                         break;
3409                 case OP_X86_XOR_MEMBASE_IMM:
3410                         g_assert (amd64_is_imm32 (ins->inst_imm));
3411                         amd64_alu_membase_imm_size (code, X86_XOR, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3412                         break;
3413                 case OP_X86_ADD_MEMBASE_REG:
3414                         amd64_alu_membase_reg_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
3415                         break;
3416                 case OP_X86_SUB_MEMBASE_REG:
3417                         amd64_alu_membase_reg_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
3418                         break;
3419                 case OP_X86_AND_MEMBASE_REG:
3420                         amd64_alu_membase_reg_size (code, X86_AND, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
3421                         break;
3422                 case OP_X86_OR_MEMBASE_REG:
3423                         amd64_alu_membase_reg_size (code, X86_OR, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
3424                         break;
3425                 case OP_X86_XOR_MEMBASE_REG:
3426                         amd64_alu_membase_reg_size (code, X86_XOR, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
3427                         break;
3428                 case OP_X86_INC_MEMBASE:
3429                         amd64_inc_membase_size (code, ins->inst_basereg, ins->inst_offset, 4);
3430                         break;
3431                 case OP_X86_INC_REG:
3432                         amd64_inc_reg_size (code, ins->dreg, 4);
3433                         break;
3434                 case OP_X86_DEC_MEMBASE:
3435                         amd64_dec_membase_size (code, ins->inst_basereg, ins->inst_offset, 4);
3436                         break;
3437                 case OP_X86_DEC_REG:
3438                         amd64_dec_reg_size (code, ins->dreg, 4);
3439                         break;
3440                 case OP_X86_MUL_REG_MEMBASE:
3441                 case OP_X86_MUL_MEMBASE_REG:
3442                         amd64_imul_reg_membase_size (code, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3443                         break;
3444                 case OP_AMD64_ICOMPARE_MEMBASE_REG:
3445                         amd64_alu_membase_reg_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
3446                         break;
3447                 case OP_AMD64_ICOMPARE_MEMBASE_IMM:
3448                         amd64_alu_membase_imm_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3449                         break;
3450                 case OP_AMD64_COMPARE_MEMBASE_REG:
3451                         amd64_alu_membase_reg_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
3452                         break;
3453                 case OP_AMD64_COMPARE_MEMBASE_IMM:
3454                         g_assert (amd64_is_imm32 (ins->inst_imm));
3455                         amd64_alu_membase_imm_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
3456                         break;
3457                 case OP_X86_COMPARE_MEMBASE8_IMM:
3458                         amd64_alu_membase8_imm_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3459                         break;
3460                 case OP_AMD64_ICOMPARE_REG_MEMBASE:
3461                         amd64_alu_reg_membase_size (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3462                         break;
3463                 case OP_AMD64_COMPARE_REG_MEMBASE:
3464                         amd64_alu_reg_membase_size (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
3465                         break;
3466
3467                 case OP_AMD64_ADD_REG_MEMBASE:
3468                         amd64_alu_reg_membase_size (code, X86_ADD, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
3469                         break;
3470                 case OP_AMD64_SUB_REG_MEMBASE:
3471                         amd64_alu_reg_membase_size (code, X86_SUB, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
3472                         break;
3473                 case OP_AMD64_AND_REG_MEMBASE:
3474                         amd64_alu_reg_membase_size (code, X86_AND, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
3475                         break;
3476                 case OP_AMD64_OR_REG_MEMBASE:
3477                         amd64_alu_reg_membase_size (code, X86_OR, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
3478                         break;
3479                 case OP_AMD64_XOR_REG_MEMBASE:
3480                         amd64_alu_reg_membase_size (code, X86_XOR, ins->sreg1, ins->sreg2, ins->inst_offset, 8);
3481                         break;
3482
3483                 case OP_AMD64_ADD_MEMBASE_REG:
3484                         amd64_alu_membase_reg_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
3485                         break;
3486                 case OP_AMD64_SUB_MEMBASE_REG:
3487                         amd64_alu_membase_reg_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
3488                         break;
3489                 case OP_AMD64_AND_MEMBASE_REG:
3490                         amd64_alu_membase_reg_size (code, X86_AND, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
3491                         break;
3492                 case OP_AMD64_OR_MEMBASE_REG:
3493                         amd64_alu_membase_reg_size (code, X86_OR, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
3494                         break;
3495                 case OP_AMD64_XOR_MEMBASE_REG:
3496                         amd64_alu_membase_reg_size (code, X86_XOR, ins->inst_basereg, ins->inst_offset, ins->sreg2, 8);
3497                         break;
3498
3499                 case OP_AMD64_ADD_MEMBASE_IMM:
3500                         g_assert (amd64_is_imm32 (ins->inst_imm));
3501                         amd64_alu_membase_imm_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
3502                         break;
3503                 case OP_AMD64_SUB_MEMBASE_IMM:
3504                         g_assert (amd64_is_imm32 (ins->inst_imm));
3505                         amd64_alu_membase_imm_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
3506                         break;
3507                 case OP_AMD64_AND_MEMBASE_IMM:
3508                         g_assert (amd64_is_imm32 (ins->inst_imm));
3509                         amd64_alu_membase_imm_size (code, X86_AND, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
3510                         break;
3511                 case OP_AMD64_OR_MEMBASE_IMM:
3512                         g_assert (amd64_is_imm32 (ins->inst_imm));
3513                         amd64_alu_membase_imm_size (code, X86_OR, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
3514                         break;
3515                 case OP_AMD64_XOR_MEMBASE_IMM:
3516                         g_assert (amd64_is_imm32 (ins->inst_imm));
3517                         amd64_alu_membase_imm_size (code, X86_XOR, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 8);
3518                         break;
3519
3520                 case OP_BREAK:
3521                         amd64_breakpoint (code);
3522                         break;
3523                 case OP_RELAXED_NOP:
3524                         x86_prefix (code, X86_REP_PREFIX);
3525                         x86_nop (code);
3526                         break;
3527                 case OP_HARD_NOP:
3528                         x86_nop (code);
3529                         break;
3530                 case OP_NOP:
3531                 case OP_DUMMY_USE:
3532                 case OP_DUMMY_STORE:
3533                 case OP_NOT_REACHED:
3534                 case OP_NOT_NULL:
3535                         break;
3536                 case OP_SEQ_POINT: {
3537                         int i;
3538
3539                         if (cfg->compile_aot)
3540                                 NOT_IMPLEMENTED;
3541
3542                         /*
3543                          * Read from the single stepping trigger page. This will cause a
3544                          * SIGSEGV when single stepping is enabled.
3545                          * We do this _before_ the breakpoint, so single stepping after
3546                          * a breakpoint is hit will step to the next IL offset.
3547                          */
3548                         if (ins->flags & MONO_INST_SINGLE_STEP_LOC) {
3549                                 if (((guint64)ss_trigger_page >> 32) == 0)
3550                                         amd64_mov_reg_mem (code, AMD64_R11, (guint64)ss_trigger_page, 4);
3551                                 else {
3552                                         MonoInst *var = cfg->arch.ss_trigger_page_var;
3553
3554                                         amd64_mov_reg_membase (code, AMD64_R11, var->inst_basereg, var->inst_offset, 8);
3555                                         amd64_alu_membase_imm_size (code, X86_CMP, AMD64_R11, 0, 0, 4);
3556                                 }
3557                         }
3558
3559                         /*
3560                          * This is the address which is saved in seq points,
3561                          * get_ip_for_single_step () / get_ip_for_breakpoint () needs to compute this
3562                          * from the address of the instruction causing the fault.
3563                          */
3564                         mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
3565
3566                         /*
3567                          * A placeholder for a possible breakpoint inserted by
3568                          * mono_arch_set_breakpoint ().
3569                          */
3570                         for (i = 0; i < breakpoint_size; ++i)
3571                                 x86_nop (code);
3572                         break;
3573                 }
3574                 case OP_ADDCC:
3575                 case OP_LADD:
3576                         amd64_alu_reg_reg (code, X86_ADD, ins->sreg1, ins->sreg2);
3577                         break;
3578                 case OP_ADC:
3579                         amd64_alu_reg_reg (code, X86_ADC, ins->sreg1, ins->sreg2);
3580                         break;
3581                 case OP_ADD_IMM:
3582                 case OP_LADD_IMM:
3583                         g_assert (amd64_is_imm32 (ins->inst_imm));
3584                         amd64_alu_reg_imm (code, X86_ADD, ins->dreg, ins->inst_imm);
3585                         break;
3586                 case OP_ADC_IMM:
3587                         g_assert (amd64_is_imm32 (ins->inst_imm));
3588                         amd64_alu_reg_imm (code, X86_ADC, ins->dreg, ins->inst_imm);
3589                         break;
3590                 case OP_SUBCC:
3591                 case OP_LSUB:
3592                         amd64_alu_reg_reg (code, X86_SUB, ins->sreg1, ins->sreg2);
3593                         break;
3594                 case OP_SBB:
3595                         amd64_alu_reg_reg (code, X86_SBB, ins->sreg1, ins->sreg2);
3596                         break;
3597                 case OP_SUB_IMM:
3598                 case OP_LSUB_IMM:
3599                         g_assert (amd64_is_imm32 (ins->inst_imm));
3600                         amd64_alu_reg_imm (code, X86_SUB, ins->dreg, ins->inst_imm);
3601                         break;
3602                 case OP_SBB_IMM:
3603                         g_assert (amd64_is_imm32 (ins->inst_imm));
3604                         amd64_alu_reg_imm (code, X86_SBB, ins->dreg, ins->inst_imm);
3605                         break;
3606                 case OP_LAND:
3607                         amd64_alu_reg_reg (code, X86_AND, ins->sreg1, ins->sreg2);
3608                         break;
3609                 case OP_AND_IMM:
3610                 case OP_LAND_IMM:
3611                         g_assert (amd64_is_imm32 (ins->inst_imm));
3612                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ins->inst_imm);
3613                         break;
3614                 case OP_LMUL:
3615                         amd64_imul_reg_reg (code, ins->sreg1, ins->sreg2);
3616                         break;
3617                 case OP_MUL_IMM:
3618                 case OP_LMUL_IMM:
3619                 case OP_IMUL_IMM: {
3620                         guint32 size = (ins->opcode == OP_IMUL_IMM) ? 4 : 8;
3621
3622                         switch (ins->inst_imm) {
3623                         case 2:
3624                                 /* MOV r1, r2 */
3625                                 /* ADD r1, r1 */
3626                                 if (ins->dreg != ins->sreg1)
3627                                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, size);
3628                                 amd64_alu_reg_reg (code, X86_ADD, ins->dreg, ins->dreg);
3629                                 break;
3630                         case 3:
3631                                 /* LEA r1, [r2 + r2*2] */
3632                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 1);
3633                                 break;
3634                         case 5:
3635                                 /* LEA r1, [r2 + r2*4] */
3636                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
3637                                 break;
3638                         case 6:
3639                                 /* LEA r1, [r2 + r2*2] */
3640                                 /* ADD r1, r1          */
3641                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 1);
3642                                 amd64_alu_reg_reg (code, X86_ADD, ins->dreg, ins->dreg);
3643                                 break;
3644                         case 9:
3645                                 /* LEA r1, [r2 + r2*8] */
3646                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 3);
3647                                 break;
3648                         case 10:
3649                                 /* LEA r1, [r2 + r2*4] */
3650                                 /* ADD r1, r1          */
3651                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
3652                                 amd64_alu_reg_reg (code, X86_ADD, ins->dreg, ins->dreg);
3653                                 break;
3654                         case 12:
3655                                 /* LEA r1, [r2 + r2*2] */
3656                                 /* SHL r1, 2           */
3657                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 1);
3658                                 amd64_shift_reg_imm (code, X86_SHL, ins->dreg, 2);
3659                                 break;
3660                         case 25:
3661                                 /* LEA r1, [r2 + r2*4] */
3662                                 /* LEA r1, [r1 + r1*4] */
3663                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
3664                                 amd64_lea_memindex (code, ins->dreg, ins->dreg, 0, ins->dreg, 2);
3665                                 break;
3666                         case 100:
3667                                 /* LEA r1, [r2 + r2*4] */
3668                                 /* SHL r1, 2           */
3669                                 /* LEA r1, [r1 + r1*4] */
3670                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
3671                                 amd64_shift_reg_imm (code, X86_SHL, ins->dreg, 2);
3672                                 amd64_lea_memindex (code, ins->dreg, ins->dreg, 0, ins->dreg, 2);
3673                                 break;
3674                         default:
3675                                 amd64_imul_reg_reg_imm_size (code, ins->dreg, ins->sreg1, ins->inst_imm, size);
3676                                 break;
3677                         }
3678                         break;
3679                 }
3680                 case OP_LDIV:
3681                 case OP_LREM:
3682                         /* Regalloc magic makes the div/rem cases the same */
3683                         if (ins->sreg2 == AMD64_RDX) {
3684                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
3685                                 amd64_cdq (code);
3686                                 amd64_div_membase (code, AMD64_RSP, -8, TRUE);
3687                         } else {
3688                                 amd64_cdq (code);
3689                                 amd64_div_reg (code, ins->sreg2, TRUE);
3690                         }
3691                         break;
3692                 case OP_LDIV_UN:
3693                 case OP_LREM_UN:
3694                         if (ins->sreg2 == AMD64_RDX) {
3695                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
3696                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
3697                                 amd64_div_membase (code, AMD64_RSP, -8, FALSE);
3698                         } else {
3699                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
3700                                 amd64_div_reg (code, ins->sreg2, FALSE);
3701                         }
3702                         break;
3703                 case OP_IDIV:
3704                 case OP_IREM:
3705                         if (ins->sreg2 == AMD64_RDX) {
3706                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
3707                                 amd64_cdq_size (code, 4);
3708                                 amd64_div_membase_size (code, AMD64_RSP, -8, TRUE, 4);
3709                         } else {
3710                                 amd64_cdq_size (code, 4);
3711                                 amd64_div_reg_size (code, ins->sreg2, TRUE, 4);
3712                         }
3713                         break;
3714                 case OP_IDIV_UN:
3715                 case OP_IREM_UN:
3716                         if (ins->sreg2 == AMD64_RDX) {
3717                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
3718                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
3719                                 amd64_div_membase_size (code, AMD64_RSP, -8, FALSE, 4);
3720                         } else {
3721                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
3722                                 amd64_div_reg_size (code, ins->sreg2, FALSE, 4);
3723                         }
3724                         break;
3725                 case OP_IREM_IMM: {
3726                         int power = mono_is_power_of_two (ins->inst_imm);
3727
3728                         g_assert (ins->sreg1 == X86_EAX);
3729                         g_assert (ins->dreg == X86_EAX);
3730                         g_assert (power >= 0);
3731
3732                         if (power == 0) {
3733                                 amd64_mov_reg_imm (code, ins->dreg, 0);
3734                                 break;
3735                         }
3736
3737                         /* Based on gcc code */
3738
3739                         /* Add compensation for negative dividents */
3740                         amd64_mov_reg_reg_size (code, AMD64_RDX, AMD64_RAX, 4);
3741                         if (power > 1)
3742                                 amd64_shift_reg_imm_size (code, X86_SAR, AMD64_RDX, 31, 4);
3743                         amd64_shift_reg_imm_size (code, X86_SHR, AMD64_RDX, 32 - power, 4);
3744                         amd64_alu_reg_reg_size (code, X86_ADD, AMD64_RAX, AMD64_RDX, 4);
3745                         /* Compute remainder */
3746                         amd64_alu_reg_imm_size (code, X86_AND, AMD64_RAX, (1 << power) - 1, 4);
3747                         /* Remove compensation */
3748                         amd64_alu_reg_reg_size (code, X86_SUB, AMD64_RAX, AMD64_RDX, 4);
3749                         break;
3750                 }
3751                 case OP_LMUL_OVF:
3752                         amd64_imul_reg_reg (code, ins->sreg1, ins->sreg2);
3753                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
3754                         break;
3755                 case OP_LOR:
3756                         amd64_alu_reg_reg (code, X86_OR, ins->sreg1, ins->sreg2);
3757                         break;
3758                 case OP_OR_IMM:
3759                 case OP_LOR_IMM:
3760                         g_assert (amd64_is_imm32 (ins->inst_imm));
3761                         amd64_alu_reg_imm (code, X86_OR, ins->sreg1, ins->inst_imm);
3762                         break;
3763                 case OP_LXOR:
3764                         amd64_alu_reg_reg (code, X86_XOR, ins->sreg1, ins->sreg2);
3765                         break;
3766                 case OP_XOR_IMM:
3767                 case OP_LXOR_IMM:
3768                         g_assert (amd64_is_imm32 (ins->inst_imm));
3769                         amd64_alu_reg_imm (code, X86_XOR, ins->sreg1, ins->inst_imm);
3770                         break;
3771                 case OP_LSHL:
3772                         g_assert (ins->sreg2 == AMD64_RCX);
3773                         amd64_shift_reg (code, X86_SHL, ins->dreg);
3774                         break;
3775                 case OP_LSHR:
3776                         g_assert (ins->sreg2 == AMD64_RCX);
3777                         amd64_shift_reg (code, X86_SAR, ins->dreg);
3778                         break;
3779                 case OP_SHR_IMM:
3780                         g_assert (amd64_is_imm32 (ins->inst_imm));
3781                         amd64_shift_reg_imm_size (code, X86_SAR, ins->dreg, ins->inst_imm, 4);
3782                         break;
3783                 case OP_LSHR_IMM:
3784                         g_assert (amd64_is_imm32 (ins->inst_imm));
3785                         amd64_shift_reg_imm (code, X86_SAR, ins->dreg, ins->inst_imm);
3786                         break;
3787                 case OP_SHR_UN_IMM:
3788                         g_assert (amd64_is_imm32 (ins->inst_imm));
3789                         amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, ins->inst_imm, 4);
3790                         break;
3791                 case OP_LSHR_UN_IMM:
3792                         g_assert (amd64_is_imm32 (ins->inst_imm));
3793                         amd64_shift_reg_imm (code, X86_SHR, ins->dreg, ins->inst_imm);
3794                         break;
3795                 case OP_LSHR_UN:
3796                         g_assert (ins->sreg2 == AMD64_RCX);
3797                         amd64_shift_reg (code, X86_SHR, ins->dreg);
3798                         break;
3799                 case OP_SHL_IMM:
3800                         g_assert (amd64_is_imm32 (ins->inst_imm));
3801                         amd64_shift_reg_imm_size (code, X86_SHL, ins->dreg, ins->inst_imm, 4);
3802                         break;
3803                 case OP_LSHL_IMM:
3804                         g_assert (amd64_is_imm32 (ins->inst_imm));
3805                         amd64_shift_reg_imm (code, X86_SHL, ins->dreg, ins->inst_imm);
3806                         break;
3807
3808                 case OP_IADDCC:
3809                 case OP_IADD:
3810                         amd64_alu_reg_reg_size (code, X86_ADD, ins->sreg1, ins->sreg2, 4);
3811                         break;
3812                 case OP_IADC:
3813                         amd64_alu_reg_reg_size (code, X86_ADC, ins->sreg1, ins->sreg2, 4);
3814                         break;
3815                 case OP_IADD_IMM:
3816                         amd64_alu_reg_imm_size (code, X86_ADD, ins->dreg, ins->inst_imm, 4);
3817                         break;
3818                 case OP_IADC_IMM:
3819                         amd64_alu_reg_imm_size (code, X86_ADC, ins->dreg, ins->inst_imm, 4);
3820                         break;
3821                 case OP_ISUBCC:
3822                 case OP_ISUB:
3823                         amd64_alu_reg_reg_size (code, X86_SUB, ins->sreg1, ins->sreg2, 4);
3824                         break;
3825                 case OP_ISBB:
3826                         amd64_alu_reg_reg_size (code, X86_SBB, ins->sreg1, ins->sreg2, 4);
3827                         break;
3828                 case OP_ISUB_IMM:
3829                         amd64_alu_reg_imm_size (code, X86_SUB, ins->dreg, ins->inst_imm, 4);
3830                         break;
3831                 case OP_ISBB_IMM:
3832                         amd64_alu_reg_imm_size (code, X86_SBB, ins->dreg, ins->inst_imm, 4);
3833                         break;
3834                 case OP_IAND:
3835                         amd64_alu_reg_reg_size (code, X86_AND, ins->sreg1, ins->sreg2, 4);
3836                         break;
3837                 case OP_IAND_IMM:
3838                         amd64_alu_reg_imm_size (code, X86_AND, ins->sreg1, ins->inst_imm, 4);
3839                         break;
3840                 case OP_IOR:
3841                         amd64_alu_reg_reg_size (code, X86_OR, ins->sreg1, ins->sreg2, 4);
3842                         break;
3843                 case OP_IOR_IMM:
3844                         amd64_alu_reg_imm_size (code, X86_OR, ins->sreg1, ins->inst_imm, 4);
3845                         break;
3846                 case OP_IXOR:
3847                         amd64_alu_reg_reg_size (code, X86_XOR, ins->sreg1, ins->sreg2, 4);
3848                         break;
3849                 case OP_IXOR_IMM:
3850                         amd64_alu_reg_imm_size (code, X86_XOR, ins->sreg1, ins->inst_imm, 4);
3851                         break;
3852                 case OP_INEG:
3853                         amd64_neg_reg_size (code, ins->sreg1, 4);
3854                         break;
3855                 case OP_INOT:
3856                         amd64_not_reg_size (code, ins->sreg1, 4);
3857                         break;
3858                 case OP_ISHL:
3859                         g_assert (ins->sreg2 == AMD64_RCX);
3860                         amd64_shift_reg_size (code, X86_SHL, ins->dreg, 4);
3861                         break;
3862                 case OP_ISHR:
3863                         g_assert (ins->sreg2 == AMD64_RCX);
3864                         amd64_shift_reg_size (code, X86_SAR, ins->dreg, 4);
3865                         break;
3866                 case OP_ISHR_IMM:
3867                         amd64_shift_reg_imm_size (code, X86_SAR, ins->dreg, ins->inst_imm, 4);
3868                         break;
3869                 case OP_ISHR_UN_IMM:
3870                         amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, ins->inst_imm, 4);
3871                         break;
3872                 case OP_ISHR_UN:
3873                         g_assert (ins->sreg2 == AMD64_RCX);
3874                         amd64_shift_reg_size (code, X86_SHR, ins->dreg, 4);
3875                         break;
3876                 case OP_ISHL_IMM:
3877                         amd64_shift_reg_imm_size (code, X86_SHL, ins->dreg, ins->inst_imm, 4);
3878                         break;
3879                 case OP_IMUL:
3880                         amd64_imul_reg_reg_size (code, ins->sreg1, ins->sreg2, 4);
3881                         break;
3882                 case OP_IMUL_OVF:
3883                         amd64_imul_reg_reg_size (code, ins->sreg1, ins->sreg2, 4);
3884                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
3885                         break;
3886                 case OP_IMUL_OVF_UN:
3887                 case OP_LMUL_OVF_UN: {
3888                         /* the mul operation and the exception check should most likely be split */
3889                         int non_eax_reg, saved_eax = FALSE, saved_edx = FALSE;
3890                         int size = (ins->opcode == OP_IMUL_OVF_UN) ? 4 : 8;
3891                         /*g_assert (ins->sreg2 == X86_EAX);
3892                         g_assert (ins->dreg == X86_EAX);*/
3893                         if (ins->sreg2 == X86_EAX) {
3894                                 non_eax_reg = ins->sreg1;
3895                         } else if (ins->sreg1 == X86_EAX) {
3896                                 non_eax_reg = ins->sreg2;
3897                         } else {
3898                                 /* no need to save since we're going to store to it anyway */
3899                                 if (ins->dreg != X86_EAX) {
3900                                         saved_eax = TRUE;
3901                                         amd64_push_reg (code, X86_EAX);
3902                                 }
3903                                 amd64_mov_reg_reg (code, X86_EAX, ins->sreg1, size);
3904                                 non_eax_reg = ins->sreg2;
3905                         }
3906                         if (ins->dreg == X86_EDX) {
3907                                 if (!saved_eax) {
3908                                         saved_eax = TRUE;
3909                                         amd64_push_reg (code, X86_EAX);
3910                                 }
3911                         } else {
3912                                 saved_edx = TRUE;
3913                                 amd64_push_reg (code, X86_EDX);
3914                         }
3915                         amd64_mul_reg_size (code, non_eax_reg, FALSE, size);
3916                         /* save before the check since pop and mov don't change the flags */
3917                         if (ins->dreg != X86_EAX)
3918                                 amd64_mov_reg_reg (code, ins->dreg, X86_EAX, size);
3919                         if (saved_edx)
3920                                 amd64_pop_reg (code, X86_EDX);
3921                         if (saved_eax)
3922                                 amd64_pop_reg (code, X86_EAX);
3923                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
3924                         break;
3925                 }
3926                 case OP_ICOMPARE:
3927                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
3928                         break;
3929                 case OP_ICOMPARE_IMM:
3930                         amd64_alu_reg_imm_size (code, X86_CMP, ins->sreg1, ins->inst_imm, 4);
3931                         break;
3932                 case OP_IBEQ:
3933                 case OP_IBLT:
3934                 case OP_IBGT:
3935                 case OP_IBGE:
3936                 case OP_IBLE:
3937                 case OP_LBEQ:
3938                 case OP_LBLT:
3939                 case OP_LBGT:
3940                 case OP_LBGE:
3941                 case OP_LBLE:
3942                 case OP_IBNE_UN:
3943                 case OP_IBLT_UN:
3944                 case OP_IBGT_UN:
3945                 case OP_IBGE_UN:
3946                 case OP_IBLE_UN:
3947                 case OP_LBNE_UN:
3948                 case OP_LBLT_UN:
3949                 case OP_LBGT_UN:
3950                 case OP_LBGE_UN:
3951                 case OP_LBLE_UN:
3952                         EMIT_COND_BRANCH (ins, cc_table [mono_opcode_to_cond (ins->opcode)], cc_signed_table [mono_opcode_to_cond (ins->opcode)]);
3953                         break;
3954
3955                 case OP_CMOV_IEQ:
3956                 case OP_CMOV_IGE:
3957                 case OP_CMOV_IGT:
3958                 case OP_CMOV_ILE:
3959                 case OP_CMOV_ILT:
3960                 case OP_CMOV_INE_UN:
3961                 case OP_CMOV_IGE_UN:
3962                 case OP_CMOV_IGT_UN:
3963                 case OP_CMOV_ILE_UN:
3964                 case OP_CMOV_ILT_UN:
3965                 case OP_CMOV_LEQ:
3966                 case OP_CMOV_LGE:
3967                 case OP_CMOV_LGT:
3968                 case OP_CMOV_LLE:
3969                 case OP_CMOV_LLT:
3970                 case OP_CMOV_LNE_UN:
3971                 case OP_CMOV_LGE_UN:
3972                 case OP_CMOV_LGT_UN:
3973                 case OP_CMOV_LLE_UN:
3974                 case OP_CMOV_LLT_UN:
3975                         g_assert (ins->dreg == ins->sreg1);
3976                         /* This needs to operate on 64 bit values */
3977                         amd64_cmov_reg (code, cc_table [mono_opcode_to_cond (ins->opcode)], cc_signed_table [mono_opcode_to_cond (ins->opcode)], ins->dreg, ins->sreg2);
3978                         break;
3979
3980                 case OP_LNOT:
3981                         amd64_not_reg (code, ins->sreg1);
3982                         break;
3983                 case OP_LNEG:
3984                         amd64_neg_reg (code, ins->sreg1);
3985                         break;
3986
3987                 case OP_ICONST:
3988                 case OP_I8CONST:
3989                         if ((((guint64)ins->inst_c0) >> 32) == 0)
3990                                 amd64_mov_reg_imm_size (code, ins->dreg, ins->inst_c0, 4);
3991                         else
3992                                 amd64_mov_reg_imm_size (code, ins->dreg, ins->inst_c0, 8);
3993                         break;
3994                 case OP_AOTCONST:
3995                         mono_add_patch_info (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
3996                         amd64_mov_reg_membase (code, ins->dreg, AMD64_RIP, 0, 8);
3997                         break;
3998                 case OP_JUMP_TABLE:
3999                         mono_add_patch_info (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
4000                         amd64_mov_reg_imm_size (code, ins->dreg, 0, 8);
4001                         break;
4002                 case OP_MOVE:
4003                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, sizeof (gpointer));
4004                         break;
4005                 case OP_AMD64_SET_XMMREG_R4: {
4006                         amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg1);
4007                         break;
4008                 }
4009                 case OP_AMD64_SET_XMMREG_R8: {
4010                         if (ins->dreg != ins->sreg1)
4011                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
4012                         break;
4013                 }
4014                 case OP_TAILCALL: {
4015                         /*
4016                          * Note: this 'frame destruction' logic is useful for tail calls, too.
4017                          * Keep in sync with the code in emit_epilog.
4018                          */
4019                         int pos = 0, i;
4020
4021                         /* FIXME: no tracing support... */
4022                         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
4023                                 code = mono_arch_instrument_epilog_full (cfg, mono_profiler_method_leave, code, FALSE, FALSE);
4024
4025                         g_assert (!cfg->method->save_lmf);
4026
4027                         if (cfg->arch.omit_fp) {
4028                                 guint32 save_offset = 0;
4029                                 /* Pop callee-saved registers */
4030                                 for (i = 0; i < AMD64_NREG; ++i)
4031                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
4032                                                 amd64_mov_reg_membase (code, i, AMD64_RSP, save_offset, 8);
4033                                                 save_offset += 8;
4034                                         }
4035                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, cfg->arch.stack_alloc_size);
4036                         }
4037                         else {
4038                                 for (i = 0; i < AMD64_NREG; ++i)
4039                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i)))
4040                                                 pos -= sizeof (gpointer);
4041
4042                                 if (pos)
4043                                         amd64_lea_membase (code, AMD64_RSP, AMD64_RBP, pos);
4044
4045                                 /* Pop registers in reverse order */
4046                                 for (i = AMD64_NREG - 1; i > 0; --i)
4047                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
4048                                                 amd64_pop_reg (code, i);
4049                                         }
4050
4051                                 amd64_leave (code);
4052                         }
4053
4054                         offset = code - cfg->native_code;
4055                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_METHOD_JUMP, ins->inst_p0);
4056                         if (cfg->compile_aot)
4057                                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_RIP, 0, 8);
4058                         else
4059                                 amd64_set_reg_template (code, AMD64_R11);
4060                         amd64_jump_reg (code, AMD64_R11);
4061                         break;
4062                 }
4063                 case OP_CHECK_THIS:
4064                         /* ensure ins->sreg1 is not NULL */
4065                         amd64_alu_membase_imm_size (code, X86_CMP, ins->sreg1, 0, 0, 4);
4066                         break;
4067                 case OP_ARGLIST: {
4068                         amd64_lea_membase (code, AMD64_R11, cfg->frame_reg, cfg->sig_cookie);
4069                         amd64_mov_membase_reg (code, ins->sreg1, 0, AMD64_R11, 8);
4070                         break;
4071                 }
4072                 case OP_CALL:
4073                 case OP_FCALL:
4074                 case OP_LCALL:
4075                 case OP_VCALL:
4076                 case OP_VCALL2:
4077                 case OP_VOIDCALL:
4078                         call = (MonoCallInst*)ins;
4079                         /*
4080                          * The AMD64 ABI forces callers to know about varargs.
4081                          */
4082                         if ((call->signature->call_convention == MONO_CALL_VARARG) && (call->signature->pinvoke))
4083                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
4084                         else if ((cfg->method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) && (cfg->method->klass->image != mono_defaults.corlib)) {
4085                                 /*
4086                                  * Since the unmanaged calling convention doesn't contain a
4087                                  * 'vararg' entry, we have to treat every pinvoke call as a
4088                                  * potential vararg call.
4089                                  */
4090                                 guint32 nregs, i;
4091                                 nregs = 0;
4092                                 for (i = 0; i < AMD64_XMM_NREG; ++i)
4093                                         if (call->used_fregs & (1 << i))
4094                                                 nregs ++;
4095                                 if (!nregs)
4096                                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
4097                                 else
4098                                         amd64_mov_reg_imm (code, AMD64_RAX, nregs);
4099                         }
4100
4101                         if (ins->flags & MONO_INST_HAS_METHOD)
4102                                 code = emit_call (cfg, code, MONO_PATCH_INFO_METHOD, call->method, FALSE);
4103                         else
4104                                 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, call->fptr, FALSE);
4105                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention) && !cfg->arch.no_pushes)
4106                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
4107                         code = emit_move_return_value (cfg, ins, code);
4108                         break;
4109                 case OP_FCALL_REG:
4110                 case OP_LCALL_REG:
4111                 case OP_VCALL_REG:
4112                 case OP_VCALL2_REG:
4113                 case OP_VOIDCALL_REG:
4114                 case OP_CALL_REG:
4115                         call = (MonoCallInst*)ins;
4116
4117                         if (AMD64_IS_ARGUMENT_REG (ins->sreg1)) {
4118                                 amd64_mov_reg_reg (code, AMD64_R11, ins->sreg1, 8);
4119                                 ins->sreg1 = AMD64_R11;
4120                         }
4121
4122                         /*
4123                          * The AMD64 ABI forces callers to know about varargs.
4124                          */
4125                         if ((call->signature->call_convention == MONO_CALL_VARARG) && (call->signature->pinvoke)) {
4126                                 if (ins->sreg1 == AMD64_RAX) {
4127                                         amd64_mov_reg_reg (code, AMD64_R11, AMD64_RAX, 8);
4128                                         ins->sreg1 = AMD64_R11;
4129                                 }
4130                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
4131                         } else if ((cfg->method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) && (cfg->method->klass->image != mono_defaults.corlib)) {
4132                                 /*
4133                                  * Since the unmanaged calling convention doesn't contain a
4134                                  * 'vararg' entry, we have to treat every pinvoke call as a
4135                                  * potential vararg call.
4136                                  */
4137                                 guint32 nregs, i;
4138                                 nregs = 0;
4139                                 for (i = 0; i < AMD64_XMM_NREG; ++i)
4140                                         if (call->used_fregs & (1 << i))
4141                                                 nregs ++;
4142                                 if (ins->sreg1 == AMD64_RAX) {
4143                                         amd64_mov_reg_reg (code, AMD64_R11, AMD64_RAX, 8);
4144                                         ins->sreg1 = AMD64_R11;
4145                                 }
4146                                 if (!nregs)
4147                                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
4148                                 else
4149                                         amd64_mov_reg_imm (code, AMD64_RAX, nregs);
4150                         }
4151
4152                         amd64_call_reg (code, ins->sreg1);
4153                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention) && !cfg->arch.no_pushes)
4154                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
4155                         code = emit_move_return_value (cfg, ins, code);
4156                         break;
4157                 case OP_FCALL_MEMBASE:
4158                 case OP_LCALL_MEMBASE:
4159                 case OP_VCALL_MEMBASE:
4160                 case OP_VCALL2_MEMBASE:
4161                 case OP_VOIDCALL_MEMBASE:
4162                 case OP_CALL_MEMBASE:
4163                         call = (MonoCallInst*)ins;
4164
4165                         if (AMD64_IS_ARGUMENT_REG (ins->sreg1)) {
4166                                 /*
4167                                  * Can't use R11 because it is clobbered by the trampoline
4168                                  * code, and the reg value is needed by get_vcall_slot_addr.
4169                                  */
4170                                 amd64_mov_reg_reg (code, AMD64_RAX, ins->sreg1, 8);
4171                                 ins->sreg1 = AMD64_RAX;
4172                         }
4173
4174                         /*
4175                          * Emit a few nops to simplify get_vcall_slot ().
4176                          */
4177                         amd64_nop (code);
4178                         amd64_nop (code);
4179                         amd64_nop (code);
4180
4181                         amd64_call_membase (code, ins->sreg1, ins->inst_offset);
4182                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention) && !cfg->arch.no_pushes)
4183                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
4184                         code = emit_move_return_value (cfg, ins, code);
4185                         break;
4186                 case OP_DYN_CALL: {
4187                         int i;
4188                         MonoInst *var = cfg->dyn_call_var;
4189
4190                         g_assert (var->opcode == OP_REGOFFSET);
4191
4192                         /* r11 = args buffer filled by mono_arch_get_dyn_call_args () */
4193                         amd64_mov_reg_reg (code, AMD64_R11, ins->sreg1, 8);
4194                         /* r10 = ftn */
4195                         amd64_mov_reg_reg (code, AMD64_R10, ins->sreg2, 8);
4196
4197                         /* Save args buffer */
4198                         amd64_mov_membase_reg (code, var->inst_basereg, var->inst_offset, AMD64_R11, 8);
4199
4200                         /* Set argument registers */
4201                         for (i = 0; i < PARAM_REGS; ++i)
4202                                 amd64_mov_reg_membase (code, param_regs [i], AMD64_R11, i * sizeof (gpointer), 8);
4203
4204                         /* Make the call */
4205                         amd64_call_reg (code, AMD64_R10);
4206
4207                         /* Save result */
4208                         amd64_mov_reg_membase (code, AMD64_R11, var->inst_basereg, var->inst_offset, 8);
4209                         amd64_mov_membase_reg (code, AMD64_R11, G_STRUCT_OFFSET (DynCallArgs, res), AMD64_RAX, 8);
4210                         break;
4211                 }
4212                 case OP_AMD64_SAVE_SP_TO_LMF:
4213                         amd64_mov_membase_reg (code, cfg->frame_reg, cfg->arch.lmf_offset + G_STRUCT_OFFSET (MonoLMF, rsp), AMD64_RSP, 8);
4214                         break;
4215                 case OP_X86_PUSH:
4216                         g_assert (!cfg->arch.no_pushes);
4217                         amd64_push_reg (code, ins->sreg1);
4218                         break;
4219                 case OP_X86_PUSH_IMM:
4220                         g_assert (!cfg->arch.no_pushes);
4221                         g_assert (amd64_is_imm32 (ins->inst_imm));
4222                         amd64_push_imm (code, ins->inst_imm);
4223                         break;
4224                 case OP_X86_PUSH_MEMBASE:
4225                         g_assert (!cfg->arch.no_pushes);
4226                         amd64_push_membase (code, ins->inst_basereg, ins->inst_offset);
4227                         break;
4228                 case OP_X86_PUSH_OBJ: {
4229                         int size = ALIGN_TO (ins->inst_imm, 8);
4230
4231                         g_assert (!cfg->arch.no_pushes);
4232
4233                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, size);
4234                         amd64_push_reg (code, AMD64_RDI);
4235                         amd64_push_reg (code, AMD64_RSI);
4236                         amd64_push_reg (code, AMD64_RCX);
4237                         if (ins->inst_offset)
4238                                 amd64_lea_membase (code, AMD64_RSI, ins->inst_basereg, ins->inst_offset);
4239                         else
4240                                 amd64_mov_reg_reg (code, AMD64_RSI, ins->inst_basereg, 8);
4241                         amd64_lea_membase (code, AMD64_RDI, AMD64_RSP, (3 * 8));
4242                         amd64_mov_reg_imm (code, AMD64_RCX, (size >> 3));
4243                         amd64_cld (code);
4244                         amd64_prefix (code, X86_REP_PREFIX);
4245                         amd64_movsd (code);
4246                         amd64_pop_reg (code, AMD64_RCX);
4247                         amd64_pop_reg (code, AMD64_RSI);
4248                         amd64_pop_reg (code, AMD64_RDI);
4249                         break;
4250                 }
4251                 case OP_X86_LEA:
4252                         amd64_lea_memindex (code, ins->dreg, ins->sreg1, ins->inst_imm, ins->sreg2, ins->backend.shift_amount);
4253                         break;
4254                 case OP_X86_LEA_MEMBASE:
4255                         amd64_lea_membase (code, ins->dreg, ins->sreg1, ins->inst_imm);
4256                         break;
4257                 case OP_X86_XCHG:
4258                         amd64_xchg_reg_reg (code, ins->sreg1, ins->sreg2, 4);
4259                         break;
4260                 case OP_LOCALLOC:
4261                         /* keep alignment */
4262                         amd64_alu_reg_imm (code, X86_ADD, ins->sreg1, MONO_ARCH_FRAME_ALIGNMENT - 1);
4263                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ~(MONO_ARCH_FRAME_ALIGNMENT - 1));
4264                         code = mono_emit_stack_alloc (cfg, code, ins);
4265                         amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
4266                         if (cfg->param_area && cfg->arch.no_pushes)
4267                                 amd64_alu_reg_imm (code, X86_ADD, ins->dreg, cfg->param_area);
4268                         break;
4269                 case OP_LOCALLOC_IMM: {
4270                         guint32 size = ins->inst_imm;
4271                         size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
4272
4273                         if (ins->flags & MONO_INST_INIT) {
4274                                 if (size < 64) {
4275                                         int i;
4276
4277                                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, size);
4278                                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
4279
4280                                         for (i = 0; i < size; i += 8)
4281                                                 amd64_mov_membase_reg (code, AMD64_RSP, i, ins->dreg, 8);
4282                                         amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
4283                                 } else {
4284                                         amd64_mov_reg_imm (code, ins->dreg, size);
4285                                         ins->sreg1 = ins->dreg;
4286
4287                                         code = mono_emit_stack_alloc (cfg, code, ins);
4288                                         amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
4289                                 }
4290                         } else {
4291                                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, size);
4292                                 amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
4293                         }
4294                         if (cfg->param_area && cfg->arch.no_pushes)
4295                                 amd64_alu_reg_imm (code, X86_ADD, ins->dreg, cfg->param_area);
4296                         break;
4297                 }
4298                 case OP_THROW: {
4299                         amd64_mov_reg_reg (code, AMD64_ARG_REG1, ins->sreg1, 8);
4300                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4301                                              (gpointer)"mono_arch_throw_exception", FALSE);
4302                         break;
4303                 }
4304                 case OP_RETHROW: {
4305                         amd64_mov_reg_reg (code, AMD64_ARG_REG1, ins->sreg1, 8);
4306                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4307                                              (gpointer)"mono_arch_rethrow_exception", FALSE);
4308                         break;
4309                 }
4310                 case OP_CALL_HANDLER:
4311                         /* Align stack */
4312                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
4313                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_BB, ins->inst_target_bb);
4314                         amd64_call_imm (code, 0);
4315                         /* Restore stack alignment */
4316                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
4317                         break;
4318                 case OP_START_HANDLER: {
4319                         MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4320                         amd64_mov_membase_reg (code, spvar->inst_basereg, spvar->inst_offset, AMD64_RSP, 8);
4321
4322                         if ((MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_FINALLY) ||
4323                                  MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_FINALLY)) &&
4324                                 cfg->param_area && cfg->arch.no_pushes) {
4325                                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, ALIGN_TO (cfg->param_area, MONO_ARCH_FRAME_ALIGNMENT));
4326                         }
4327                         break;
4328                 }
4329                 case OP_ENDFINALLY: {
4330                         MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4331                         amd64_mov_reg_membase (code, AMD64_RSP, spvar->inst_basereg, spvar->inst_offset, 8);
4332                         amd64_ret (code);
4333                         break;
4334                 }
4335                 case OP_ENDFILTER: {
4336                         MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4337                         amd64_mov_reg_membase (code, AMD64_RSP, spvar->inst_basereg, spvar->inst_offset, 8);
4338                         /* The local allocator will put the result into RAX */
4339                         amd64_ret (code);
4340                         break;
4341                 }
4342
4343                 case OP_LABEL:
4344                         ins->inst_c0 = code - cfg->native_code;
4345                         break;
4346                 case OP_BR:
4347                         //g_print ("target: %p, next: %p, curr: %p, last: %p\n", ins->inst_target_bb, bb->next_bb, ins, bb->last_ins);
4348                         //if ((ins->inst_target_bb == bb->next_bb) && ins == bb->last_ins)
4349                         //break;
4350                                 if (ins->inst_target_bb->native_offset) {
4351                                         amd64_jump_code (code, cfg->native_code + ins->inst_target_bb->native_offset);
4352                                 } else {
4353                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb);
4354                                         if ((cfg->opt & MONO_OPT_BRANCH) &&
4355                                             x86_is_imm8 (ins->inst_target_bb->max_offset - offset))
4356                                                 x86_jump8 (code, 0);
4357                                         else
4358                                                 x86_jump32 (code, 0);
4359                         }
4360                         break;
4361                 case OP_BR_REG:
4362                         amd64_jump_reg (code, ins->sreg1);
4363                         break;
4364                 case OP_CEQ:
4365                 case OP_LCEQ:
4366                 case OP_ICEQ:
4367                 case OP_CLT:
4368                 case OP_LCLT:
4369                 case OP_ICLT:
4370                 case OP_CGT:
4371                 case OP_ICGT:
4372                 case OP_LCGT:
4373                 case OP_CLT_UN:
4374                 case OP_LCLT_UN:
4375                 case OP_ICLT_UN:
4376                 case OP_CGT_UN:
4377                 case OP_LCGT_UN:
4378                 case OP_ICGT_UN:
4379                         amd64_set_reg (code, cc_table [mono_opcode_to_cond (ins->opcode)], ins->dreg, cc_signed_table [mono_opcode_to_cond (ins->opcode)]);
4380                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
4381                         break;
4382                 case OP_COND_EXC_EQ:
4383                 case OP_COND_EXC_NE_UN:
4384                 case OP_COND_EXC_LT:
4385                 case OP_COND_EXC_LT_UN:
4386                 case OP_COND_EXC_GT:
4387                 case OP_COND_EXC_GT_UN:
4388                 case OP_COND_EXC_GE:
4389                 case OP_COND_EXC_GE_UN:
4390                 case OP_COND_EXC_LE:
4391                 case OP_COND_EXC_LE_UN:
4392                 case OP_COND_EXC_IEQ:
4393                 case OP_COND_EXC_INE_UN:
4394                 case OP_COND_EXC_ILT:
4395                 case OP_COND_EXC_ILT_UN:
4396                 case OP_COND_EXC_IGT:
4397                 case OP_COND_EXC_IGT_UN:
4398                 case OP_COND_EXC_IGE:
4399                 case OP_COND_EXC_IGE_UN:
4400                 case OP_COND_EXC_ILE:
4401                 case OP_COND_EXC_ILE_UN:
4402                         EMIT_COND_SYSTEM_EXCEPTION (cc_table [mono_opcode_to_cond (ins->opcode)], cc_signed_table [mono_opcode_to_cond (ins->opcode)], ins->inst_p1);
4403                         break;
4404                 case OP_COND_EXC_OV:
4405                 case OP_COND_EXC_NO:
4406                 case OP_COND_EXC_C:
4407                 case OP_COND_EXC_NC:
4408                         EMIT_COND_SYSTEM_EXCEPTION (branch_cc_table [ins->opcode - OP_COND_EXC_EQ],
4409                                                     (ins->opcode < OP_COND_EXC_NE_UN), ins->inst_p1);
4410                         break;
4411                 case OP_COND_EXC_IOV:
4412                 case OP_COND_EXC_INO:
4413                 case OP_COND_EXC_IC:
4414                 case OP_COND_EXC_INC:
4415                         EMIT_COND_SYSTEM_EXCEPTION (branch_cc_table [ins->opcode - OP_COND_EXC_IEQ],
4416                                                     (ins->opcode < OP_COND_EXC_INE_UN), ins->inst_p1);
4417                         break;
4418
4419                 /* floating point opcodes */
4420                 case OP_R8CONST: {
4421                         double d = *(double *)ins->inst_p0;
4422
4423                         if ((d == 0.0) && (mono_signbit (d) == 0)) {
4424                                 amd64_sse_xorpd_reg_reg (code, ins->dreg, ins->dreg);
4425                         }
4426                         else {
4427                                 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, ins->inst_p0);
4428                                 amd64_sse_movsd_reg_membase (code, ins->dreg, AMD64_RIP, 0);
4429                         }
4430                         break;
4431                 }
4432                 case OP_R4CONST: {
4433                         float f = *(float *)ins->inst_p0;
4434
4435                         if ((f == 0.0) && (mono_signbit (f) == 0)) {
4436                                 amd64_sse_xorpd_reg_reg (code, ins->dreg, ins->dreg);
4437                         }
4438                         else {
4439                                 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R4, ins->inst_p0);
4440                                 amd64_sse_movss_reg_membase (code, ins->dreg, AMD64_RIP, 0);
4441                                 amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
4442                         }
4443                         break;
4444                 }
4445                 case OP_STORER8_MEMBASE_REG:
4446                         amd64_sse_movsd_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1);
4447                         break;
4448                 case OP_LOADR8_MEMBASE:
4449                         amd64_sse_movsd_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
4450                         break;
4451                 case OP_STORER4_MEMBASE_REG:
4452                         /* This requires a double->single conversion */
4453                         amd64_sse_cvtsd2ss_reg_reg (code, AMD64_XMM15, ins->sreg1);
4454                         amd64_sse_movss_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, AMD64_XMM15);
4455                         break;
4456                 case OP_LOADR4_MEMBASE:
4457                         amd64_sse_movss_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
4458                         amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
4459                         break;
4460                 case OP_ICONV_TO_R4: /* FIXME: change precision */
4461                 case OP_ICONV_TO_R8:
4462                         amd64_sse_cvtsi2sd_reg_reg_size (code, ins->dreg, ins->sreg1, 4);
4463                         break;
4464                 case OP_LCONV_TO_R4: /* FIXME: change precision */
4465                 case OP_LCONV_TO_R8:
4466                         amd64_sse_cvtsi2sd_reg_reg (code, ins->dreg, ins->sreg1);
4467                         break;
4468                 case OP_FCONV_TO_R4:
4469                         /* FIXME: nothing to do ?? */
4470                         break;
4471                 case OP_FCONV_TO_I1:
4472                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 1, TRUE);
4473                         break;
4474                 case OP_FCONV_TO_U1:
4475                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 1, FALSE);
4476                         break;
4477                 case OP_FCONV_TO_I2:
4478                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 2, TRUE);
4479                         break;
4480                 case OP_FCONV_TO_U2:
4481                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 2, FALSE);
4482                         break;
4483                 case OP_FCONV_TO_U4:
4484                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 4, FALSE);
4485                         break;
4486                 case OP_FCONV_TO_I4:
4487                 case OP_FCONV_TO_I:
4488                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 4, TRUE);
4489                         break;
4490                 case OP_FCONV_TO_I8:
4491                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 8, TRUE);
4492                         break;
4493                 case OP_LCONV_TO_R_UN: {
4494                         guint8 *br [2];
4495
4496                         /* Based on gcc code */
4497                         amd64_test_reg_reg (code, ins->sreg1, ins->sreg1);
4498                         br [0] = code; x86_branch8 (code, X86_CC_S, 0, TRUE);
4499
4500                         /* Positive case */
4501                         amd64_sse_cvtsi2sd_reg_reg (code, ins->dreg, ins->sreg1);
4502                         br [1] = code; x86_jump8 (code, 0);
4503                         amd64_patch (br [0], code);
4504
4505                         /* Negative case */
4506                         /* Save to the red zone */
4507                         amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RAX, 8);
4508                         amd64_mov_membase_reg (code, AMD64_RSP, -16, AMD64_RCX, 8);
4509                         amd64_mov_reg_reg (code, AMD64_RCX, ins->sreg1, 8);
4510                         amd64_mov_reg_reg (code, AMD64_RAX, ins->sreg1, 8);
4511                         amd64_alu_reg_imm (code, X86_AND, AMD64_RCX, 1);
4512                         amd64_shift_reg_imm (code, X86_SHR, AMD64_RAX, 1);
4513                         amd64_alu_reg_imm (code, X86_OR, AMD64_RAX, AMD64_RCX);
4514                         amd64_sse_cvtsi2sd_reg_reg (code, ins->dreg, AMD64_RAX);
4515                         amd64_sse_addsd_reg_reg (code, ins->dreg, ins->dreg);
4516                         /* Restore */
4517                         amd64_mov_reg_membase (code, AMD64_RCX, AMD64_RSP, -16, 8);
4518                         amd64_mov_reg_membase (code, AMD64_RAX, AMD64_RSP, -8, 8);
4519                         amd64_patch (br [1], code);
4520                         break;
4521                 }
4522                 case OP_LCONV_TO_OVF_U4:
4523                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, 0);
4524                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_LT, TRUE, "OverflowException");
4525                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, 8);
4526                         break;
4527                 case OP_LCONV_TO_OVF_I4_UN:
4528                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, 0x7fffffff);
4529                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_GT, FALSE, "OverflowException");
4530                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, 8);
4531                         break;
4532                 case OP_FMOVE:
4533                         if (ins->dreg != ins->sreg1)
4534                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
4535                         break;
4536                 case OP_FADD:
4537                         amd64_sse_addsd_reg_reg (code, ins->dreg, ins->sreg2);
4538                         break;
4539                 case OP_FSUB:
4540                         amd64_sse_subsd_reg_reg (code, ins->dreg, ins->sreg2);
4541                         break;
4542                 case OP_FMUL:
4543                         amd64_sse_mulsd_reg_reg (code, ins->dreg, ins->sreg2);
4544                         break;
4545                 case OP_FDIV:
4546                         amd64_sse_divsd_reg_reg (code, ins->dreg, ins->sreg2);
4547                         break;
4548                 case OP_FNEG: {
4549                         static double r8_0 = -0.0;
4550
4551                         g_assert (ins->sreg1 == ins->dreg);
4552
4553                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, &r8_0);
4554                         amd64_sse_xorpd_reg_membase (code, ins->dreg, AMD64_RIP, 0);
4555                         break;
4556                 }
4557                 case OP_SIN:
4558                         EMIT_SSE2_FPFUNC (code, fsin, ins->dreg, ins->sreg1);
4559                         break;
4560                 case OP_COS:
4561                         EMIT_SSE2_FPFUNC (code, fcos, ins->dreg, ins->sreg1);
4562                         break;
4563                 case OP_ABS: {
4564                         static guint64 d = 0x7fffffffffffffffUL;
4565
4566                         g_assert (ins->sreg1 == ins->dreg);
4567
4568                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, &d);
4569                         amd64_sse_andpd_reg_membase (code, ins->dreg, AMD64_RIP, 0);
4570                         break;
4571                 }
4572                 case OP_SQRT:
4573                         EMIT_SSE2_FPFUNC (code, fsqrt, ins->dreg, ins->sreg1);
4574                         break;
4575                 case OP_IMIN:
4576                         g_assert (cfg->opt & MONO_OPT_CMOV);
4577                         g_assert (ins->dreg == ins->sreg1);
4578                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
4579                         amd64_cmov_reg_size (code, X86_CC_GT, TRUE, ins->dreg, ins->sreg2, 4);
4580                         break;
4581                 case OP_IMIN_UN:
4582                         g_assert (cfg->opt & MONO_OPT_CMOV);
4583                         g_assert (ins->dreg == ins->sreg1);
4584                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
4585                         amd64_cmov_reg_size (code, X86_CC_GT, FALSE, ins->dreg, ins->sreg2, 4);
4586                         break;
4587                 case OP_IMAX:
4588                         g_assert (cfg->opt & MONO_OPT_CMOV);
4589                         g_assert (ins->dreg == ins->sreg1);
4590                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
4591                         amd64_cmov_reg_size (code, X86_CC_LT, TRUE, ins->dreg, ins->sreg2, 4);
4592                         break;
4593                 case OP_IMAX_UN:
4594                         g_assert (cfg->opt & MONO_OPT_CMOV);
4595                         g_assert (ins->dreg == ins->sreg1);
4596                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
4597                         amd64_cmov_reg_size (code, X86_CC_LT, FALSE, ins->dreg, ins->sreg2, 4);
4598                         break;
4599                 case OP_LMIN:
4600                         g_assert (cfg->opt & MONO_OPT_CMOV);
4601                         g_assert (ins->dreg == ins->sreg1);
4602                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
4603                         amd64_cmov_reg (code, X86_CC_GT, TRUE, ins->dreg, ins->sreg2);
4604                         break;
4605                 case OP_LMIN_UN:
4606                         g_assert (cfg->opt & MONO_OPT_CMOV);
4607                         g_assert (ins->dreg == ins->sreg1);
4608                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
4609                         amd64_cmov_reg (code, X86_CC_GT, FALSE, ins->dreg, ins->sreg2);
4610                         break;
4611                 case OP_LMAX:
4612                         g_assert (cfg->opt & MONO_OPT_CMOV);
4613                         g_assert (ins->dreg == ins->sreg1);
4614                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
4615                         amd64_cmov_reg (code, X86_CC_LT, TRUE, ins->dreg, ins->sreg2);
4616                         break;
4617                 case OP_LMAX_UN:
4618                         g_assert (cfg->opt & MONO_OPT_CMOV);
4619                         g_assert (ins->dreg == ins->sreg1);
4620                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
4621                         amd64_cmov_reg (code, X86_CC_LT, FALSE, ins->dreg, ins->sreg2);
4622                         break;
4623                 case OP_X86_FPOP:
4624                         break;
4625                 case OP_FCOMPARE:
4626                         /*
4627                          * The two arguments are swapped because the fbranch instructions
4628                          * depend on this for the non-sse case to work.
4629                          */
4630                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
4631                         break;
4632                 case OP_FCEQ: {
4633                         /* zeroing the register at the start results in
4634                          * shorter and faster code (we can also remove the widening op)
4635                          */
4636                         guchar *unordered_check;
4637                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
4638                         amd64_sse_comisd_reg_reg (code, ins->sreg1, ins->sreg2);
4639                         unordered_check = code;
4640                         x86_branch8 (code, X86_CC_P, 0, FALSE);
4641                         amd64_set_reg (code, X86_CC_EQ, ins->dreg, FALSE);
4642                         amd64_patch (unordered_check, code);
4643                         break;
4644                 }
4645                 case OP_FCLT:
4646                 case OP_FCLT_UN:
4647                         /* zeroing the register at the start results in
4648                          * shorter and faster code (we can also remove the widening op)
4649                          */
4650                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
4651                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
4652                         if (ins->opcode == OP_FCLT_UN) {
4653                                 guchar *unordered_check = code;
4654                                 guchar *jump_to_end;
4655                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
4656                                 amd64_set_reg (code, X86_CC_GT, ins->dreg, FALSE);
4657                                 jump_to_end = code;
4658                                 x86_jump8 (code, 0);
4659                                 amd64_patch (unordered_check, code);
4660                                 amd64_inc_reg (code, ins->dreg);
4661                                 amd64_patch (jump_to_end, code);
4662                         } else {
4663                                 amd64_set_reg (code, X86_CC_GT, ins->dreg, FALSE);
4664                         }
4665                         break;
4666                 case OP_FCGT:
4667                 case OP_FCGT_UN: {
4668                         /* zeroing the register at the start results in
4669                          * shorter and faster code (we can also remove the widening op)
4670                          */
4671                         guchar *unordered_check;
4672                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
4673                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
4674                         if (ins->opcode == OP_FCGT) {
4675                                 unordered_check = code;
4676                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
4677                                 amd64_set_reg (code, X86_CC_LT, ins->dreg, FALSE);
4678                                 amd64_patch (unordered_check, code);
4679                         } else {
4680                                 amd64_set_reg (code, X86_CC_LT, ins->dreg, FALSE);
4681                         }
4682                         break;
4683                 }
4684                 case OP_FCLT_MEMBASE:
4685                 case OP_FCGT_MEMBASE:
4686                 case OP_FCLT_UN_MEMBASE:
4687                 case OP_FCGT_UN_MEMBASE:
4688                 case OP_FCEQ_MEMBASE: {
4689                         guchar *unordered_check, *jump_to_end;
4690                         int x86_cond;
4691
4692                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
4693                         amd64_sse_comisd_reg_membase (code, ins->sreg1, ins->sreg2, ins->inst_offset);
4694
4695                         switch (ins->opcode) {
4696                         case OP_FCEQ_MEMBASE:
4697                                 x86_cond = X86_CC_EQ;
4698                                 break;
4699                         case OP_FCLT_MEMBASE:
4700                         case OP_FCLT_UN_MEMBASE:
4701                                 x86_cond = X86_CC_LT;
4702                                 break;
4703                         case OP_FCGT_MEMBASE:
4704                         case OP_FCGT_UN_MEMBASE:
4705                                 x86_cond = X86_CC_GT;
4706                                 break;
4707                         default:
4708                                 g_assert_not_reached ();
4709                         }
4710
4711                         unordered_check = code;
4712                         x86_branch8 (code, X86_CC_P, 0, FALSE);
4713                         amd64_set_reg (code, x86_cond, ins->dreg, FALSE);
4714
4715                         switch (ins->opcode) {
4716                         case OP_FCEQ_MEMBASE:
4717                         case OP_FCLT_MEMBASE:
4718                         case OP_FCGT_MEMBASE:
4719                                 amd64_patch (unordered_check, code);
4720                                 break;
4721                         case OP_FCLT_UN_MEMBASE:
4722                         case OP_FCGT_UN_MEMBASE:
4723                                 jump_to_end = code;
4724                                 x86_jump8 (code, 0);
4725                                 amd64_patch (unordered_check, code);
4726                                 amd64_inc_reg (code, ins->dreg);
4727                                 amd64_patch (jump_to_end, code);
4728                                 break;
4729                         default:
4730                                 break;
4731                         }
4732                         break;
4733                 }
4734                 case OP_FBEQ: {
4735                         guchar *jump = code;
4736                         x86_branch8 (code, X86_CC_P, 0, TRUE);
4737                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4738                         amd64_patch (jump, code);
4739                         break;
4740                 }
4741                 case OP_FBNE_UN:
4742                         /* Branch if C013 != 100 */
4743                         /* branch if !ZF or (PF|CF) */
4744                         EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
4745                         EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
4746                         EMIT_COND_BRANCH (ins, X86_CC_B, FALSE);
4747                         break;
4748                 case OP_FBLT:
4749                         EMIT_COND_BRANCH (ins, X86_CC_GT, FALSE);
4750                         break;
4751                 case OP_FBLT_UN:
4752                         EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
4753                         EMIT_COND_BRANCH (ins, X86_CC_GT, FALSE);
4754                         break;
4755                 case OP_FBGT:
4756                 case OP_FBGT_UN:
4757                         if (ins->opcode == OP_FBGT) {
4758                                 guchar *br1;
4759
4760                                 /* skip branch if C1=1 */
4761                                 br1 = code;
4762                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
4763                                 /* branch if (C0 | C3) = 1 */
4764                                 EMIT_COND_BRANCH (ins, X86_CC_LT, FALSE);
4765                                 amd64_patch (br1, code);
4766                                 break;
4767                         } else {
4768                                 EMIT_COND_BRANCH (ins, X86_CC_LT, FALSE);
4769                         }
4770                         break;
4771                 case OP_FBGE: {
4772                         /* Branch if C013 == 100 or 001 */
4773                         guchar *br1;
4774
4775                         /* skip branch if C1=1 */
4776                         br1 = code;
4777                         x86_branch8 (code, X86_CC_P, 0, FALSE);
4778                         /* branch if (C0 | C3) = 1 */
4779                         EMIT_COND_BRANCH (ins, X86_CC_BE, FALSE);
4780                         amd64_patch (br1, code);
4781                         break;
4782                 }
4783                 case OP_FBGE_UN:
4784                         /* Branch if C013 == 000 */
4785                         EMIT_COND_BRANCH (ins, X86_CC_LE, FALSE);
4786                         break;
4787                 case OP_FBLE: {
4788                         /* Branch if C013=000 or 100 */
4789                         guchar *br1;
4790
4791                         /* skip branch if C1=1 */
4792                         br1 = code;
4793                         x86_branch8 (code, X86_CC_P, 0, FALSE);
4794                         /* branch if C0=0 */
4795                         EMIT_COND_BRANCH (ins, X86_CC_NB, FALSE);
4796                         amd64_patch (br1, code);
4797                         break;
4798                 }
4799                 case OP_FBLE_UN:
4800                         /* Branch if C013 != 001 */
4801                         EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
4802                         EMIT_COND_BRANCH (ins, X86_CC_GE, FALSE);
4803                         break;
4804                 case OP_CKFINITE:
4805                         /* Transfer value to the fp stack */
4806                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 16);
4807                         amd64_movsd_membase_reg (code, AMD64_RSP, 0, ins->sreg1);
4808                         amd64_fld_membase (code, AMD64_RSP, 0, TRUE);
4809
4810                         amd64_push_reg (code, AMD64_RAX);
4811                         amd64_fxam (code);
4812                         amd64_fnstsw (code);
4813                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, 0x4100);
4814                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4815                         amd64_pop_reg (code, AMD64_RAX);
4816                         amd64_fstp (code, 0);
4817                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_EQ, FALSE, "ArithmeticException");
4818                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 16);
4819                         break;
4820                 case OP_TLS_GET: {
4821                         code = mono_amd64_emit_tls_get (code, ins->dreg, ins->inst_offset);
4822                         break;
4823                 }
4824                 case OP_MEMORY_BARRIER: {
4825                         /* Not needed on amd64 */
4826                         break;
4827                 }
4828                 case OP_ATOMIC_ADD_I4:
4829                 case OP_ATOMIC_ADD_I8: {
4830                         int dreg = ins->dreg;
4831                         guint32 size = (ins->opcode == OP_ATOMIC_ADD_I4) ? 4 : 8;
4832
4833                         if (dreg == ins->inst_basereg)
4834                                 dreg = AMD64_R11;
4835
4836                         if (dreg != ins->sreg2)
4837                                 amd64_mov_reg_reg (code, ins->dreg, ins->sreg2, size);
4838
4839                         x86_prefix (code, X86_LOCK_PREFIX);
4840                         amd64_xadd_membase_reg (code, ins->inst_basereg, ins->inst_offset, dreg, size);
4841
4842                         if (dreg != ins->dreg)
4843                                 amd64_mov_reg_reg (code, ins->dreg, dreg, size);
4844
4845                         break;
4846                 }
4847                 case OP_ATOMIC_ADD_NEW_I4:
4848                 case OP_ATOMIC_ADD_NEW_I8: {
4849                         int dreg = ins->dreg;
4850                         guint32 size = (ins->opcode == OP_ATOMIC_ADD_NEW_I4) ? 4 : 8;
4851
4852                         if ((dreg == ins->sreg2) || (dreg == ins->inst_basereg))
4853                                 dreg = AMD64_R11;
4854
4855                         amd64_mov_reg_reg (code, dreg, ins->sreg2, size);
4856                         amd64_prefix (code, X86_LOCK_PREFIX);
4857                         amd64_xadd_membase_reg (code, ins->inst_basereg, ins->inst_offset, dreg, size);
4858                         /* dreg contains the old value, add with sreg2 value */
4859                         amd64_alu_reg_reg_size (code, X86_ADD, dreg, ins->sreg2, size);
4860
4861                         if (ins->dreg != dreg)
4862                                 amd64_mov_reg_reg (code, ins->dreg, dreg, size);
4863
4864                         break;
4865                 }
4866                 case OP_ATOMIC_EXCHANGE_I4:
4867                 case OP_ATOMIC_EXCHANGE_I8: {
4868                         guchar *br[2];
4869                         int sreg2 = ins->sreg2;
4870                         int breg = ins->inst_basereg;
4871                         guint32 size;
4872                         gboolean need_push = FALSE, rdx_pushed = FALSE;
4873
4874                         if (ins->opcode == OP_ATOMIC_EXCHANGE_I8)
4875                                 size = 8;
4876                         else
4877                                 size = 4;
4878
4879                         /*
4880                          * See http://msdn.microsoft.com/en-us/magazine/cc302329.aspx for
4881                          * an explanation of how this works.
4882                          */
4883
4884                         /* cmpxchg uses eax as comperand, need to make sure we can use it
4885                          * hack to overcome limits in x86 reg allocator
4886                          * (req: dreg == eax and sreg2 != eax and breg != eax)
4887                          */
4888                         g_assert (ins->dreg == AMD64_RAX);
4889
4890                         if (breg == AMD64_RAX && ins->sreg2 == AMD64_RAX)
4891                                 /* Highly unlikely, but possible */
4892                                 need_push = TRUE;
4893
4894                         /* The pushes invalidate rsp */
4895                         if ((breg == AMD64_RAX) || need_push) {
4896                                 amd64_mov_reg_reg (code, AMD64_R11, breg, 8);
4897                                 breg = AMD64_R11;
4898                         }
4899
4900                         /* We need the EAX reg for the comparand */
4901                         if (ins->sreg2 == AMD64_RAX) {
4902                                 if (breg != AMD64_R11) {
4903                                         amd64_mov_reg_reg (code, AMD64_R11, AMD64_RAX, 8);
4904                                         sreg2 = AMD64_R11;
4905                                 } else {
4906                                         g_assert (need_push);
4907                                         amd64_push_reg (code, AMD64_RDX);
4908                                         amd64_mov_reg_reg (code, AMD64_RDX, AMD64_RAX, size);
4909                                         sreg2 = AMD64_RDX;
4910                                         rdx_pushed = TRUE;
4911                                 }
4912                         }
4913
4914                         amd64_mov_reg_membase (code, AMD64_RAX, breg, ins->inst_offset, size);
4915
4916                         br [0] = code; amd64_prefix (code, X86_LOCK_PREFIX);
4917                         amd64_cmpxchg_membase_reg_size (code, breg, ins->inst_offset, sreg2, size);
4918                         br [1] = code; amd64_branch8 (code, X86_CC_NE, -1, FALSE);
4919                         amd64_patch (br [1], br [0]);
4920
4921                         if (rdx_pushed)
4922                                 amd64_pop_reg (code, AMD64_RDX);
4923
4924                         break;
4925                 }
4926                 case OP_ATOMIC_CAS_I4:
4927                 case OP_ATOMIC_CAS_I8: {
4928                         guint32 size;
4929
4930                         if (ins->opcode == OP_ATOMIC_CAS_I8)
4931                                 size = 8;
4932                         else
4933                                 size = 4;
4934
4935                         /*
4936                          * See http://msdn.microsoft.com/en-us/magazine/cc302329.aspx for
4937                          * an explanation of how this works.
4938                          */
4939                         g_assert (ins->sreg3 == AMD64_RAX);
4940                         g_assert (ins->sreg1 != AMD64_RAX);
4941                         g_assert (ins->sreg1 != ins->sreg2);
4942
4943                         amd64_prefix (code, X86_LOCK_PREFIX);
4944                         amd64_cmpxchg_membase_reg_size (code, ins->sreg1, ins->inst_offset, ins->sreg2, size);
4945
4946                         if (ins->dreg != AMD64_RAX)
4947                                 amd64_mov_reg_reg (code, ins->dreg, AMD64_RAX, size);
4948                         break;
4949                 }
4950 #ifdef MONO_ARCH_SIMD_INTRINSICS
4951                 /* TODO: Some of these IR opcodes are marked as no clobber when they indeed do. */
4952                 case OP_ADDPS:
4953                         amd64_sse_addps_reg_reg (code, ins->sreg1, ins->sreg2);
4954                         break;
4955                 case OP_DIVPS:
4956                         amd64_sse_divps_reg_reg (code, ins->sreg1, ins->sreg2);
4957                         break;
4958                 case OP_MULPS:
4959                         amd64_sse_mulps_reg_reg (code, ins->sreg1, ins->sreg2);
4960                         break;
4961                 case OP_SUBPS:
4962                         amd64_sse_subps_reg_reg (code, ins->sreg1, ins->sreg2);
4963                         break;
4964                 case OP_MAXPS:
4965                         amd64_sse_maxps_reg_reg (code, ins->sreg1, ins->sreg2);
4966                         break;
4967                 case OP_MINPS:
4968                         amd64_sse_minps_reg_reg (code, ins->sreg1, ins->sreg2);
4969                         break;
4970                 case OP_COMPPS:
4971                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 7);
4972                         amd64_sse_cmpps_reg_reg_imm (code, ins->sreg1, ins->sreg2, ins->inst_c0);
4973                         break;
4974                 case OP_ANDPS:
4975                         amd64_sse_andps_reg_reg (code, ins->sreg1, ins->sreg2);
4976                         break;
4977                 case OP_ANDNPS:
4978                         amd64_sse_andnps_reg_reg (code, ins->sreg1, ins->sreg2);
4979                         break;
4980                 case OP_ORPS:
4981                         amd64_sse_orps_reg_reg (code, ins->sreg1, ins->sreg2);
4982                         break;
4983                 case OP_XORPS:
4984                         amd64_sse_xorps_reg_reg (code, ins->sreg1, ins->sreg2);
4985                         break;
4986                 case OP_SQRTPS:
4987                         amd64_sse_sqrtps_reg_reg (code, ins->dreg, ins->sreg1);
4988                         break;
4989                 case OP_RSQRTPS:
4990                         amd64_sse_rsqrtps_reg_reg (code, ins->dreg, ins->sreg1);
4991                         break;
4992                 case OP_RCPPS:
4993                         amd64_sse_rcpps_reg_reg (code, ins->dreg, ins->sreg1);
4994                         break;
4995                 case OP_ADDSUBPS:
4996                         amd64_sse_addsubps_reg_reg (code, ins->sreg1, ins->sreg2);
4997                         break;
4998                 case OP_HADDPS:
4999                         amd64_sse_haddps_reg_reg (code, ins->sreg1, ins->sreg2);
5000                         break;
5001                 case OP_HSUBPS:
5002                         amd64_sse_hsubps_reg_reg (code, ins->sreg1, ins->sreg2);
5003                         break;
5004                 case OP_DUPPS_HIGH:
5005                         amd64_sse_movshdup_reg_reg (code, ins->dreg, ins->sreg1);
5006                         break;
5007                 case OP_DUPPS_LOW:
5008                         amd64_sse_movsldup_reg_reg (code, ins->dreg, ins->sreg1);
5009                         break;
5010
5011                 case OP_PSHUFLEW_HIGH:
5012                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 0xFF);
5013                         amd64_sse_pshufhw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
5014                         break;
5015                 case OP_PSHUFLEW_LOW:
5016                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 0xFF);
5017                         amd64_sse_pshuflw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
5018                         break;
5019                 case OP_PSHUFLED:
5020                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 0xFF);
5021                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
5022                         break;
5023
5024                 case OP_ADDPD:
5025                         amd64_sse_addpd_reg_reg (code, ins->sreg1, ins->sreg2);
5026                         break;
5027                 case OP_DIVPD:
5028                         amd64_sse_divpd_reg_reg (code, ins->sreg1, ins->sreg2);
5029                         break;
5030                 case OP_MULPD:
5031                         amd64_sse_mulpd_reg_reg (code, ins->sreg1, ins->sreg2);
5032                         break;
5033                 case OP_SUBPD:
5034                         amd64_sse_subpd_reg_reg (code, ins->sreg1, ins->sreg2);
5035                         break;
5036                 case OP_MAXPD:
5037                         amd64_sse_maxpd_reg_reg (code, ins->sreg1, ins->sreg2);
5038                         break;
5039                 case OP_MINPD:
5040                         amd64_sse_minpd_reg_reg (code, ins->sreg1, ins->sreg2);
5041                         break;
5042                 case OP_COMPPD:
5043                         g_assert (ins->inst_c0 >= 0 && ins->inst_c0 <= 7);
5044                         amd64_sse_cmppd_reg_reg_imm (code, ins->sreg1, ins->sreg2, ins->inst_c0);
5045                         break;
5046                 case OP_ANDPD:
5047                         amd64_sse_andpd_reg_reg (code, ins->sreg1, ins->sreg2);
5048                         break;
5049                 case OP_ANDNPD:
5050                         amd64_sse_andnpd_reg_reg (code, ins->sreg1, ins->sreg2);
5051                         break;
5052                 case OP_ORPD:
5053                         amd64_sse_orpd_reg_reg (code, ins->sreg1, ins->sreg2);
5054                         break;
5055                 case OP_XORPD:
5056                         amd64_sse_xorpd_reg_reg (code, ins->sreg1, ins->sreg2);
5057                         break;
5058                 case OP_SQRTPD:
5059                         amd64_sse_sqrtpd_reg_reg (code, ins->dreg, ins->sreg1);
5060                         break;
5061                 case OP_ADDSUBPD:
5062                         amd64_sse_addsubpd_reg_reg (code, ins->sreg1, ins->sreg2);
5063                         break;
5064                 case OP_HADDPD:
5065                         amd64_sse_haddpd_reg_reg (code, ins->sreg1, ins->sreg2);
5066                         break;
5067                 case OP_HSUBPD:
5068                         amd64_sse_hsubpd_reg_reg (code, ins->sreg1, ins->sreg2);
5069                         break;
5070                 case OP_DUPPD:
5071                         amd64_sse_movddup_reg_reg (code, ins->dreg, ins->sreg1);
5072                         break;
5073
5074                 case OP_EXTRACT_MASK:
5075                         amd64_sse_pmovmskb_reg_reg (code, ins->dreg, ins->sreg1);
5076                         break;
5077
5078                 case OP_PAND:
5079                         amd64_sse_pand_reg_reg (code, ins->sreg1, ins->sreg2);
5080                         break;
5081                 case OP_POR:
5082                         amd64_sse_por_reg_reg (code, ins->sreg1, ins->sreg2);
5083                         break;
5084                 case OP_PXOR:
5085                         amd64_sse_pxor_reg_reg (code, ins->sreg1, ins->sreg2);
5086                         break;
5087
5088                 case OP_PADDB:
5089                         amd64_sse_paddb_reg_reg (code, ins->sreg1, ins->sreg2);
5090                         break;
5091                 case OP_PADDW:
5092                         amd64_sse_paddw_reg_reg (code, ins->sreg1, ins->sreg2);
5093                         break;
5094                 case OP_PADDD:
5095                         amd64_sse_paddd_reg_reg (code, ins->sreg1, ins->sreg2);
5096                         break;
5097                 case OP_PADDQ:
5098                         amd64_sse_paddq_reg_reg (code, ins->sreg1, ins->sreg2);
5099                         break;
5100
5101                 case OP_PSUBB:
5102                         amd64_sse_psubb_reg_reg (code, ins->sreg1, ins->sreg2);
5103                         break;
5104                 case OP_PSUBW:
5105                         amd64_sse_psubw_reg_reg (code, ins->sreg1, ins->sreg2);
5106                         break;
5107                 case OP_PSUBD:
5108                         amd64_sse_psubd_reg_reg (code, ins->sreg1, ins->sreg2);
5109                         break;
5110                 case OP_PSUBQ:
5111                         amd64_sse_psubq_reg_reg (code, ins->sreg1, ins->sreg2);
5112                         break;
5113
5114                 case OP_PMAXB_UN:
5115                         amd64_sse_pmaxub_reg_reg (code, ins->sreg1, ins->sreg2);
5116                         break;
5117                 case OP_PMAXW_UN:
5118                         amd64_sse_pmaxuw_reg_reg (code, ins->sreg1, ins->sreg2);
5119                         break;
5120                 case OP_PMAXD_UN:
5121                         amd64_sse_pmaxud_reg_reg (code, ins->sreg1, ins->sreg2);
5122                         break;
5123
5124                 case OP_PMAXB:
5125                         amd64_sse_pmaxsb_reg_reg (code, ins->sreg1, ins->sreg2);
5126                         break;
5127                 case OP_PMAXW:
5128                         amd64_sse_pmaxsw_reg_reg (code, ins->sreg1, ins->sreg2);
5129                         break;
5130                 case OP_PMAXD:
5131                         amd64_sse_pmaxsd_reg_reg (code, ins->sreg1, ins->sreg2);
5132                         break;
5133
5134                 case OP_PAVGB_UN:
5135                         amd64_sse_pavgb_reg_reg (code, ins->sreg1, ins->sreg2);
5136                         break;
5137                 case OP_PAVGW_UN:
5138                         amd64_sse_pavgw_reg_reg (code, ins->sreg1, ins->sreg2);
5139                         break;
5140
5141                 case OP_PMINB_UN:
5142                         amd64_sse_pminub_reg_reg (code, ins->sreg1, ins->sreg2);
5143                         break;
5144                 case OP_PMINW_UN:
5145                         amd64_sse_pminuw_reg_reg (code, ins->sreg1, ins->sreg2);
5146                         break;
5147                 case OP_PMIND_UN:
5148                         amd64_sse_pminud_reg_reg (code, ins->sreg1, ins->sreg2);
5149                         break;
5150
5151                 case OP_PMINB:
5152                         amd64_sse_pminsb_reg_reg (code, ins->sreg1, ins->sreg2);
5153                         break;
5154                 case OP_PMINW:
5155                         amd64_sse_pminsw_reg_reg (code, ins->sreg1, ins->sreg2);
5156                         break;
5157                 case OP_PMIND:
5158                         amd64_sse_pminsd_reg_reg (code, ins->sreg1, ins->sreg2);
5159                         break;
5160
5161                 case OP_PCMPEQB:
5162                         amd64_sse_pcmpeqb_reg_reg (code, ins->sreg1, ins->sreg2);
5163                         break;
5164                 case OP_PCMPEQW:
5165                         amd64_sse_pcmpeqw_reg_reg (code, ins->sreg1, ins->sreg2);
5166                         break;
5167                 case OP_PCMPEQD:
5168                         amd64_sse_pcmpeqd_reg_reg (code, ins->sreg1, ins->sreg2);
5169                         break;
5170                 case OP_PCMPEQQ:
5171                         amd64_sse_pcmpeqq_reg_reg (code, ins->sreg1, ins->sreg2);
5172                         break;
5173
5174                 case OP_PCMPGTB:
5175                         amd64_sse_pcmpgtb_reg_reg (code, ins->sreg1, ins->sreg2);
5176                         break;
5177                 case OP_PCMPGTW:
5178                         amd64_sse_pcmpgtw_reg_reg (code, ins->sreg1, ins->sreg2);
5179                         break;
5180                 case OP_PCMPGTD:
5181                         amd64_sse_pcmpgtd_reg_reg (code, ins->sreg1, ins->sreg2);
5182                         break;
5183                 case OP_PCMPGTQ:
5184                         amd64_sse_pcmpgtq_reg_reg (code, ins->sreg1, ins->sreg2);
5185                         break;
5186
5187                 case OP_PSUM_ABS_DIFF:
5188                         amd64_sse_psadbw_reg_reg (code, ins->sreg1, ins->sreg2);
5189                         break;
5190
5191                 case OP_UNPACK_LOWB:
5192                         amd64_sse_punpcklbw_reg_reg (code, ins->sreg1, ins->sreg2);
5193                         break;
5194                 case OP_UNPACK_LOWW:
5195                         amd64_sse_punpcklwd_reg_reg (code, ins->sreg1, ins->sreg2);
5196                         break;
5197                 case OP_UNPACK_LOWD:
5198                         amd64_sse_punpckldq_reg_reg (code, ins->sreg1, ins->sreg2);
5199                         break;
5200                 case OP_UNPACK_LOWQ:
5201                         amd64_sse_punpcklqdq_reg_reg (code, ins->sreg1, ins->sreg2);
5202                         break;
5203                 case OP_UNPACK_LOWPS:
5204                         amd64_sse_unpcklps_reg_reg (code, ins->sreg1, ins->sreg2);
5205                         break;
5206                 case OP_UNPACK_LOWPD:
5207                         amd64_sse_unpcklpd_reg_reg (code, ins->sreg1, ins->sreg2);
5208                         break;
5209
5210                 case OP_UNPACK_HIGHB:
5211                         amd64_sse_punpckhbw_reg_reg (code, ins->sreg1, ins->sreg2);
5212                         break;
5213                 case OP_UNPACK_HIGHW:
5214                         amd64_sse_punpckhwd_reg_reg (code, ins->sreg1, ins->sreg2);
5215                         break;
5216                 case OP_UNPACK_HIGHD:
5217                         amd64_sse_punpckhdq_reg_reg (code, ins->sreg1, ins->sreg2);
5218                         break;
5219                 case OP_UNPACK_HIGHQ:
5220                         amd64_sse_punpckhqdq_reg_reg (code, ins->sreg1, ins->sreg2);
5221                         break;
5222                 case OP_UNPACK_HIGHPS:
5223                         amd64_sse_unpckhps_reg_reg (code, ins->sreg1, ins->sreg2);
5224                         break;
5225                 case OP_UNPACK_HIGHPD:
5226                         amd64_sse_unpckhpd_reg_reg (code, ins->sreg1, ins->sreg2);
5227                         break;
5228
5229                 case OP_PACKW:
5230                         amd64_sse_packsswb_reg_reg (code, ins->sreg1, ins->sreg2);
5231                         break;
5232                 case OP_PACKD:
5233                         amd64_sse_packssdw_reg_reg (code, ins->sreg1, ins->sreg2);
5234                         break;
5235                 case OP_PACKW_UN:
5236                         amd64_sse_packuswb_reg_reg (code, ins->sreg1, ins->sreg2);
5237                         break;
5238                 case OP_PACKD_UN:
5239                         amd64_sse_packusdw_reg_reg (code, ins->sreg1, ins->sreg2);
5240                         break;
5241
5242                 case OP_PADDB_SAT_UN:
5243                         amd64_sse_paddusb_reg_reg (code, ins->sreg1, ins->sreg2);
5244                         break;
5245                 case OP_PSUBB_SAT_UN:
5246                         amd64_sse_psubusb_reg_reg (code, ins->sreg1, ins->sreg2);
5247                         break;
5248                 case OP_PADDW_SAT_UN:
5249                         amd64_sse_paddusw_reg_reg (code, ins->sreg1, ins->sreg2);
5250                         break;
5251                 case OP_PSUBW_SAT_UN:
5252                         amd64_sse_psubusw_reg_reg (code, ins->sreg1, ins->sreg2);
5253                         break;
5254
5255                 case OP_PADDB_SAT:
5256                         amd64_sse_paddsb_reg_reg (code, ins->sreg1, ins->sreg2);
5257                         break;
5258                 case OP_PSUBB_SAT:
5259                         amd64_sse_psubsb_reg_reg (code, ins->sreg1, ins->sreg2);
5260                         break;
5261                 case OP_PADDW_SAT:
5262                         amd64_sse_paddsw_reg_reg (code, ins->sreg1, ins->sreg2);
5263                         break;
5264                 case OP_PSUBW_SAT:
5265                         amd64_sse_psubsw_reg_reg (code, ins->sreg1, ins->sreg2);
5266                         break;
5267
5268                 case OP_PMULW:
5269                         amd64_sse_pmullw_reg_reg (code, ins->sreg1, ins->sreg2);
5270                         break;
5271                 case OP_PMULD:
5272                         amd64_sse_pmulld_reg_reg (code, ins->sreg1, ins->sreg2);
5273                         break;
5274                 case OP_PMULQ:
5275                         amd64_sse_pmuludq_reg_reg (code, ins->sreg1, ins->sreg2);
5276                         break;
5277                 case OP_PMULW_HIGH_UN:
5278                         amd64_sse_pmulhuw_reg_reg (code, ins->sreg1, ins->sreg2);
5279                         break;
5280                 case OP_PMULW_HIGH:
5281                         amd64_sse_pmulhw_reg_reg (code, ins->sreg1, ins->sreg2);
5282                         break;
5283
5284                 case OP_PSHRW:
5285                         amd64_sse_psrlw_reg_imm (code, ins->dreg, ins->inst_imm);
5286                         break;
5287                 case OP_PSHRW_REG:
5288                         amd64_sse_psrlw_reg_reg (code, ins->dreg, ins->sreg2);
5289                         break;
5290
5291                 case OP_PSARW:
5292                         amd64_sse_psraw_reg_imm (code, ins->dreg, ins->inst_imm);
5293                         break;
5294                 case OP_PSARW_REG:
5295                         amd64_sse_psraw_reg_reg (code, ins->dreg, ins->sreg2);
5296                         break;
5297
5298                 case OP_PSHLW:
5299                         amd64_sse_psllw_reg_imm (code, ins->dreg, ins->inst_imm);
5300                         break;
5301                 case OP_PSHLW_REG:
5302                         amd64_sse_psllw_reg_reg (code, ins->dreg, ins->sreg2);
5303                         break;
5304
5305                 case OP_PSHRD:
5306                         amd64_sse_psrld_reg_imm (code, ins->dreg, ins->inst_imm);
5307                         break;
5308                 case OP_PSHRD_REG:
5309                         amd64_sse_psrld_reg_reg (code, ins->dreg, ins->sreg2);
5310                         break;
5311
5312                 case OP_PSARD:
5313                         amd64_sse_psrad_reg_imm (code, ins->dreg, ins->inst_imm);
5314                         break;
5315                 case OP_PSARD_REG:
5316                         amd64_sse_psrad_reg_reg (code, ins->dreg, ins->sreg2);
5317                         break;
5318
5319                 case OP_PSHLD:
5320                         amd64_sse_pslld_reg_imm (code, ins->dreg, ins->inst_imm);
5321                         break;
5322                 case OP_PSHLD_REG:
5323                         amd64_sse_pslld_reg_reg (code, ins->dreg, ins->sreg2);
5324                         break;
5325
5326                 case OP_PSHRQ:
5327                         amd64_sse_psrlq_reg_imm (code, ins->dreg, ins->inst_imm);
5328                         break;
5329                 case OP_PSHRQ_REG:
5330                         amd64_sse_psrlq_reg_reg (code, ins->dreg, ins->sreg2);
5331                         break;
5332
5333                 /*TODO: This is appart of the sse spec but not added
5334                 case OP_PSARQ:
5335                         amd64_sse_psraq_reg_imm (code, ins->dreg, ins->inst_imm);
5336                         break;
5337                 case OP_PSARQ_REG:
5338                         amd64_sse_psraq_reg_reg (code, ins->dreg, ins->sreg2);
5339                         break;
5340                 */
5341
5342                 case OP_PSHLQ:
5343                         amd64_sse_psllq_reg_imm (code, ins->dreg, ins->inst_imm);
5344                         break;
5345                 case OP_PSHLQ_REG:
5346                         amd64_sse_psllq_reg_reg (code, ins->dreg, ins->sreg2);
5347                         break;
5348
5349                 case OP_ICONV_TO_X:
5350                         amd64_movd_xreg_reg_size (code, ins->dreg, ins->sreg1, 4);
5351                         break;
5352                 case OP_EXTRACT_I4:
5353                         amd64_movd_reg_xreg_size (code, ins->dreg, ins->sreg1, 4);
5354                         break;
5355                 case OP_EXTRACT_I8:
5356                         if (ins->inst_c0) {
5357                                 amd64_movhlps_reg_reg (code, AMD64_XMM15, ins->sreg1);
5358                                 amd64_movd_reg_xreg_size (code, ins->dreg, AMD64_XMM15, 8);
5359                         } else {
5360                                 amd64_movd_reg_xreg_size (code, ins->dreg, ins->sreg1, 8);
5361                         }
5362                         break;
5363                 case OP_EXTRACT_I1:
5364                 case OP_EXTRACT_U1:
5365                         amd64_movd_reg_xreg_size (code, ins->dreg, ins->sreg1, 4);
5366                         if (ins->inst_c0)
5367                                 amd64_shift_reg_imm (code, X86_SHR, ins->dreg, ins->inst_c0 * 8);
5368                         amd64_widen_reg (code, ins->dreg, ins->dreg, ins->opcode == OP_EXTRACT_I1, FALSE);
5369                         break;
5370                 case OP_EXTRACT_I2:
5371                 case OP_EXTRACT_U2:
5372                         /*amd64_movd_reg_xreg_size (code, ins->dreg, ins->sreg1, 4);
5373                         if (ins->inst_c0)
5374                                 amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, 16, 4);*/
5375                         amd64_sse_pextrw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
5376                         amd64_widen_reg_size (code, ins->dreg, ins->dreg, ins->opcode == OP_EXTRACT_I2, TRUE, 4);
5377                         break;
5378                 case OP_EXTRACT_R8:
5379                         if (ins->inst_c0)
5380                                 amd64_movhlps_reg_reg (code, ins->dreg, ins->sreg1);
5381                         else
5382                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
5383                         break;
5384                 case OP_INSERT_I2:
5385                         amd64_sse_pinsrw_reg_reg_imm (code, ins->sreg1, ins->sreg2, ins->inst_c0);
5386                         break;
5387                 case OP_EXTRACTX_U2:
5388                         amd64_sse_pextrw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0);
5389                         break;
5390                 case OP_INSERTX_U1_SLOW:
5391                         /*sreg1 is the extracted ireg (scratch)
5392                         /sreg2 is the to be inserted ireg (scratch)
5393                         /dreg is the xreg to receive the value*/
5394
5395                         /*clear the bits from the extracted word*/
5396                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ins->inst_c0 & 1 ? 0x00FF : 0xFF00);
5397                         /*shift the value to insert if needed*/
5398                         if (ins->inst_c0 & 1)
5399                                 amd64_shift_reg_imm_size (code, X86_SHL, ins->sreg2, 8, 4);
5400                         /*join them together*/
5401                         amd64_alu_reg_reg (code, X86_OR, ins->sreg1, ins->sreg2);
5402                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_c0 / 2);
5403                         break;
5404                 case OP_INSERTX_I4_SLOW:
5405                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg2, ins->inst_c0 * 2);
5406                         amd64_shift_reg_imm (code, X86_SHR, ins->sreg2, 16);
5407                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg2, ins->inst_c0 * 2 + 1);
5408                         break;
5409                 case OP_INSERTX_I8_SLOW:
5410                         amd64_movd_xreg_reg_size(code, AMD64_XMM15, ins->sreg2, 8);
5411                         if (ins->inst_c0)
5412                                 amd64_movlhps_reg_reg (code, ins->dreg, AMD64_XMM15);
5413                         else
5414                                 amd64_sse_movsd_reg_reg (code, ins->dreg, AMD64_XMM15);
5415                         break;
5416
5417                 case OP_INSERTX_R4_SLOW:
5418                         switch (ins->inst_c0) {
5419                         case 0:
5420                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg2);
5421                                 break;
5422                         case 1:
5423                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(1, 0, 2, 3));
5424                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg2);
5425                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(1, 0, 2, 3));
5426                                 break;
5427                         case 2:
5428                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(2, 1, 0, 3));
5429                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg2);
5430                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(2, 1, 0, 3));
5431                                 break;
5432                         case 3:
5433                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(3, 1, 2, 0));
5434                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg2);
5435                                 amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, mono_simd_shuffle_mask(3, 1, 2, 0));
5436                                 break;
5437                         }
5438                         break;
5439                 case OP_INSERTX_R8_SLOW:
5440                         if (ins->inst_c0)
5441                                 amd64_movlhps_reg_reg (code, ins->dreg, ins->sreg2);
5442                         else
5443                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg2);
5444                         break;
5445                 case OP_STOREX_MEMBASE_REG:
5446                 case OP_STOREX_MEMBASE:
5447                         amd64_sse_movups_membase_reg (code, ins->dreg, ins->inst_offset, ins->sreg1);
5448                         break;
5449                 case OP_LOADX_MEMBASE:
5450                         amd64_sse_movups_reg_membase (code, ins->dreg, ins->sreg1, ins->inst_offset);
5451                         break;
5452                 case OP_LOADX_ALIGNED_MEMBASE:
5453                         amd64_sse_movaps_reg_membase (code, ins->dreg, ins->sreg1, ins->inst_offset);
5454                         break;
5455                 case OP_STOREX_ALIGNED_MEMBASE_REG:
5456                         amd64_sse_movaps_membase_reg (code, ins->dreg, ins->inst_offset, ins->sreg1);
5457                         break;
5458                 case OP_STOREX_NTA_MEMBASE_REG:
5459                         amd64_sse_movntps_reg_membase (code, ins->dreg, ins->sreg1, ins->inst_offset);
5460                         break;
5461                 case OP_PREFETCH_MEMBASE:
5462                         amd64_sse_prefetch_reg_membase (code, ins->backend.arg_info, ins->sreg1, ins->inst_offset);
5463                         break;
5464
5465                 case OP_XMOVE:
5466                         /*FIXME the peephole pass should have killed this*/
5467                         if (ins->dreg != ins->sreg1)
5468                                 amd64_sse_movaps_reg_reg (code, ins->dreg, ins->sreg1);
5469                         break;
5470                 case OP_XZERO:
5471                         amd64_sse_pxor_reg_reg (code, ins->dreg, ins->dreg);
5472                         break;
5473                 case OP_ICONV_TO_R8_RAW:
5474                         amd64_movd_xreg_reg_size (code, ins->dreg, ins->sreg1, 4);
5475                         amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
5476                         break;
5477
5478                 case OP_FCONV_TO_R8_X:
5479                         amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
5480                         break;
5481
5482                 case OP_XCONV_R8_TO_I4:
5483                         amd64_sse_cvttsd2si_reg_xreg_size (code, ins->dreg, ins->sreg1, 4);
5484                         switch (ins->backend.source_opcode) {
5485                         case OP_FCONV_TO_I1:
5486                                 amd64_widen_reg (code, ins->dreg, ins->dreg, TRUE, FALSE);
5487                                 break;
5488                         case OP_FCONV_TO_U1:
5489                                 amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
5490                                 break;
5491                         case OP_FCONV_TO_I2:
5492                                 amd64_widen_reg (code, ins->dreg, ins->dreg, TRUE, TRUE);
5493                                 break;
5494                         case OP_FCONV_TO_U2:
5495                                 amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, TRUE);
5496                                 break;
5497                         }
5498                         break;
5499
5500                 case OP_EXPAND_I2:
5501                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg1, 0);
5502                         amd64_sse_pinsrw_reg_reg_imm (code, ins->dreg, ins->sreg1, 1);
5503                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0);
5504                         break;
5505                 case OP_EXPAND_I4:
5506                         amd64_movd_xreg_reg_size (code, ins->dreg, ins->sreg1, 4);
5507                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0);
5508                         break;
5509                 case OP_EXPAND_I8:
5510                         amd64_movd_xreg_reg_size (code, ins->dreg, ins->sreg1, 8);
5511                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0x44);
5512                         break;
5513                 case OP_EXPAND_R4:
5514                         amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
5515                         amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->dreg);
5516                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0);
5517                         break;
5518                 case OP_EXPAND_R8:
5519                         amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
5520                         amd64_sse_pshufd_reg_reg_imm (code, ins->dreg, ins->dreg, 0x44);
5521                         break;
5522 #endif
5523                 case OP_LIVERANGE_START: {
5524                         if (cfg->verbose_level > 1)
5525                                 printf ("R%d START=0x%x\n", MONO_VARINFO (cfg, ins->inst_c0)->vreg, (int)(code - cfg->native_code));
5526                         MONO_VARINFO (cfg, ins->inst_c0)->live_range_start = code - cfg->native_code;
5527                         break;
5528                 }
5529                 case OP_LIVERANGE_END: {
5530                         if (cfg->verbose_level > 1)
5531                                 printf ("R%d END=0x%x\n", MONO_VARINFO (cfg, ins->inst_c0)->vreg, (int)(code - cfg->native_code));
5532                         MONO_VARINFO (cfg, ins->inst_c0)->live_range_end = code - cfg->native_code;
5533                         break;
5534                 }
5535                 default:
5536                         g_warning ("unknown opcode %s in %s()\n", mono_inst_name (ins->opcode), __FUNCTION__);
5537                         g_assert_not_reached ();
5538                 }
5539
5540                 if ((code - cfg->native_code - offset) > max_len) {
5541                         g_warning ("wrong maximal instruction length of instruction %s (expected %d, got %ld)",
5542                                    mono_inst_name (ins->opcode), max_len, code - cfg->native_code - offset);
5543                         g_assert_not_reached ();
5544                 }
5545
5546                 last_ins = ins;
5547                 last_offset = offset;
5548         }
5549
5550         cfg->code_len = code - cfg->native_code;
5551 }
5552
5553 #endif /* DISABLE_JIT */
5554
5555 void
5556 mono_arch_register_lowlevel_calls (void)
5557 {
5558         /* The signature doesn't matter */
5559         mono_register_jit_icall (mono_amd64_throw_exception, "mono_amd64_throw_exception", mono_create_icall_signature ("void"), TRUE);
5560 }
5561
5562 void
5563 mono_arch_patch_code (MonoMethod *method, MonoDomain *domain, guint8 *code, MonoJumpInfo *ji, gboolean run_cctors)
5564 {
5565         MonoJumpInfo *patch_info;
5566         gboolean compile_aot = !run_cctors;
5567
5568         for (patch_info = ji; patch_info; patch_info = patch_info->next) {
5569                 unsigned char *ip = patch_info->ip.i + code;
5570                 unsigned char *target;
5571
5572                 target = mono_resolve_patch_target (method, domain, code, patch_info, run_cctors);
5573
5574                 if (compile_aot) {
5575                         switch (patch_info->type) {
5576                         case MONO_PATCH_INFO_BB:
5577                         case MONO_PATCH_INFO_LABEL:
5578                                 break;
5579                         default:
5580                                 /* No need to patch these */
5581                                 continue;
5582                         }
5583                 }
5584
5585                 switch (patch_info->type) {
5586                 case MONO_PATCH_INFO_NONE:
5587                         continue;
5588                 case MONO_PATCH_INFO_METHOD_REL:
5589                 case MONO_PATCH_INFO_R8:
5590                 case MONO_PATCH_INFO_R4:
5591                         g_assert_not_reached ();
5592                         continue;
5593                 case MONO_PATCH_INFO_BB:
5594                         break;
5595                 default:
5596                         break;
5597                 }
5598
5599                 /*
5600                  * Debug code to help track down problems where the target of a near call is
5601                  * is not valid.
5602                  */
5603                 if (amd64_is_near_call (ip)) {
5604                         gint64 disp = (guint8*)target - (guint8*)ip;
5605
5606                         if (!amd64_is_imm32 (disp)) {
5607                                 printf ("TYPE: %d\n", patch_info->type);
5608                                 switch (patch_info->type) {
5609                                 case MONO_PATCH_INFO_INTERNAL_METHOD:
5610                                         printf ("V: %s\n", patch_info->data.name);
5611                                         break;
5612                                 case MONO_PATCH_INFO_METHOD_JUMP:
5613                                 case MONO_PATCH_INFO_METHOD:
5614                                         printf ("V: %s\n", patch_info->data.method->name);
5615                                         break;
5616                                 default:
5617                                         break;
5618                                 }
5619                         }
5620                 }
5621
5622                 amd64_patch (ip, (gpointer)target);
5623         }
5624 }
5625
5626 static int
5627 get_max_epilog_size (MonoCompile *cfg)
5628 {
5629         int max_epilog_size = 16;
5630
5631         if (cfg->method->save_lmf)
5632                 max_epilog_size += 256;
5633
5634         if (mono_jit_trace_calls != NULL)
5635                 max_epilog_size += 50;
5636
5637         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
5638                 max_epilog_size += 50;
5639
5640         max_epilog_size += (AMD64_NREG * 2);
5641
5642         return max_epilog_size;
5643 }
5644
5645 /*
5646  * This macro is used for testing whenever the unwinder works correctly at every point
5647  * where an async exception can happen.
5648  */
5649 /* This will generate a SIGSEGV at the given point in the code */
5650 #define async_exc_point(code) do { \
5651     if (mono_inject_async_exc_method && mono_method_desc_full_match (mono_inject_async_exc_method, cfg->method)) { \
5652          if (cfg->arch.async_point_count == mono_inject_async_exc_pos) \
5653              amd64_mov_reg_mem (code, AMD64_RAX, 0, 4); \
5654          cfg->arch.async_point_count ++; \
5655     } \
5656 } while (0)
5657
5658 guint8 *
5659 mono_arch_emit_prolog (MonoCompile *cfg)
5660 {
5661         MonoMethod *method = cfg->method;
5662         MonoBasicBlock *bb;
5663         MonoMethodSignature *sig;
5664         MonoInst *ins;
5665         int alloc_size, pos, i, cfa_offset, quad, max_epilog_size;
5666         guint8 *code;
5667         CallInfo *cinfo;
5668         gint32 lmf_offset = cfg->arch.lmf_offset;
5669         gboolean args_clobbered = FALSE;
5670         gboolean trace = FALSE;
5671
5672         cfg->code_size =  MAX (((MonoMethodNormal *)method)->header->code_size * 4, 10240);
5673
5674         code = cfg->native_code = g_malloc (cfg->code_size);
5675
5676         if (mono_jit_trace_calls != NULL && mono_trace_eval (method))
5677                 trace = TRUE;
5678
5679         /* Amount of stack space allocated by register saving code */
5680         pos = 0;
5681
5682         /* Offset between RSP and the CFA */
5683         cfa_offset = 0;
5684
5685         /*
5686          * The prolog consists of the following parts:
5687          * FP present:
5688          * - push rbp, mov rbp, rsp
5689          * - save callee saved regs using pushes
5690          * - allocate frame
5691          * - save rgctx if needed
5692          * - save lmf if needed
5693          * FP not present:
5694          * - allocate frame
5695          * - save rgctx if needed
5696          * - save lmf if needed
5697          * - save callee saved regs using moves
5698          */
5699
5700         // CFA = sp + 8
5701         cfa_offset = 8;
5702         mono_emit_unwind_op_def_cfa (cfg, code, AMD64_RSP, 8);
5703         // IP saved at CFA - 8
5704         mono_emit_unwind_op_offset (cfg, code, AMD64_RIP, -cfa_offset);
5705         async_exc_point (code);
5706
5707         if (!cfg->arch.omit_fp) {
5708                 amd64_push_reg (code, AMD64_RBP);
5709                 cfa_offset += 8;
5710                 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
5711                 mono_emit_unwind_op_offset (cfg, code, AMD64_RBP, - cfa_offset);
5712                 async_exc_point (code);
5713 #ifdef HOST_WIN32
5714                 mono_arch_unwindinfo_add_push_nonvol (&cfg->arch.unwindinfo, cfg->native_code, code, AMD64_RBP);
5715 #endif
5716
5717                 amd64_mov_reg_reg (code, AMD64_RBP, AMD64_RSP, sizeof (gpointer));
5718                 mono_emit_unwind_op_def_cfa_reg (cfg, code, AMD64_RBP);
5719                 async_exc_point (code);
5720 #ifdef HOST_WIN32
5721                 mono_arch_unwindinfo_add_set_fpreg (&cfg->arch.unwindinfo, cfg->native_code, code, AMD64_RBP);
5722 #endif
5723         }
5724
5725         /* Save callee saved registers */
5726         if (!cfg->arch.omit_fp && !method->save_lmf) {
5727                 int offset = cfa_offset;
5728
5729                 for (i = 0; i < AMD64_NREG; ++i)
5730                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
5731                                 amd64_push_reg (code, i);
5732                                 pos += sizeof (gpointer);
5733                                 offset += 8;
5734                                 mono_emit_unwind_op_offset (cfg, code, i, - offset);
5735                                 async_exc_point (code);
5736                         }
5737         }
5738
5739         /* The param area is always at offset 0 from sp */
5740         /* This needs to be allocated here, since it has to come after the spill area */
5741         if (cfg->arch.no_pushes && cfg->param_area) {
5742                 if (cfg->arch.omit_fp)
5743                         // FIXME:
5744                         g_assert_not_reached ();
5745                 cfg->stack_offset += ALIGN_TO (cfg->param_area, sizeof (gpointer));
5746         }
5747
5748         if (cfg->arch.omit_fp) {
5749                 /*
5750                  * On enter, the stack is misaligned by the the pushing of the return
5751                  * address. It is either made aligned by the pushing of %rbp, or by
5752                  * this.
5753                  */
5754                 alloc_size = ALIGN_TO (cfg->stack_offset, 8);
5755                 if ((alloc_size % 16) == 0)
5756                         alloc_size += 8;
5757         } else {
5758                 alloc_size = ALIGN_TO (cfg->stack_offset, MONO_ARCH_FRAME_ALIGNMENT);
5759
5760                 alloc_size -= pos;
5761         }
5762
5763         cfg->arch.stack_alloc_size = alloc_size;
5764
5765         /* Allocate stack frame */
5766         if (alloc_size) {
5767                 /* See mono_emit_stack_alloc */
5768 #if defined(HOST_WIN32) || defined(MONO_ARCH_SIGSEGV_ON_ALTSTACK)
5769                 guint32 remaining_size = alloc_size;
5770                 /*FIXME handle unbounded code expansion, we should use a loop in case of more than X interactions*/
5771                 guint32 required_code_size = ((remaining_size / 0x1000) + 1) * 10; /*10 is the max size of amd64_alu_reg_imm + amd64_test_membase_reg*/
5772                 guint32 offset = code - cfg->native_code;
5773                 if (G_UNLIKELY (required_code_size >= (cfg->code_size - offset))) {
5774                         while (required_code_size >= (cfg->code_size - offset))
5775                                 cfg->code_size *= 2;
5776                         cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
5777                         code = cfg->native_code + offset;
5778                         mono_jit_stats.code_reallocs++;
5779                 }
5780
5781                 while (remaining_size >= 0x1000) {
5782                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 0x1000);
5783                         if (cfg->arch.omit_fp) {
5784                                 cfa_offset += 0x1000;
5785                                 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
5786                         }
5787                         async_exc_point (code);
5788 #ifdef HOST_WIN32
5789                         if (cfg->arch.omit_fp)
5790                                 mono_arch_unwindinfo_add_alloc_stack (&cfg->arch.unwindinfo, cfg->native_code, code, 0x1000);
5791 #endif
5792
5793                         amd64_test_membase_reg (code, AMD64_RSP, 0, AMD64_RSP);
5794                         remaining_size -= 0x1000;
5795                 }
5796                 if (remaining_size) {
5797                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, remaining_size);
5798                         if (cfg->arch.omit_fp) {
5799                                 cfa_offset += remaining_size;
5800                                 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
5801                                 async_exc_point (code);
5802                         }
5803 #ifdef HOST_WIN32
5804                         if (cfg->arch.omit_fp)
5805                                 mono_arch_unwindinfo_add_alloc_stack (&cfg->arch.unwindinfo, cfg->native_code, code, remaining_size);
5806 #endif
5807                 }
5808 #else
5809                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, alloc_size);
5810                 if (cfg->arch.omit_fp) {
5811                         cfa_offset += alloc_size;
5812                         mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
5813                         async_exc_point (code);
5814                 }
5815 #endif
5816         }
5817
5818         /* Stack alignment check */
5819 #if 0
5820         {
5821                 amd64_mov_reg_reg (code, AMD64_RAX, AMD64_RSP, 8);
5822                 amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, 0xf);
5823                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0);
5824                 x86_branch8 (code, X86_CC_EQ, 2, FALSE);
5825                 amd64_breakpoint (code);
5826         }
5827 #endif
5828
5829 #ifndef TARGET_WIN32
5830         if (mini_get_debug_options ()->init_stacks) {
5831                 /* Fill the stack frame with a dummy value to force deterministic behavior */
5832
5833                 /* Save registers to the red zone */
5834                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDI, 8);
5835                 amd64_mov_membase_reg (code, AMD64_RSP, -16, AMD64_RCX, 8);
5836
5837                 amd64_mov_reg_imm (code, AMD64_RAX, 0x2a2a2a2a2a2a2a2a);
5838                 amd64_mov_reg_imm (code, AMD64_RCX, alloc_size / 8);
5839                 amd64_mov_reg_reg (code, AMD64_RDI, AMD64_RSP, 8);
5840
5841                 amd64_cld (code);
5842                 amd64_prefix (code, X86_REP_PREFIX);
5843                 amd64_stosl (code);
5844
5845                 amd64_mov_reg_membase (code, AMD64_RDI, AMD64_RSP, -8, 8);
5846                 amd64_mov_reg_membase (code, AMD64_RCX, AMD64_RSP, -16, 8);
5847         }
5848 #endif
5849
5850         /* Save LMF */
5851         if (method->save_lmf) {
5852                 /*
5853                  * The ip field is not set, the exception handling code will obtain it from the stack location pointed to by the sp field.
5854                  */
5855                 /*
5856                  * sp is saved right before calls but we need to save it here too so
5857                  * async stack walks would work.
5858                  */
5859                 amd64_mov_membase_reg (code, cfg->frame_reg, cfg->arch.lmf_offset + G_STRUCT_OFFSET (MonoLMF, rsp), AMD64_RSP, 8);
5860                 /* Skip method (only needed for trampoline LMF frames) */
5861                 /* Save callee saved regs */
5862                 for (i = 0; i < MONO_MAX_IREGS; ++i) {
5863                         int offset;
5864
5865                         switch (i) {
5866                         case AMD64_RBX: offset = G_STRUCT_OFFSET (MonoLMF, rbx); break;
5867                         case AMD64_RBP: offset = G_STRUCT_OFFSET (MonoLMF, rbp); break;
5868                         case AMD64_R12: offset = G_STRUCT_OFFSET (MonoLMF, r12); break;
5869                         case AMD64_R13: offset = G_STRUCT_OFFSET (MonoLMF, r13); break;
5870                         case AMD64_R14: offset = G_STRUCT_OFFSET (MonoLMF, r14); break;
5871                         case AMD64_R15: offset = G_STRUCT_OFFSET (MonoLMF, r15); break;
5872 #ifdef HOST_WIN32
5873                         case AMD64_RDI: offset = G_STRUCT_OFFSET (MonoLMF, rdi); break;
5874                         case AMD64_RSI: offset = G_STRUCT_OFFSET (MonoLMF, rsi); break;
5875 #endif
5876                         default:
5877                                 offset = -1;
5878                                 break;
5879                         }
5880
5881                         if (offset != -1) {
5882                                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + offset, i, 8);
5883                                 if (cfg->arch.omit_fp || (i != AMD64_RBP))
5884                                         mono_emit_unwind_op_offset (cfg, code, i, - (cfa_offset - (lmf_offset + offset)));
5885                         }
5886                 }
5887         }
5888
5889         /* Save callee saved registers */
5890         if (cfg->arch.omit_fp && !method->save_lmf) {
5891                 gint32 save_area_offset = cfg->arch.reg_save_area_offset;
5892
5893                 /* Save caller saved registers after sp is adjusted */
5894                 /* The registers are saved at the bottom of the frame */
5895                 /* FIXME: Optimize this so the regs are saved at the end of the frame in increasing order */
5896                 for (i = 0; i < AMD64_NREG; ++i)
5897                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
5898                                 amd64_mov_membase_reg (code, AMD64_RSP, save_area_offset, i, 8);
5899                                 mono_emit_unwind_op_offset (cfg, code, i, - (cfa_offset - save_area_offset));
5900                                 save_area_offset += 8;
5901                                 async_exc_point (code);
5902                         }
5903         }
5904
5905         /* store runtime generic context */
5906         if (cfg->rgctx_var) {
5907                 g_assert (cfg->rgctx_var->opcode == OP_REGOFFSET &&
5908                                 (cfg->rgctx_var->inst_basereg == AMD64_RBP || cfg->rgctx_var->inst_basereg == AMD64_RSP));
5909
5910                 amd64_mov_membase_reg (code, cfg->rgctx_var->inst_basereg, cfg->rgctx_var->inst_offset, MONO_ARCH_RGCTX_REG, 8);
5911         }
5912
5913         /* compute max_length in order to use short forward jumps */
5914         max_epilog_size = get_max_epilog_size (cfg);
5915         if (cfg->opt & MONO_OPT_BRANCH) {
5916                 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
5917                         MonoInst *ins;
5918                         int max_length = 0;
5919
5920                         if (cfg->prof_options & MONO_PROFILE_COVERAGE)
5921                                 max_length += 6;
5922                         /* max alignment for loops */
5923                         if ((cfg->opt & MONO_OPT_LOOP) && bb_is_loop_start (bb))
5924                                 max_length += LOOP_ALIGNMENT;
5925
5926                         MONO_BB_FOR_EACH_INS (bb, ins) {
5927                                 max_length += ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
5928                         }
5929
5930                         /* Take prolog and epilog instrumentation into account */
5931                         if (bb == cfg->bb_entry || bb == cfg->bb_exit)
5932                                 max_length += max_epilog_size;
5933
5934                         bb->max_length = max_length;
5935                 }
5936         }
5937
5938         sig = mono_method_signature (method);
5939         pos = 0;
5940
5941         cinfo = cfg->arch.cinfo;
5942
5943         if (sig->ret->type != MONO_TYPE_VOID) {
5944                 /* Save volatile arguments to the stack */
5945                 if (cfg->vret_addr && (cfg->vret_addr->opcode != OP_REGVAR))
5946                         amd64_mov_membase_reg (code, cfg->vret_addr->inst_basereg, cfg->vret_addr->inst_offset, cinfo->ret.reg, 8);
5947         }
5948
5949         /* Keep this in sync with emit_load_volatile_arguments */
5950         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
5951                 ArgInfo *ainfo = cinfo->args + i;
5952                 gint32 stack_offset;
5953                 MonoType *arg_type;
5954
5955                 ins = cfg->args [i];
5956
5957                 if ((ins->flags & MONO_INST_IS_DEAD) && !trace)
5958                         /* Unused arguments */
5959                         continue;
5960
5961                 if (sig->hasthis && (i == 0))
5962                         arg_type = &mono_defaults.object_class->byval_arg;
5963                 else
5964                         arg_type = sig->params [i - sig->hasthis];
5965
5966                 stack_offset = ainfo->offset + ARGS_OFFSET;
5967
5968                 if (cfg->globalra) {
5969                         /* All the other moves are done by the register allocator */
5970                         switch (ainfo->storage) {
5971                         case ArgInFloatSSEReg:
5972                                 amd64_sse_cvtss2sd_reg_reg (code, ainfo->reg, ainfo->reg);
5973                                 break;
5974                         case ArgValuetypeInReg:
5975                                 for (quad = 0; quad < 2; quad ++) {
5976                                         switch (ainfo->pair_storage [quad]) {
5977                                         case ArgInIReg:
5978                                                 amd64_mov_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad], sizeof (gpointer));
5979                                                 break;
5980                                         case ArgInFloatSSEReg:
5981                                                 amd64_movss_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad]);
5982                                                 break;
5983                                         case ArgInDoubleSSEReg:
5984                                                 amd64_movsd_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad]);
5985                                                 break;
5986                                         case ArgNone:
5987                                                 break;
5988                                         default:
5989                                                 g_assert_not_reached ();
5990                                         }
5991                                 }
5992                                 break;
5993                         default:
5994                                 break;
5995                         }
5996
5997                         continue;
5998                 }
5999
6000                 /* Save volatile arguments to the stack */
6001                 if (ins->opcode != OP_REGVAR) {
6002                         switch (ainfo->storage) {
6003                         case ArgInIReg: {
6004                                 guint32 size = 8;
6005
6006                                 /* FIXME: I1 etc */
6007                                 /*
6008                                 if (stack_offset & 0x1)
6009                                         size = 1;
6010                                 else if (stack_offset & 0x2)
6011                                         size = 2;
6012                                 else if (stack_offset & 0x4)
6013                                         size = 4;
6014                                 else
6015                                         size = 8;
6016                                 */
6017                                 amd64_mov_membase_reg (code, ins->inst_basereg, ins->inst_offset, ainfo->reg, size);
6018                                 break;
6019                         }
6020                         case ArgInFloatSSEReg:
6021                                 amd64_movss_membase_reg (code, ins->inst_basereg, ins->inst_offset, ainfo->reg);
6022                                 break;
6023                         case ArgInDoubleSSEReg:
6024                                 amd64_movsd_membase_reg (code, ins->inst_basereg, ins->inst_offset, ainfo->reg);
6025                                 break;
6026                         case ArgValuetypeInReg:
6027                                 for (quad = 0; quad < 2; quad ++) {
6028                                         switch (ainfo->pair_storage [quad]) {
6029                                         case ArgInIReg:
6030                                                 amd64_mov_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad], sizeof (gpointer));
6031                                                 break;
6032                                         case ArgInFloatSSEReg:
6033                                                 amd64_movss_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad]);
6034                                                 break;
6035                                         case ArgInDoubleSSEReg:
6036                                                 amd64_movsd_membase_reg (code, ins->inst_basereg, ins->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad]);
6037                                                 break;
6038                                         case ArgNone:
6039                                                 break;
6040                                         default:
6041                                                 g_assert_not_reached ();
6042                                         }
6043                                 }
6044                                 break;
6045                         case ArgValuetypeAddrInIReg:
6046                                 if (ainfo->pair_storage [0] == ArgInIReg)
6047                                         amd64_mov_membase_reg (code, ins->inst_left->inst_basereg, ins->inst_left->inst_offset, ainfo->pair_regs [0],  sizeof (gpointer));
6048                                 break;
6049                         default:
6050                                 break;
6051                         }
6052                 } else {
6053                         /* Argument allocated to (non-volatile) register */
6054                         switch (ainfo->storage) {
6055                         case ArgInIReg:
6056                                 amd64_mov_reg_reg (code, ins->dreg, ainfo->reg, 8);
6057                                 break;
6058                         case ArgOnStack:
6059                                 amd64_mov_reg_membase (code, ins->dreg, AMD64_RBP, ARGS_OFFSET + ainfo->offset, 8);
6060                                 break;
6061                         default:
6062                                 g_assert_not_reached ();
6063                         }
6064                 }
6065         }
6066
6067         /* Might need to attach the thread to the JIT  or change the domain for the callback */
6068         if (method->wrapper_type == MONO_WRAPPER_NATIVE_TO_MANAGED) {
6069                 guint64 domain = (guint64)cfg->domain;
6070
6071                 args_clobbered = TRUE;
6072
6073                 /*
6074                  * The call might clobber argument registers, but they are already
6075                  * saved to the stack/global regs.
6076                  */
6077                 if (appdomain_tls_offset != -1 && lmf_tls_offset != -1) {
6078                         guint8 *buf, *no_domain_branch;
6079
6080                         code = mono_amd64_emit_tls_get (code, AMD64_RAX, appdomain_tls_offset);
6081                         if (cfg->compile_aot) {
6082                                 /* AOT code is only used in the root domain */
6083                                 amd64_mov_reg_imm (code, AMD64_ARG_REG1, 0);
6084                         } else {
6085                                 if ((domain >> 32) == 0)
6086                                         amd64_mov_reg_imm_size (code, AMD64_ARG_REG1, domain, 4);
6087                                 else
6088                                         amd64_mov_reg_imm_size (code, AMD64_ARG_REG1, domain, 8);
6089                         }
6090                         amd64_alu_reg_reg (code, X86_CMP, AMD64_RAX, AMD64_ARG_REG1);
6091                         no_domain_branch = code;
6092                         x86_branch8 (code, X86_CC_NE, 0, 0);
6093                         code = mono_amd64_emit_tls_get ( code, AMD64_RAX, lmf_addr_tls_offset);
6094                         amd64_test_reg_reg (code, AMD64_RAX, AMD64_RAX);
6095                         buf = code;
6096                         x86_branch8 (code, X86_CC_NE, 0, 0);
6097                         amd64_patch (no_domain_branch, code);
6098                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
6099                                           (gpointer)"mono_jit_thread_attach", TRUE);
6100                         amd64_patch (buf, code);
6101 #ifdef HOST_WIN32
6102                         /* The TLS key actually contains a pointer to the MonoJitTlsData structure */
6103                         /* FIXME: Add a separate key for LMF to avoid this */
6104                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RAX, G_STRUCT_OFFSET (MonoJitTlsData, lmf));
6105 #endif
6106                 } else {
6107                         g_assert (!cfg->compile_aot);
6108                         if (cfg->compile_aot) {
6109                                 /* AOT code is only used in the root domain */
6110                                 amd64_mov_reg_imm (code, AMD64_ARG_REG1, 0);
6111                         } else {
6112                                 if ((domain >> 32) == 0)
6113                                         amd64_mov_reg_imm_size (code, AMD64_ARG_REG1, domain, 4);
6114                                 else
6115                                         amd64_mov_reg_imm_size (code, AMD64_ARG_REG1, domain, 8);
6116                         }
6117                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
6118                                           (gpointer)"mono_jit_thread_attach", TRUE);
6119                 }
6120         }
6121
6122         if (method->save_lmf) {
6123                 if ((lmf_tls_offset != -1) && !optimize_for_xen) {
6124                         /*
6125                          * Optimized version which uses the mono_lmf TLS variable instead of
6126                          * indirection through the mono_lmf_addr TLS variable.
6127                          */
6128                         /* %rax = previous_lmf */
6129                         x86_prefix (code, X86_FS_PREFIX);
6130                         amd64_mov_reg_mem (code, AMD64_RAX, lmf_tls_offset, 8);
6131
6132                         /* Save previous_lmf */
6133                         amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), AMD64_RAX, 8);
6134                         /* Set new lmf */
6135                         if (lmf_offset == 0) {
6136                                 x86_prefix (code, X86_FS_PREFIX);
6137                                 amd64_mov_mem_reg (code, lmf_tls_offset, cfg->frame_reg, 8);
6138                         } else {
6139                                 amd64_lea_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset);
6140                                 x86_prefix (code, X86_FS_PREFIX);
6141                                 amd64_mov_mem_reg (code, lmf_tls_offset, AMD64_R11, 8);
6142                         }
6143                 } else {
6144                         if (lmf_addr_tls_offset != -1) {
6145                                 /* Load lmf quicky using the FS register */
6146                                 code = mono_amd64_emit_tls_get (code, AMD64_RAX, lmf_addr_tls_offset);
6147 #ifdef HOST_WIN32
6148                                 /* The TLS key actually contains a pointer to the MonoJitTlsData structure */
6149                                 /* FIXME: Add a separate key for LMF to avoid this */
6150                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RAX, G_STRUCT_OFFSET (MonoJitTlsData, lmf));
6151 #endif
6152                         }
6153                         else {
6154                                 /*
6155                                  * The call might clobber argument registers, but they are already
6156                                  * saved to the stack/global regs.
6157                                  */
6158                                 args_clobbered = TRUE;
6159                                 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
6160                                                                   (gpointer)"mono_get_lmf_addr", TRUE);
6161                         }
6162
6163                         /* Save lmf_addr */
6164                         amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, lmf_addr), AMD64_RAX, 8);
6165                         /* Save previous_lmf */
6166                         amd64_mov_reg_membase (code, AMD64_R11, AMD64_RAX, 0, 8);
6167                         amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), AMD64_R11, 8);
6168                         /* Set new lmf */
6169                         amd64_lea_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset);
6170                         amd64_mov_membase_reg (code, AMD64_RAX, 0, AMD64_R11, 8);
6171                 }
6172         }
6173
6174         if (trace) {
6175                 args_clobbered = TRUE;
6176                 code = mono_arch_instrument_prolog (cfg, mono_trace_enter_method, code, TRUE);
6177         }
6178
6179         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
6180                 args_clobbered = TRUE;
6181
6182         /*
6183          * Optimize the common case of the first bblock making a call with the same
6184          * arguments as the method. This works because the arguments are still in their
6185          * original argument registers.
6186          * FIXME: Generalize this
6187          */
6188         if (!args_clobbered) {
6189                 MonoBasicBlock *first_bb = cfg->bb_entry;
6190                 MonoInst *next;
6191
6192                 next = mono_bb_first_ins (first_bb);
6193                 if (!next && first_bb->next_bb) {
6194                         first_bb = first_bb->next_bb;
6195                         next = mono_bb_first_ins (first_bb);
6196                 }
6197
6198                 if (first_bb->in_count > 1)
6199                         next = NULL;
6200
6201                 for (i = 0; next && i < sig->param_count + sig->hasthis; ++i) {
6202                         ArgInfo *ainfo = cinfo->args + i;
6203                         gboolean match = FALSE;
6204
6205                         ins = cfg->args [i];
6206                         if (ins->opcode != OP_REGVAR) {
6207                                 switch (ainfo->storage) {
6208                                 case ArgInIReg: {
6209                                         if (((next->opcode == OP_LOAD_MEMBASE) || (next->opcode == OP_LOADI4_MEMBASE)) && next->inst_basereg == ins->inst_basereg && next->inst_offset == ins->inst_offset) {
6210                                                 if (next->dreg == ainfo->reg) {
6211                                                         NULLIFY_INS (next);
6212                                                         match = TRUE;
6213                                                 } else {
6214                                                         next->opcode = OP_MOVE;
6215                                                         next->sreg1 = ainfo->reg;
6216                                                         /* Only continue if the instruction doesn't change argument regs */
6217                                                         if (next->dreg == ainfo->reg || next->dreg == AMD64_RAX)
6218                                                                 match = TRUE;
6219                                                 }
6220                                         }
6221                                         break;
6222                                 }
6223                                 default:
6224                                         break;
6225                                 }
6226                         } else {
6227                                 /* Argument allocated to (non-volatile) register */
6228                                 switch (ainfo->storage) {
6229                                 case ArgInIReg:
6230                                         if (next->opcode == OP_MOVE && next->sreg1 == ins->dreg && next->dreg == ainfo->reg) {
6231                                                 NULLIFY_INS (next);
6232                                                 match = TRUE;
6233                                         }
6234                                         break;
6235                                 default:
6236                                         break;
6237                                 }
6238                         }
6239
6240                         if (match) {
6241                                 next = next->next;
6242                                 //next = mono_inst_list_next (&next->node, &first_bb->ins_list);
6243                                 if (!next)
6244                                         break;
6245                         }
6246                 }
6247         }
6248
6249         /* Initialize ss_trigger_page_var */
6250         if (cfg->arch.ss_trigger_page_var) {
6251                 MonoInst *var = cfg->arch.ss_trigger_page_var;
6252
6253                 g_assert (!cfg->compile_aot);
6254                 g_assert (var->opcode == OP_REGOFFSET);
6255
6256                 amd64_mov_reg_imm (code, AMD64_R11, (guint64)ss_trigger_page);
6257                 amd64_mov_membase_reg (code, var->inst_basereg, var->inst_offset, AMD64_R11, 8);
6258         }
6259
6260         cfg->code_len = code - cfg->native_code;
6261
6262         g_assert (cfg->code_len < cfg->code_size);
6263
6264         return code;
6265 }
6266
6267 void
6268 mono_arch_emit_epilog (MonoCompile *cfg)
6269 {
6270         MonoMethod *method = cfg->method;
6271         int quad, pos, i;
6272         guint8 *code;
6273         int max_epilog_size;
6274         CallInfo *cinfo;
6275         gint32 lmf_offset = cfg->arch.lmf_offset;
6276
6277         max_epilog_size = get_max_epilog_size (cfg);
6278
6279         while (cfg->code_len + max_epilog_size > (cfg->code_size - 16)) {
6280                 cfg->code_size *= 2;
6281                 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
6282                 mono_jit_stats.code_reallocs++;
6283         }
6284
6285         code = cfg->native_code + cfg->code_len;
6286
6287         if (mono_jit_trace_calls != NULL && mono_trace_eval (method))
6288                 code = mono_arch_instrument_epilog (cfg, mono_trace_leave_method, code, TRUE);
6289
6290         /* the code restoring the registers must be kept in sync with OP_JMP */
6291         pos = 0;
6292
6293         if (method->save_lmf) {
6294                 /* check if we need to restore protection of the stack after a stack overflow */
6295                 if (mono_get_jit_tls_offset () != -1) {
6296                         guint8 *patch;
6297                         code = mono_amd64_emit_tls_get (code, X86_ECX, mono_get_jit_tls_offset ());
6298                         /* we load the value in a separate instruction: this mechanism may be
6299                          * used later as a safer way to do thread interruption
6300                          */
6301                         amd64_mov_reg_membase (code, X86_ECX, X86_ECX, G_STRUCT_OFFSET (MonoJitTlsData, restore_stack_prot), 8);
6302                         x86_alu_reg_imm (code, X86_CMP, X86_ECX, 0);
6303                         patch = code;
6304                         x86_branch8 (code, X86_CC_Z, 0, FALSE);
6305                         /* note that the call trampoline will preserve eax/edx */
6306                         x86_call_reg (code, X86_ECX);
6307                         x86_patch (patch, code);
6308                 } else {
6309                         /* FIXME: maybe save the jit tls in the prolog */
6310                 }
6311                 if ((lmf_tls_offset != -1) && !optimize_for_xen) {
6312                         /*
6313                          * Optimized version which uses the mono_lmf TLS variable instead of indirection
6314                          * through the mono_lmf_addr TLS variable.
6315                          */
6316                         /* reg = previous_lmf */
6317                         amd64_mov_reg_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), 8);
6318                         x86_prefix (code, X86_FS_PREFIX);
6319                         amd64_mov_mem_reg (code, lmf_tls_offset, AMD64_R11, 8);
6320                 } else {
6321                         /* Restore previous lmf */
6322                         amd64_mov_reg_membase (code, AMD64_RCX, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), 8);
6323                         amd64_mov_reg_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, lmf_addr), 8);
6324                         amd64_mov_membase_reg (code, AMD64_R11, 0, AMD64_RCX, 8);
6325                 }
6326
6327                 /* Restore caller saved regs */
6328                 if (cfg->used_int_regs & (1 << AMD64_RBP)) {
6329                         amd64_mov_reg_membase (code, AMD64_RBP, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbp), 8);
6330                 }
6331                 if (cfg->used_int_regs & (1 << AMD64_RBX)) {
6332                         amd64_mov_reg_membase (code, AMD64_RBX, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbx), 8);
6333                 }
6334                 if (cfg->used_int_regs & (1 << AMD64_R12)) {
6335                         amd64_mov_reg_membase (code, AMD64_R12, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r12), 8);
6336                 }
6337                 if (cfg->used_int_regs & (1 << AMD64_R13)) {
6338                         amd64_mov_reg_membase (code, AMD64_R13, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r13), 8);
6339                 }
6340                 if (cfg->used_int_regs & (1 << AMD64_R14)) {
6341                         amd64_mov_reg_membase (code, AMD64_R14, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r14), 8);
6342                 }
6343                 if (cfg->used_int_regs & (1 << AMD64_R15)) {
6344                         amd64_mov_reg_membase (code, AMD64_R15, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r15), 8);
6345                 }
6346 #ifdef HOST_WIN32
6347                 if (cfg->used_int_regs & (1 << AMD64_RDI)) {
6348                         amd64_mov_reg_membase (code, AMD64_RDI, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rdi), 8);
6349                 }
6350                 if (cfg->used_int_regs & (1 << AMD64_RSI)) {
6351                         amd64_mov_reg_membase (code, AMD64_RSI, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rsi), 8);
6352                 }
6353 #endif
6354         } else {
6355
6356                 if (cfg->arch.omit_fp) {
6357                         gint32 save_area_offset = cfg->arch.reg_save_area_offset;
6358
6359                         for (i = 0; i < AMD64_NREG; ++i)
6360                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
6361                                         amd64_mov_reg_membase (code, i, AMD64_RSP, save_area_offset, 8);
6362                                         save_area_offset += 8;
6363                                 }
6364                 }
6365                 else {
6366                         for (i = 0; i < AMD64_NREG; ++i)
6367                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i)))
6368                                         pos -= sizeof (gpointer);
6369
6370                         if (pos) {
6371                                 if (pos == - sizeof (gpointer)) {
6372                                         /* Only one register, so avoid lea */
6373                                         for (i = AMD64_NREG - 1; i > 0; --i)
6374                                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
6375                                                         amd64_mov_reg_membase (code, i, AMD64_RBP, pos, 8);
6376                                                 }
6377                                 }
6378                                 else {
6379                                         amd64_lea_membase (code, AMD64_RSP, AMD64_RBP, pos);
6380
6381                                         /* Pop registers in reverse order */
6382                                         for (i = AMD64_NREG - 1; i > 0; --i)
6383                                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
6384                                                         amd64_pop_reg (code, i);
6385                                                 }
6386                                 }
6387                         }
6388                 }
6389         }
6390
6391         /* Load returned vtypes into registers if needed */
6392         cinfo = cfg->arch.cinfo;
6393         if (cinfo->ret.storage == ArgValuetypeInReg) {
6394                 ArgInfo *ainfo = &cinfo->ret;
6395                 MonoInst *inst = cfg->ret;
6396
6397                 for (quad = 0; quad < 2; quad ++) {
6398                         switch (ainfo->pair_storage [quad]) {
6399                         case ArgInIReg:
6400                                 amd64_mov_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)), sizeof (gpointer));
6401                                 break;
6402                         case ArgInFloatSSEReg:
6403                                 amd64_movss_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)));
6404                                 break;
6405                         case ArgInDoubleSSEReg:
6406                                 amd64_movsd_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)));
6407                                 break;
6408                         case ArgNone:
6409                                 break;
6410                         default:
6411                                 g_assert_not_reached ();
6412                         }
6413                 }
6414         }
6415
6416         if (cfg->arch.omit_fp) {
6417                 if (cfg->arch.stack_alloc_size)
6418                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, cfg->arch.stack_alloc_size);
6419         } else {
6420                 amd64_leave (code);
6421         }
6422         async_exc_point (code);
6423         amd64_ret (code);
6424
6425         cfg->code_len = code - cfg->native_code;
6426
6427         g_assert (cfg->code_len < cfg->code_size);
6428 }
6429
6430 void
6431 mono_arch_emit_exceptions (MonoCompile *cfg)
6432 {
6433         MonoJumpInfo *patch_info;
6434         int nthrows, i;
6435         guint8 *code;
6436         MonoClass *exc_classes [16];
6437         guint8 *exc_throw_start [16], *exc_throw_end [16];
6438         guint32 code_size = 0;
6439
6440         /* Compute needed space */
6441         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
6442                 if (patch_info->type == MONO_PATCH_INFO_EXC)
6443                         code_size += 40;
6444                 if (patch_info->type == MONO_PATCH_INFO_R8)
6445                         code_size += 8 + 15; /* sizeof (double) + alignment */
6446                 if (patch_info->type == MONO_PATCH_INFO_R4)
6447                         code_size += 4 + 15; /* sizeof (float) + alignment */
6448         }
6449
6450         while (cfg->code_len + code_size > (cfg->code_size - 16)) {
6451                 cfg->code_size *= 2;
6452                 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
6453                 mono_jit_stats.code_reallocs++;
6454         }
6455
6456         code = cfg->native_code + cfg->code_len;
6457
6458         /* add code to raise exceptions */
6459         nthrows = 0;
6460         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
6461                 switch (patch_info->type) {
6462                 case MONO_PATCH_INFO_EXC: {
6463                         MonoClass *exc_class;
6464                         guint8 *buf, *buf2;
6465                         guint32 throw_ip;
6466
6467                         amd64_patch (patch_info->ip.i + cfg->native_code, code);
6468
6469                         exc_class = mono_class_from_name (mono_defaults.corlib, "System", patch_info->data.name);
6470                         g_assert (exc_class);
6471                         throw_ip = patch_info->ip.i;
6472
6473                         //x86_breakpoint (code);
6474                         /* Find a throw sequence for the same exception class */
6475                         for (i = 0; i < nthrows; ++i)
6476                                 if (exc_classes [i] == exc_class)
6477                                         break;
6478                         if (i < nthrows) {
6479                                 amd64_mov_reg_imm (code, AMD64_ARG_REG2, (exc_throw_end [i] - cfg->native_code) - throw_ip);
6480                                 x86_jump_code (code, exc_throw_start [i]);
6481                                 patch_info->type = MONO_PATCH_INFO_NONE;
6482                         }
6483                         else {
6484                                 buf = code;
6485                                 amd64_mov_reg_imm_size (code, AMD64_ARG_REG2, 0xf0f0f0f0, 4);
6486                                 buf2 = code;
6487
6488                                 if (nthrows < 16) {
6489                                         exc_classes [nthrows] = exc_class;
6490                                         exc_throw_start [nthrows] = code;
6491                                 }
6492                                 amd64_mov_reg_imm (code, AMD64_ARG_REG1, exc_class->type_token);
6493
6494                                 patch_info->type = MONO_PATCH_INFO_NONE;
6495
6496                                 code = emit_call_body (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_corlib_exception");
6497
6498                                 amd64_mov_reg_imm (buf, AMD64_ARG_REG2, (code - cfg->native_code) - throw_ip);
6499                                 while (buf < buf2)
6500                                         x86_nop (buf);
6501
6502                                 if (nthrows < 16) {
6503                                         exc_throw_end [nthrows] = code;
6504                                         nthrows ++;
6505                                 }
6506                         }
6507                         break;
6508                 }
6509                 default:
6510                         /* do nothing */
6511                         break;
6512                 }
6513         }
6514
6515         /* Handle relocations with RIP relative addressing */
6516         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
6517                 gboolean remove = FALSE;
6518
6519                 switch (patch_info->type) {
6520                 case MONO_PATCH_INFO_R8:
6521                 case MONO_PATCH_INFO_R4: {
6522                         guint8 *pos;
6523
6524                         /* The SSE opcodes require a 16 byte alignment */
6525                         code = (guint8*)ALIGN_TO (code, 16);
6526
6527                         pos = cfg->native_code + patch_info->ip.i;
6528
6529                         if (IS_REX (pos [1]))
6530                                 *(guint32*)(pos + 5) = (guint8*)code - pos - 9;
6531                         else
6532                                 *(guint32*)(pos + 4) = (guint8*)code - pos - 8;
6533
6534                         if (patch_info->type == MONO_PATCH_INFO_R8) {
6535                                 *(double*)code = *(double*)patch_info->data.target;
6536                                 code += sizeof (double);
6537                         } else {
6538                                 *(float*)code = *(float*)patch_info->data.target;
6539                                 code += sizeof (float);
6540                         }
6541
6542                         remove = TRUE;
6543                         break;
6544                 }
6545                 default:
6546                         break;
6547                 }
6548
6549                 if (remove) {
6550                         if (patch_info == cfg->patch_info)
6551                                 cfg->patch_info = patch_info->next;
6552                         else {
6553                                 MonoJumpInfo *tmp;
6554
6555                                 for (tmp = cfg->patch_info; tmp->next != patch_info; tmp = tmp->next)
6556                                         ;
6557                                 tmp->next = patch_info->next;
6558                         }
6559                 }
6560         }
6561
6562         cfg->code_len = code - cfg->native_code;
6563
6564         g_assert (cfg->code_len < cfg->code_size);
6565
6566 }
6567
6568 void*
6569 mono_arch_instrument_prolog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
6570 {
6571         guchar *code = p;
6572         CallInfo *cinfo = NULL;
6573         MonoMethodSignature *sig;
6574         MonoInst *inst;
6575         int i, n, stack_area = 0;
6576
6577         /* Keep this in sync with mono_arch_get_argument_info */
6578
6579         if (enable_arguments) {
6580                 /* Allocate a new area on the stack and save arguments there */
6581                 sig = mono_method_signature (cfg->method);
6582
6583                 cinfo = get_call_info (cfg->generic_sharing_context, cfg->mempool, sig, FALSE);
6584
6585                 n = sig->param_count + sig->hasthis;
6586
6587                 stack_area = ALIGN_TO (n * 8, 16);
6588
6589                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, stack_area);
6590
6591                 for (i = 0; i < n; ++i) {
6592                         inst = cfg->args [i];
6593
6594                         if (inst->opcode == OP_REGVAR)
6595                                 amd64_mov_membase_reg (code, AMD64_RSP, (i * 8), inst->dreg, 8);
6596                         else {
6597                                 amd64_mov_reg_membase (code, AMD64_R11, inst->inst_basereg, inst->inst_offset, 8);
6598                                 amd64_mov_membase_reg (code, AMD64_RSP, (i * 8), AMD64_R11, 8);
6599                         }
6600                 }
6601         }
6602
6603         mono_add_patch_info (cfg, code-cfg->native_code, MONO_PATCH_INFO_METHODCONST, cfg->method);
6604         amd64_set_reg_template (code, AMD64_ARG_REG1);
6605         amd64_mov_reg_reg (code, AMD64_ARG_REG2, AMD64_RSP, 8);
6606         code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)func, TRUE);
6607
6608         if (enable_arguments)
6609                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, stack_area);
6610
6611         return code;
6612 }
6613
6614 enum {
6615         SAVE_NONE,
6616         SAVE_STRUCT,
6617         SAVE_EAX,
6618         SAVE_EAX_EDX,
6619         SAVE_XMM
6620 };
6621
6622 void*
6623 mono_arch_instrument_epilog_full (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments, gboolean preserve_argument_registers)
6624 {
6625         guchar *code = p;
6626         int save_mode = SAVE_NONE;
6627         MonoMethod *method = cfg->method;
6628         MonoType *ret_type = mini_type_get_underlying_type (NULL, mono_method_signature (method)->ret);
6629
6630         switch (ret_type->type) {
6631         case MONO_TYPE_VOID:
6632                 /* special case string .ctor icall */
6633                 if (strcmp (".ctor", method->name) && method->klass == mono_defaults.string_class)
6634                         save_mode = SAVE_EAX;
6635                 else
6636                         save_mode = SAVE_NONE;
6637                 break;
6638         case MONO_TYPE_I8:
6639         case MONO_TYPE_U8:
6640                 save_mode = SAVE_EAX;
6641                 break;
6642         case MONO_TYPE_R4:
6643         case MONO_TYPE_R8:
6644                 save_mode = SAVE_XMM;
6645                 break;
6646         case MONO_TYPE_GENERICINST:
6647                 if (!mono_type_generic_inst_is_valuetype (ret_type)) {
6648                         save_mode = SAVE_EAX;
6649                         break;
6650                 }
6651                 /* Fall through */
6652         case MONO_TYPE_VALUETYPE:
6653                 save_mode = SAVE_STRUCT;
6654                 break;
6655         default:
6656                 save_mode = SAVE_EAX;
6657                 break;
6658         }
6659
6660         /* Save the result and copy it into the proper argument register */
6661         switch (save_mode) {
6662         case SAVE_EAX:
6663                 amd64_push_reg (code, AMD64_RAX);
6664                 /* Align stack */
6665                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
6666                 if (enable_arguments)
6667                         amd64_mov_reg_reg (code, AMD64_ARG_REG2, AMD64_RAX, 8);
6668                 break;
6669         case SAVE_STRUCT:
6670                 /* FIXME: */
6671                 if (enable_arguments)
6672                         amd64_mov_reg_imm (code, AMD64_ARG_REG2, 0);
6673                 break;
6674         case SAVE_XMM:
6675                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
6676                 amd64_movsd_membase_reg (code, AMD64_RSP, 0, AMD64_XMM0);
6677                 /* Align stack */
6678                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
6679                 /*
6680                  * The result is already in the proper argument register so no copying
6681                  * needed.
6682                  */
6683                 break;
6684         case SAVE_NONE:
6685                 break;
6686         default:
6687                 g_assert_not_reached ();
6688         }
6689
6690         /* Set %al since this is a varargs call */
6691         if (save_mode == SAVE_XMM)
6692                 amd64_mov_reg_imm (code, AMD64_RAX, 1);
6693         else
6694                 amd64_mov_reg_imm (code, AMD64_RAX, 0);
6695
6696         if (preserve_argument_registers) {
6697                 amd64_push_reg (code, MONO_AMD64_ARG_REG1);
6698                 amd64_push_reg (code, MONO_AMD64_ARG_REG2);
6699         }
6700
6701         mono_add_patch_info (cfg, code-cfg->native_code, MONO_PATCH_INFO_METHODCONST, method);
6702         amd64_set_reg_template (code, AMD64_ARG_REG1);
6703         code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)func, TRUE);
6704
6705         if (preserve_argument_registers) {
6706                 amd64_pop_reg (code, MONO_AMD64_ARG_REG2);
6707                 amd64_pop_reg (code, MONO_AMD64_ARG_REG1);
6708         }
6709
6710         /* Restore result */
6711         switch (save_mode) {
6712         case SAVE_EAX:
6713                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
6714                 amd64_pop_reg (code, AMD64_RAX);
6715                 break;
6716         case SAVE_STRUCT:
6717                 /* FIXME: */
6718                 break;
6719         case SAVE_XMM:
6720                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
6721                 amd64_movsd_reg_membase (code, AMD64_XMM0, AMD64_RSP, 0);
6722                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
6723                 break;
6724         case SAVE_NONE:
6725                 break;
6726         default:
6727                 g_assert_not_reached ();
6728         }
6729
6730         return code;
6731 }
6732
6733 void
6734 mono_arch_flush_icache (guint8 *code, gint size)
6735 {
6736         /* Not needed */
6737 }
6738
6739 void
6740 mono_arch_flush_register_windows (void)
6741 {
6742 }
6743
6744 gboolean
6745 mono_arch_is_inst_imm (gint64 imm)
6746 {
6747         return amd64_is_imm32 (imm);
6748 }
6749
6750 /*
6751  * Determine whenever the trap whose info is in SIGINFO is caused by
6752  * integer overflow.
6753  */
6754 gboolean
6755 mono_arch_is_int_overflow (void *sigctx, void *info)
6756 {
6757         MonoContext ctx;
6758         guint8* rip;
6759         int reg;
6760         gint64 value;
6761
6762         mono_arch_sigctx_to_monoctx (sigctx, &ctx);
6763
6764         rip = (guint8*)ctx.rip;
6765
6766         if (IS_REX (rip [0])) {
6767                 reg = amd64_rex_b (rip [0]);
6768                 rip ++;
6769         }
6770         else
6771                 reg = 0;
6772
6773         if ((rip [0] == 0xf7) && (x86_modrm_mod (rip [1]) == 0x3) && (x86_modrm_reg (rip [1]) == 0x7)) {
6774                 /* idiv REG */
6775                 reg += x86_modrm_rm (rip [1]);
6776
6777                 switch (reg) {
6778                 case AMD64_RAX:
6779                         value = ctx.rax;
6780                         break;
6781                 case AMD64_RBX:
6782                         value = ctx.rbx;
6783                         break;
6784                 case AMD64_RCX:
6785                         value = ctx.rcx;
6786                         break;
6787                 case AMD64_RDX:
6788                         value = ctx.rdx;
6789                         break;
6790                 case AMD64_RBP:
6791                         value = ctx.rbp;
6792                         break;
6793                 case AMD64_RSP:
6794                         value = ctx.rsp;
6795                         break;
6796                 case AMD64_RSI:
6797                         value = ctx.rsi;
6798                         break;
6799                 case AMD64_RDI:
6800                         value = ctx.rdi;
6801                         break;
6802                 case AMD64_R12:
6803                         value = ctx.r12;
6804                         break;
6805                 case AMD64_R13:
6806                         value = ctx.r13;
6807                         break;
6808                 case AMD64_R14:
6809                         value = ctx.r14;
6810                         break;
6811                 case AMD64_R15:
6812                         value = ctx.r15;
6813                         break;
6814                 default:
6815                         g_assert_not_reached ();
6816                         reg = -1;
6817                 }
6818
6819                 if (value == -1)
6820                         return TRUE;
6821         }
6822
6823         return FALSE;
6824 }
6825
6826 guint32
6827 mono_arch_get_patch_offset (guint8 *code)
6828 {
6829         return 3;
6830 }
6831
6832 /**
6833  * mono_breakpoint_clean_code:
6834  *
6835  * Copy @size bytes from @code - @offset to the buffer @buf. If the debugger inserted software
6836  * breakpoints in the original code, they are removed in the copy.
6837  *
6838  * Returns TRUE if no sw breakpoint was present.
6839  */
6840 gboolean
6841 mono_breakpoint_clean_code (guint8 *method_start, guint8 *code, int offset, guint8 *buf, int size)
6842 {
6843         int i;
6844         gboolean can_write = TRUE;
6845         /*
6846          * If method_start is non-NULL we need to perform bound checks, since we access memory
6847          * at code - offset we could go before the start of the method and end up in a different
6848          * page of memory that is not mapped or read incorrect data anyway. We zero-fill the bytes
6849          * instead.
6850          */
6851         if (!method_start || code - offset >= method_start) {
6852                 memcpy (buf, code - offset, size);
6853         } else {
6854                 int diff = code - method_start;
6855                 memset (buf, 0, size);
6856                 memcpy (buf + offset - diff, method_start, diff + size - offset);
6857         }
6858         code -= offset;
6859         for (i = 0; i < MONO_BREAKPOINT_ARRAY_SIZE; ++i) {
6860                 int idx = mono_breakpoint_info_index [i];
6861                 guint8 *ptr;
6862                 if (idx < 1)
6863                         continue;
6864                 ptr = mono_breakpoint_info [idx].address;
6865                 if (ptr >= code && ptr < code + size) {
6866                         guint8 saved_byte = mono_breakpoint_info [idx].saved_byte;
6867                         can_write = FALSE;
6868                         /*g_print ("patching %p with 0x%02x (was: 0x%02x)\n", ptr, saved_byte, buf [ptr - code]);*/
6869                         buf [ptr - code] = saved_byte;
6870                 }
6871         }
6872         return can_write;
6873 }
6874
6875 gpointer
6876 mono_arch_get_vcall_slot (guint8 *code, mgreg_t *regs, int *displacement)
6877 {
6878         guint8 buf [10];
6879         guint32 reg;
6880         gint32 disp;
6881         guint8 rex = 0;
6882         MonoJitInfo *ji = NULL;
6883
6884 #ifdef ENABLE_LLVM
6885         /* code - 9 might be before the start of the method */
6886         /* FIXME: Avoid this expensive call somehow */
6887         ji = mono_jit_info_table_find (mono_domain_get (), (char*)code);
6888 #endif
6889
6890         mono_breakpoint_clean_code (ji ? ji->code_start : NULL, code, 9, buf, sizeof (buf));
6891         code = buf + 9;
6892
6893         *displacement = 0;
6894
6895         code -= 7;
6896
6897         /*
6898          * A given byte sequence can match more than case here, so we have to be
6899          * really careful about the ordering of the cases. Longer sequences
6900          * come first.
6901          * There are two types of calls:
6902          * - direct calls: 0xff address_byte 8/32 bits displacement
6903          * - indirect calls: nop nop nop <call>
6904          * The nops make sure we don't confuse the instruction preceeding an indirect
6905          * call with a direct call.
6906          */
6907         if ((code [0] == 0x41) && (code [1] == 0xff) && (code [2] == 0x15)) {
6908                 /* call OFFSET(%rip) */
6909                 disp = *(guint32*)(code + 3);
6910                 return (gpointer*)(code + disp + 7);
6911         } else if ((code [0] == 0xff) && (amd64_modrm_reg (code [1]) == 0x2) && (amd64_modrm_mod (code [1]) == 0x2) && (amd64_sib_index (code [2]) == 4) && (amd64_sib_scale (code [2]) == 0)) {
6912                 /* call *[reg+disp32] using indexed addressing */
6913                 /* The LLVM JIT emits this, and we emit it too for %r12 */
6914                 if (IS_REX (code [-1])) {
6915                         rex = code [-1];
6916                         g_assert (amd64_rex_x (rex) == 0);
6917                 }
6918                 reg = amd64_sib_base (code [2]);
6919                 disp = *(gint32*)(code + 3);
6920         } else if ((code [1] == 0xff) && (amd64_modrm_reg (code [2]) == 0x2) && (amd64_modrm_mod (code [2]) == 0x2)) {
6921                 /* call *[reg+disp32] */
6922                 if (IS_REX (code [0]))
6923                         rex = code [0];
6924                 reg = amd64_modrm_rm (code [2]);
6925                 disp = *(gint32*)(code + 3);
6926                 /* R10 is clobbered by the IMT thunk code */
6927                 g_assert (reg != AMD64_R10);
6928         } else if (code [2] == 0xe8) {
6929                 /* call <ADDR> */
6930                 return NULL;
6931         } else if ((code [3] == 0xff) && (amd64_modrm_reg (code [4]) == 0x2) && (amd64_modrm_mod (code [4]) == 0x1) && (amd64_sib_index (code [5]) == 4) && (amd64_sib_scale (code [5]) == 0)) {
6932                 /* call *[r12+disp8] using indexed addressing */
6933                 if (IS_REX (code [2]))
6934                         rex = code [2];
6935                 reg = amd64_sib_base (code [5]);
6936                 disp = *(gint8*)(code + 6);
6937         } else if (IS_REX (code [4]) && (code [5] == 0xff) && (amd64_modrm_reg (code [6]) == 0x2) && (amd64_modrm_mod (code [6]) == 0x3)) {
6938                 /* call *%reg */
6939                 return NULL;
6940         } else if ((code [4] == 0xff) && (amd64_modrm_reg (code [5]) == 0x2) && (amd64_modrm_mod (code [5]) == 0x1)) {
6941                 /* call *[reg+disp8] */
6942                 if (IS_REX (code [3]))
6943                         rex = code [3];
6944                 reg = amd64_modrm_rm (code [5]);
6945                 disp = *(gint8*)(code + 6);
6946                 //printf ("B: [%%r%d+0x%x]\n", reg, disp);
6947         }
6948         else if ((code [5] == 0xff) && (amd64_modrm_reg (code [6]) == 0x2) && (amd64_modrm_mod (code [6]) == 0x0)) {
6949                 /* call *%reg */
6950                 if (IS_REX (code [4]))
6951                         rex = code [4];
6952                 reg = amd64_modrm_rm (code [6]);
6953                 disp = 0;
6954         }
6955         else
6956                 g_assert_not_reached ();
6957
6958         reg += amd64_rex_b (rex);
6959
6960         /* R11 is clobbered by the trampoline code */
6961         g_assert (reg != AMD64_R11);
6962
6963         *displacement = disp;
6964         return (gpointer)regs [reg];
6965 }
6966
6967 int
6968 mono_arch_get_this_arg_reg (MonoMethodSignature *sig, MonoGenericSharingContext *gsctx, guint8 *code)
6969 {
6970         int this_reg = AMD64_ARG_REG1;
6971
6972         if (MONO_TYPE_ISSTRUCT (sig->ret)) {
6973                 CallInfo *cinfo;
6974
6975                 if (!gsctx && code)
6976                         gsctx = mono_get_generic_context_from_code (code);
6977
6978                 cinfo = get_call_info (gsctx, NULL, sig, FALSE);
6979
6980                 if (cinfo->ret.storage != ArgValuetypeInReg)
6981                         this_reg = AMD64_ARG_REG2;
6982                 g_free (cinfo);
6983         }
6984
6985         return this_reg;
6986 }
6987
6988 gpointer
6989 mono_arch_get_this_arg_from_call (MonoGenericSharingContext *gsctx, MonoMethodSignature *sig, mgreg_t *regs, guint8 *code)
6990 {
6991         return (gpointer)regs [mono_arch_get_this_arg_reg (sig, gsctx, code)];
6992 }
6993
6994 #define MAX_ARCH_DELEGATE_PARAMS 10
6995
6996 static gpointer
6997 get_delegate_invoke_impl (gboolean has_target, guint32 param_count, guint32 *code_len)
6998 {
6999         guint8 *code, *start;
7000         int i;
7001
7002         if (has_target) {
7003                 start = code = mono_global_codeman_reserve (64);
7004
7005                 /* Replace the this argument with the target */
7006                 amd64_mov_reg_reg (code, AMD64_RAX, AMD64_ARG_REG1, 8);
7007                 amd64_mov_reg_membase (code, AMD64_ARG_REG1, AMD64_RAX, G_STRUCT_OFFSET (MonoDelegate, target), 8);
7008                 amd64_jump_membase (code, AMD64_RAX, G_STRUCT_OFFSET (MonoDelegate, method_ptr));
7009
7010                 g_assert ((code - start) < 64);
7011         } else {
7012                 start = code = mono_global_codeman_reserve (64);
7013
7014                 if (param_count == 0) {
7015                         amd64_jump_membase (code, AMD64_ARG_REG1, G_STRUCT_OFFSET (MonoDelegate, method_ptr));
7016                 } else {
7017                         /* We have to shift the arguments left */
7018                         amd64_mov_reg_reg (code, AMD64_RAX, AMD64_ARG_REG1, 8);
7019                         for (i = 0; i < param_count; ++i) {
7020 #ifdef HOST_WIN32
7021                                 if (i < 3)
7022                                         amd64_mov_reg_reg (code, param_regs [i], param_regs [i + 1], 8);
7023                                 else
7024                                         amd64_mov_reg_membase (code, param_regs [i], AMD64_RSP, 0x28, 8);
7025 #else
7026                                 amd64_mov_reg_reg (code, param_regs [i], param_regs [i + 1], 8);
7027 #endif
7028                         }
7029
7030                         amd64_jump_membase (code, AMD64_RAX, G_STRUCT_OFFSET (MonoDelegate, method_ptr));
7031                 }
7032                 g_assert ((code - start) < 64);
7033         }
7034
7035         mono_debug_add_delegate_trampoline (start, code - start);
7036
7037         if (code_len)
7038                 *code_len = code - start;
7039
7040         return start;
7041 }
7042
7043 /*
7044  * mono_arch_get_delegate_invoke_impls:
7045  *
7046  *   Return a list of MonoAotTrampInfo structures for the delegate invoke impl
7047  * trampolines.
7048  */
7049 GSList*
7050 mono_arch_get_delegate_invoke_impls (void)
7051 {
7052         GSList *res = NULL;
7053         guint8 *code;
7054         guint32 code_len;
7055         int i;
7056
7057         code = get_delegate_invoke_impl (TRUE, 0, &code_len);
7058         res = g_slist_prepend (res, mono_aot_tramp_info_create (g_strdup ("delegate_invoke_impl_has_target"), code, code_len));
7059
7060         for (i = 0; i < MAX_ARCH_DELEGATE_PARAMS; ++i) {
7061                 code = get_delegate_invoke_impl (FALSE, i, &code_len);
7062                 res = g_slist_prepend (res, mono_aot_tramp_info_create (g_strdup_printf ("delegate_invoke_impl_target_%d", i), code, code_len));
7063         }
7064
7065         return res;
7066 }
7067
7068 gpointer
7069 mono_arch_get_delegate_invoke_impl (MonoMethodSignature *sig, gboolean has_target)
7070 {
7071         guint8 *code, *start;
7072         int i;
7073
7074         if (sig->param_count > MAX_ARCH_DELEGATE_PARAMS)
7075                 return NULL;
7076
7077         /* FIXME: Support more cases */
7078         if (MONO_TYPE_ISSTRUCT (sig->ret))
7079                 return NULL;
7080
7081         if (has_target) {
7082                 static guint8* cached = NULL;
7083
7084                 if (cached)
7085                         return cached;
7086
7087                 if (mono_aot_only)
7088                         start = mono_aot_get_named_code ("delegate_invoke_impl_has_target");
7089                 else
7090                         start = get_delegate_invoke_impl (TRUE, 0, NULL);
7091
7092                 mono_memory_barrier ();
7093
7094                 cached = start;
7095         } else {
7096                 static guint8* cache [MAX_ARCH_DELEGATE_PARAMS + 1] = {NULL};
7097                 for (i = 0; i < sig->param_count; ++i)
7098                         if (!mono_is_regsize_var (sig->params [i]))
7099                                 return NULL;
7100                 if (sig->param_count > 4)
7101                         return NULL;
7102
7103                 code = cache [sig->param_count];
7104                 if (code)
7105                         return code;
7106
7107                 if (mono_aot_only) {
7108                         char *name = g_strdup_printf ("delegate_invoke_impl_target_%d", sig->param_count);
7109                         start = mono_aot_get_named_code (name);
7110                         g_free (name);
7111                 } else {
7112                         start = get_delegate_invoke_impl (FALSE, sig->param_count, NULL);
7113                 }
7114
7115                 mono_memory_barrier ();
7116
7117                 cache [sig->param_count] = start;
7118         }
7119
7120         return start;
7121 }
7122
7123 /*
7124  * Support for fast access to the thread-local lmf structure using the GS
7125  * segment register on NPTL + kernel 2.6.x.
7126  */
7127
7128 static gboolean tls_offset_inited = FALSE;
7129
7130 void
7131 mono_arch_setup_jit_tls_data (MonoJitTlsData *tls)
7132 {
7133         if (!tls_offset_inited) {
7134 #ifdef HOST_WIN32
7135                 /*
7136                  * We need to init this multiple times, since when we are first called, the key might not
7137                  * be initialized yet.
7138                  */
7139                 appdomain_tls_offset = mono_domain_get_tls_key ();
7140                 lmf_tls_offset = mono_get_jit_tls_key ();
7141                 lmf_addr_tls_offset = mono_get_jit_tls_key ();
7142
7143                 /* Only 64 tls entries can be accessed using inline code */
7144                 if (appdomain_tls_offset >= 64)
7145                         appdomain_tls_offset = -1;
7146                 if (lmf_tls_offset >= 64)
7147                         lmf_tls_offset = -1;
7148 #else
7149                 tls_offset_inited = TRUE;
7150 #ifdef MONO_XEN_OPT
7151                 optimize_for_xen = access ("/proc/xen", F_OK) == 0;
7152 #endif
7153                 appdomain_tls_offset = mono_domain_get_tls_offset ();
7154                 lmf_tls_offset = mono_get_lmf_tls_offset ();
7155                 lmf_addr_tls_offset = mono_get_lmf_addr_tls_offset ();
7156 #endif
7157         }
7158 }
7159
7160 void
7161 mono_arch_free_jit_tls_data (MonoJitTlsData *tls)
7162 {
7163 }
7164
7165 #ifdef MONO_ARCH_HAVE_IMT
7166
7167 #define CMP_SIZE (6 + 1)
7168 #define CMP_REG_REG_SIZE (4 + 1)
7169 #define BR_SMALL_SIZE 2
7170 #define BR_LARGE_SIZE 6
7171 #define MOV_REG_IMM_SIZE 10
7172 #define MOV_REG_IMM_32BIT_SIZE 6
7173 #define JUMP_REG_SIZE (2 + 1)
7174
7175 static int
7176 imt_branch_distance (MonoIMTCheckItem **imt_entries, int start, int target)
7177 {
7178         int i, distance = 0;
7179         for (i = start; i < target; ++i)
7180                 distance += imt_entries [i]->chunk_size;
7181         return distance;
7182 }
7183
7184 /*
7185  * LOCKING: called with the domain lock held
7186  */
7187 gpointer
7188 mono_arch_build_imt_thunk (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
7189         gpointer fail_tramp)
7190 {
7191         int i;
7192         int size = 0;
7193         guint8 *code, *start;
7194         gboolean vtable_is_32bit = ((gsize)(vtable) == (gsize)(int)(gsize)(vtable));
7195
7196         for (i = 0; i < count; ++i) {
7197                 MonoIMTCheckItem *item = imt_entries [i];
7198                 if (item->is_equals) {
7199                         if (item->check_target_idx) {
7200                                 if (!item->compare_done) {
7201                                         if (amd64_is_imm32 (item->key))
7202                                                 item->chunk_size += CMP_SIZE;
7203                                         else
7204                                                 item->chunk_size += MOV_REG_IMM_SIZE + CMP_REG_REG_SIZE;
7205                                 }
7206                                 if (item->has_target_code) {
7207                                         item->chunk_size += MOV_REG_IMM_SIZE;
7208                                 } else {
7209                                         if (vtable_is_32bit)
7210                                                 item->chunk_size += MOV_REG_IMM_32BIT_SIZE;
7211                                         else
7212                                                 item->chunk_size += MOV_REG_IMM_SIZE;
7213                                 }
7214                                 item->chunk_size += BR_SMALL_SIZE + JUMP_REG_SIZE;
7215                         } else {
7216                                 if (fail_tramp) {
7217                                         item->chunk_size += MOV_REG_IMM_SIZE * 3 + CMP_REG_REG_SIZE +
7218                                                 BR_SMALL_SIZE + JUMP_REG_SIZE * 2;
7219                                 } else {
7220                                         if (vtable_is_32bit)
7221                                                 item->chunk_size += MOV_REG_IMM_32BIT_SIZE;
7222                                         else
7223                                                 item->chunk_size += MOV_REG_IMM_SIZE;
7224                                         item->chunk_size += JUMP_REG_SIZE;
7225                                         /* with assert below:
7226                                          * item->chunk_size += CMP_SIZE + BR_SMALL_SIZE + 1;
7227                                          */
7228                                 }
7229                         }
7230                 } else {
7231                         if (amd64_is_imm32 (item->key))
7232                                 item->chunk_size += CMP_SIZE;
7233                         else
7234                                 item->chunk_size += MOV_REG_IMM_SIZE + CMP_REG_REG_SIZE;
7235                         item->chunk_size += BR_LARGE_SIZE;
7236                         imt_entries [item->check_target_idx]->compare_done = TRUE;
7237                 }
7238                 size += item->chunk_size;
7239         }
7240         if (fail_tramp)
7241                 code = mono_method_alloc_generic_virtual_thunk (domain, size);
7242         else
7243                 code = mono_domain_code_reserve (domain, size);
7244         start = code;
7245         for (i = 0; i < count; ++i) {
7246                 MonoIMTCheckItem *item = imt_entries [i];
7247                 item->code_target = code;
7248                 if (item->is_equals) {
7249                         gboolean fail_case = !item->check_target_idx && fail_tramp;
7250
7251                         if (item->check_target_idx || fail_case) {
7252                                 if (!item->compare_done || fail_case) {
7253                                         if (amd64_is_imm32 (item->key))
7254                                                 amd64_alu_reg_imm (code, X86_CMP, MONO_ARCH_IMT_REG, (guint32)(gssize)item->key);
7255                                         else {
7256                                                 amd64_mov_reg_imm (code, AMD64_R10, item->key);
7257                                                 amd64_alu_reg_reg (code, X86_CMP, MONO_ARCH_IMT_REG, AMD64_R10);
7258                                         }
7259                                 }
7260                                 item->jmp_code = code;
7261                                 amd64_branch8 (code, X86_CC_NE, 0, FALSE);
7262                                 /* See the comment below about R10 */
7263                                 if (item->has_target_code) {
7264                                         amd64_mov_reg_imm (code, AMD64_R10, item->value.target_code);
7265                                         amd64_jump_reg (code, AMD64_R10);
7266                                 } else {
7267                                         amd64_mov_reg_imm (code, AMD64_R10, & (vtable->vtable [item->value.vtable_slot]));
7268                                         amd64_jump_membase (code, AMD64_R10, 0);
7269                                 }
7270
7271                                 if (fail_case) {
7272                                         amd64_patch (item->jmp_code, code);
7273                                         amd64_mov_reg_imm (code, AMD64_R10, fail_tramp);
7274                                         amd64_jump_reg (code, AMD64_R10);
7275                                         item->jmp_code = NULL;
7276                                 }
7277                         } else {
7278                                 /* enable the commented code to assert on wrong method */
7279 #if 0
7280                                 if (amd64_is_imm32 (item->key))
7281                                         amd64_alu_reg_imm (code, X86_CMP, MONO_ARCH_IMT_REG, (guint32)(gssize)item->key);
7282                                 else {
7283                                         amd64_mov_reg_imm (code, AMD64_R10, item->key);
7284                                         amd64_alu_reg_reg (code, X86_CMP, MONO_ARCH_IMT_REG, AMD64_R10);
7285                                 }
7286                                 item->jmp_code = code;
7287                                 amd64_branch8 (code, X86_CC_NE, 0, FALSE);
7288                                 /* See the comment below about R10 */
7289                                 amd64_mov_reg_imm (code, AMD64_R10, & (vtable->vtable [item->value.vtable_slot]));
7290                                 amd64_jump_membase (code, AMD64_R10, 0);
7291                                 amd64_patch (item->jmp_code, code);
7292                                 amd64_breakpoint (code);
7293                                 item->jmp_code = NULL;
7294 #else
7295                                 /* We're using R10 here because R11
7296                                    needs to be preserved.  R10 needs
7297                                    to be preserved for calls which
7298                                    require a runtime generic context,
7299                                    but interface calls don't. */
7300                                 amd64_mov_reg_imm (code, AMD64_R10, & (vtable->vtable [item->value.vtable_slot]));
7301                                 amd64_jump_membase (code, AMD64_R10, 0);
7302 #endif
7303                         }
7304                 } else {
7305                         if (amd64_is_imm32 (item->key))
7306                                 amd64_alu_reg_imm (code, X86_CMP, MONO_ARCH_IMT_REG, (guint32)(gssize)item->key);
7307                         else {
7308                                 amd64_mov_reg_imm (code, AMD64_R10, item->key);
7309                                 amd64_alu_reg_reg (code, X86_CMP, MONO_ARCH_IMT_REG, AMD64_R10);
7310                         }
7311                         item->jmp_code = code;
7312                         if (x86_is_imm8 (imt_branch_distance (imt_entries, i, item->check_target_idx)))
7313                                 x86_branch8 (code, X86_CC_GE, 0, FALSE);
7314                         else
7315                                 x86_branch32 (code, X86_CC_GE, 0, FALSE);
7316                 }
7317                 g_assert (code - item->code_target <= item->chunk_size);
7318         }
7319         /* patch the branches to get to the target items */
7320         for (i = 0; i < count; ++i) {
7321                 MonoIMTCheckItem *item = imt_entries [i];
7322                 if (item->jmp_code) {
7323                         if (item->check_target_idx) {
7324                                 amd64_patch (item->jmp_code, imt_entries [item->check_target_idx]->code_target);
7325                         }
7326                 }
7327         }
7328
7329         if (!fail_tramp)
7330                 mono_stats.imt_thunks_size += code - start;
7331         g_assert (code - start <= size);
7332
7333         return start;
7334 }
7335
7336 MonoMethod*
7337 mono_arch_find_imt_method (mgreg_t *regs, guint8 *code)
7338 {
7339         return (MonoMethod*)regs [MONO_ARCH_IMT_REG];
7340 }
7341 #endif
7342
7343 MonoVTable*
7344 mono_arch_find_static_call_vtable (mgreg_t *regs, guint8 *code)
7345 {
7346         return (MonoVTable*) regs [MONO_ARCH_RGCTX_REG];
7347 }
7348
7349 MonoInst*
7350 mono_arch_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args)
7351 {
7352         MonoInst *ins = NULL;
7353         int opcode = 0;
7354
7355         if (cmethod->klass == mono_defaults.math_class) {
7356                 if (strcmp (cmethod->name, "Sin") == 0) {
7357                         opcode = OP_SIN;
7358                 } else if (strcmp (cmethod->name, "Cos") == 0) {
7359                         opcode = OP_COS;
7360                 } else if (strcmp (cmethod->name, "Sqrt") == 0) {
7361                         opcode = OP_SQRT;
7362                 } else if (strcmp (cmethod->name, "Abs") == 0 && fsig->params [0]->type == MONO_TYPE_R8) {
7363                         opcode = OP_ABS;
7364                 }
7365
7366                 if (opcode) {
7367                         MONO_INST_NEW (cfg, ins, opcode);
7368                         ins->type = STACK_R8;
7369                         ins->dreg = mono_alloc_freg (cfg);
7370                         ins->sreg1 = args [0]->dreg;
7371                         MONO_ADD_INS (cfg->cbb, ins);
7372                 }
7373
7374                 opcode = 0;
7375                 if (cfg->opt & MONO_OPT_CMOV) {
7376                         if (strcmp (cmethod->name, "Min") == 0) {
7377                                 if (fsig->params [0]->type == MONO_TYPE_I4)
7378                                         opcode = OP_IMIN;
7379                                 if (fsig->params [0]->type == MONO_TYPE_U4)
7380                                         opcode = OP_IMIN_UN;
7381                                 else if (fsig->params [0]->type == MONO_TYPE_I8)
7382                                         opcode = OP_LMIN;
7383                                 else if (fsig->params [0]->type == MONO_TYPE_U8)
7384                                         opcode = OP_LMIN_UN;
7385                         } else if (strcmp (cmethod->name, "Max") == 0) {
7386                                 if (fsig->params [0]->type == MONO_TYPE_I4)
7387                                         opcode = OP_IMAX;
7388                                 if (fsig->params [0]->type == MONO_TYPE_U4)
7389                                         opcode = OP_IMAX_UN;
7390                                 else if (fsig->params [0]->type == MONO_TYPE_I8)
7391                                         opcode = OP_LMAX;
7392                                 else if (fsig->params [0]->type == MONO_TYPE_U8)
7393                                         opcode = OP_LMAX_UN;
7394                         }
7395                 }
7396
7397                 if (opcode) {
7398                         MONO_INST_NEW (cfg, ins, opcode);
7399                         ins->type = fsig->params [0]->type == MONO_TYPE_I4 ? STACK_I4 : STACK_I8;
7400                         ins->dreg = mono_alloc_ireg (cfg);
7401                         ins->sreg1 = args [0]->dreg;
7402                         ins->sreg2 = args [1]->dreg;
7403                         MONO_ADD_INS (cfg->cbb, ins);
7404                 }
7405
7406 #if 0
7407                 /* OP_FREM is not IEEE compatible */
7408                 else if (strcmp (cmethod->name, "IEEERemainder") == 0) {
7409                         MONO_INST_NEW (cfg, ins, OP_FREM);
7410                         ins->inst_i0 = args [0];
7411                         ins->inst_i1 = args [1];
7412                 }
7413 #endif
7414         }
7415
7416         /*
7417          * Can't implement CompareExchange methods this way since they have
7418          * three arguments.
7419          */
7420
7421         return ins;
7422 }
7423
7424 gboolean
7425 mono_arch_print_tree (MonoInst *tree, int arity)
7426 {
7427         return 0;
7428 }
7429
7430 MonoInst* mono_arch_get_domain_intrinsic (MonoCompile* cfg)
7431 {
7432         MonoInst* ins;
7433
7434         if (appdomain_tls_offset == -1)
7435                 return NULL;
7436
7437         MONO_INST_NEW (cfg, ins, OP_TLS_GET);
7438         ins->inst_offset = appdomain_tls_offset;
7439         return ins;
7440 }
7441
7442 #define _CTX_REG(ctx,fld,i) ((gpointer)((&ctx->fld)[i]))
7443
7444 gpointer
7445 mono_arch_context_get_int_reg (MonoContext *ctx, int reg)
7446 {
7447         switch (reg) {
7448         case AMD64_RCX: return (gpointer)ctx->rcx;
7449         case AMD64_RDX: return (gpointer)ctx->rdx;
7450         case AMD64_RBX: return (gpointer)ctx->rbx;
7451         case AMD64_RBP: return (gpointer)ctx->rbp;
7452         case AMD64_RSP: return (gpointer)ctx->rsp;
7453         default:
7454                 if (reg < 8)
7455                         return _CTX_REG (ctx, rax, reg);
7456                 else if (reg >= 12)
7457                         return _CTX_REG (ctx, r12, reg - 12);
7458                 else
7459                         g_assert_not_reached ();
7460         }
7461 }
7462
7463 /* Soft Debug support */
7464 #ifdef MONO_ARCH_SOFT_DEBUG_SUPPORTED
7465
7466 /*
7467  * mono_arch_set_breakpoint:
7468  *
7469  *   Set a breakpoint at the native code corresponding to JI at NATIVE_OFFSET.
7470  * The location should contain code emitted by OP_SEQ_POINT.
7471  */
7472 void
7473 mono_arch_set_breakpoint (MonoJitInfo *ji, guint8 *ip)
7474 {
7475         guint8 *code = ip;
7476         guint8 *orig_code = code;
7477
7478         /*
7479          * In production, we will use int3 (has to fix the size in the md
7480          * file). But that could confuse gdb, so during development, we emit a SIGSEGV
7481          * instead.
7482          */
7483         g_assert (code [0] == 0x90);
7484         if (breakpoint_size == 8) {
7485                 amd64_mov_reg_mem (code, AMD64_R11, (guint64)bp_trigger_page, 4);
7486         } else {
7487                 amd64_mov_reg_imm_size (code, AMD64_R11, (guint64)bp_trigger_page, 8);
7488                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_R11, 0, 4);
7489         }
7490
7491         g_assert (code - orig_code == breakpoint_size);
7492 }
7493
7494 /*
7495  * mono_arch_clear_breakpoint:
7496  *
7497  *   Clear the breakpoint at IP.
7498  */
7499 void
7500 mono_arch_clear_breakpoint (MonoJitInfo *ji, guint8 *ip)
7501 {
7502         guint8 *code = ip;
7503         int i;
7504
7505         for (i = 0; i < breakpoint_size; ++i)
7506                 x86_nop (code);
7507 }
7508
7509 gboolean
7510 mono_arch_is_breakpoint_event (void *info, void *sigctx)
7511 {
7512 #ifdef HOST_WIN32
7513         EXCEPTION_RECORD* einfo = (EXCEPTION_RECORD*)info;
7514         return FALSE;
7515 #else
7516         siginfo_t* sinfo = (siginfo_t*) info;
7517         /* Sometimes the address is off by 4 */
7518         if (sinfo->si_addr >= bp_trigger_page && (guint8*)sinfo->si_addr <= (guint8*)bp_trigger_page + 128)
7519                 return TRUE;
7520         else
7521                 return FALSE;
7522 #endif
7523 }
7524
7525 /*
7526  * mono_arch_get_ip_for_breakpoint:
7527  *
7528  *   Convert the ip in CTX to the address where a breakpoint was placed.
7529  */
7530 guint8*
7531 mono_arch_get_ip_for_breakpoint (MonoJitInfo *ji, MonoContext *ctx)
7532 {
7533         guint8 *ip = MONO_CONTEXT_GET_IP (ctx);
7534
7535         /* ip points to the instruction causing the fault */
7536         ip -= (breakpoint_size - breakpoint_fault_size);
7537
7538         return ip;
7539 }
7540
7541 /*
7542  * mono_arch_skip_breakpoint:
7543  *
7544  *   Modify CTX so the ip is placed after the breakpoint instruction, so when
7545  * we resume, the instruction is not executed again.
7546  */
7547 void
7548 mono_arch_skip_breakpoint (MonoContext *ctx)
7549 {
7550         MONO_CONTEXT_SET_IP (ctx, (guint8*)MONO_CONTEXT_GET_IP (ctx) + breakpoint_fault_size);
7551 }
7552
7553 /*
7554  * mono_arch_start_single_stepping:
7555  *
7556  *   Start single stepping.
7557  */
7558 void
7559 mono_arch_start_single_stepping (void)
7560 {
7561         mono_mprotect (ss_trigger_page, mono_pagesize (), 0);
7562 }
7563
7564 /*
7565  * mono_arch_stop_single_stepping:
7566  *
7567  *   Stop single stepping.
7568  */
7569 void
7570 mono_arch_stop_single_stepping (void)
7571 {
7572         mono_mprotect (ss_trigger_page, mono_pagesize (), MONO_MMAP_READ);
7573 }
7574
7575 /*
7576  * mono_arch_is_single_step_event:
7577  *
7578  *   Return whenever the machine state in SIGCTX corresponds to a single
7579  * step event.
7580  */
7581 gboolean
7582 mono_arch_is_single_step_event (void *info, void *sigctx)
7583 {
7584 #ifdef HOST_WIN32
7585         EXCEPTION_RECORD* einfo = (EXCEPTION_RECORD*)info;
7586         return FALSE;
7587 #else
7588         siginfo_t* sinfo = (siginfo_t*) info;
7589         /* Sometimes the address is off by 4 */
7590         if (sinfo->si_addr >= ss_trigger_page && (guint8*)sinfo->si_addr <= (guint8*)ss_trigger_page + 128)
7591                 return TRUE;
7592         else
7593                 return FALSE;
7594 #endif
7595 }
7596
7597 /*
7598  * mono_arch_get_ip_for_single_step:
7599  *
7600  *   Convert the ip in CTX to the address stored in seq_points.
7601  */
7602 guint8*
7603 mono_arch_get_ip_for_single_step (MonoJitInfo *ji, MonoContext *ctx)
7604 {
7605         guint8 *ip = MONO_CONTEXT_GET_IP (ctx);
7606
7607         ip += single_step_fault_size;
7608
7609         return ip;
7610 }
7611
7612 /*
7613  * mono_arch_skip_single_step:
7614  *
7615  *   Modify CTX so the ip is placed after the single step trigger instruction,
7616  * we resume, the instruction is not executed again.
7617  */
7618 void
7619 mono_arch_skip_single_step (MonoContext *ctx)
7620 {
7621         MONO_CONTEXT_SET_IP (ctx, (guint8*)MONO_CONTEXT_GET_IP (ctx) + single_step_fault_size);
7622 }
7623
7624 /*
7625  * mono_arch_create_seq_point_info:
7626  *
7627  *   Return a pointer to a data structure which is used by the sequence
7628  * point implementation in AOTed code.
7629  */
7630 gpointer
7631 mono_arch_get_seq_point_info (MonoDomain *domain, guint8 *code)
7632 {
7633         NOT_IMPLEMENTED;
7634         return NULL;
7635 }
7636
7637 #endif