mono/mini/mini-amd64.c

   1 /*
   2  * mini-amd64.c: AMD64 backend for the Mono code generator
   3  *
   4  * Based on mini-x86.c.
   5  *
   6  * Authors:
   7  *   Paolo Molaro (lupus@ximian.com)
   8  *   Dietmar Maurer (dietmar@ximian.com)
   9  *   Patrik Torstensson
  10  *
  11  * (C) 2003 Ximian, Inc.
  12  */
  13 #include "mini.h"
  14 #include <string.h>
  15 #include <math.h>
  16 #include <unistd.h>
  17
  18 #include <mono/metadata/appdomain.h>
  19 #include <mono/metadata/debug-helpers.h>
  20 #include <mono/metadata/threads.h>
  21 #include <mono/metadata/profiler-private.h>
  22 #include <mono/metadata/mono-debug.h>
  23 #include <mono/utils/mono-math.h>
  24
  25 #include "trace.h"
  26 #include "mini-amd64.h"
  27 #include "inssel.h"
  28 #include "cpu-amd64.h"
  29
  30 static gint lmf_tls_offset = -1;
  31 static gint lmf_addr_tls_offset = -1;
  32 static gint appdomain_tls_offset = -1;
  33 static gint thread_tls_offset = -1;
  34
  35 #ifdef MONO_XEN_OPT
  36 static gboolean optimize_for_xen = TRUE;
  37 #else
  38 #define optimize_for_xen 0
  39 #endif
  40
  41 static gboolean use_sse2 = !MONO_ARCH_USE_FPSTACK;
  42
  43 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
  44
  45 #define IS_IMM32(val) ((((guint64)val) >> 32) == 0)
  46
  47 #define IS_REX(inst) (((inst) >= 0x40) && ((inst) <= 0x4f))
  48
  49 #ifdef PLATFORM_WIN32
  50 /* Under windows, the default pinvoke calling convention is stdcall */
  51 #define CALLCONV_IS_STDCALL(call_conv) (((call_conv) == MONO_CALL_STDCALL) || ((call_conv) == MONO_CALL_DEFAULT))
  52 #else
  53 #define CALLCONV_IS_STDCALL(call_conv) ((call_conv) == MONO_CALL_STDCALL)
  54 #endif
  55
  56 /* This mutex protects architecture specific caches */
  57 #define mono_mini_arch_lock() EnterCriticalSection (&mini_arch_mutex)
  58 #define mono_mini_arch_unlock() LeaveCriticalSection (&mini_arch_mutex)
  59 static CRITICAL_SECTION mini_arch_mutex;
  60
  61 #define ARGS_OFFSET 16
  62 #define GP_SCRATCH_REG AMD64_R11
  63
  64 /*
  65  * AMD64 register usage:
  66  * - callee saved registers are used for global register allocation
  67  * - %r11 is used for materializing 64 bit constants in opcodes
  68  * - the rest is used for local allocation
  69  */
  70
  71 /*
  72  * Floating point comparison results:
  73  *                  ZF PF CF
  74  * A > B            0  0  0
  75  * A < B            0  0  1
  76  * A = B            1  0  0
  77  * A > B            0  0  0
  78  * UNORDERED        1  1  1
  79  */
  80
  81 #define NOT_IMPLEMENTED g_assert_not_reached ()
  82
  83 const char*
  84 mono_arch_regname (int reg) {
  85         switch (reg) {
  86         case AMD64_RAX: return "%rax";
  87         case AMD64_RBX: return "%rbx";
  88         case AMD64_RCX: return "%rcx";
  89         case AMD64_RDX: return "%rdx";
  90         case AMD64_RSP: return "%rsp";
  91         case AMD64_RBP: return "%rbp";
  92         case AMD64_RDI: return "%rdi";
  93         case AMD64_RSI: return "%rsi";
  94         case AMD64_R8: return "%r8";
  95         case AMD64_R9: return "%r9";
  96         case AMD64_R10: return "%r10";
  97         case AMD64_R11: return "%r11";
  98         case AMD64_R12: return "%r12";
  99         case AMD64_R13: return "%r13";
 100         case AMD64_R14: return "%r14";
 101         case AMD64_R15: return "%r15";
 102         }
 103         return "unknown";
 104 }
 105
 106 static const char * xmmregs [] = {
 107         "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8",
 108         "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"
 109 };
 110
 111 const char*
 112 mono_arch_fregname (int reg)
 113 {
 114         if (reg < AMD64_XMM_NREG)
 115                 return xmmregs [reg];
 116         else
 117                 return "unknown";
 118 }
 119
 120 G_GNUC_UNUSED static void
 121 break_count (void)
 122 {
 123 }
 124
 125 G_GNUC_UNUSED static gboolean
 126 debug_count (void)
 127 {
 128         static int count = 0;
 129         count ++;
 130
 131         if (!getenv ("COUNT"))
 132                 return TRUE;
 133
 134         if (count == atoi (getenv ("COUNT"))) {
 135                 break_count ();
 136         }
 137
 138         if (count > atoi (getenv ("COUNT"))) {
 139                 return FALSE;
 140         }
 141
 142         return TRUE;
 143 }
 144
 145 static gboolean
 146 debug_omit_fp (void)
 147 {
 148 #if 0
 149         return debug_count ();
 150 #else
 151         return TRUE;
 152 #endif
 153 }
 154
 155 static inline gboolean
 156 amd64_is_near_call (guint8 *code)
 157 {
 158         /* Skip REX */
 159         if ((code [0] >= 0x40) && (code [0] <= 0x4f))
 160                 code += 1;
 161
 162         return code [0] == 0xe8;
 163 }
 164
 165 static inline void
 166 amd64_patch (unsigned char* code, gpointer target)
 167 {
 168         /* Skip REX */
 169         if ((code [0] >= 0x40) && (code [0] <= 0x4f))
 170                 code += 1;
 171
 172         if ((code [0] & 0xf8) == 0xb8) {
 173                 /* amd64_set_reg_template */
 174                 *(guint64*)(code + 1) = (guint64)target;
 175         }
 176         else if (code [0] == 0x8b) {
 177                 /* mov 0(%rip), %dreg */
 178                 *(guint32*)(code + 2) = (guint32)(guint64)target - 7;
 179         }
 180         else if ((code [0] == 0xff) && (code [1] == 0x15)) {
 181                 /* call *<OFFSET>(%rip) */
 182                 *(guint32*)(code + 2) = ((guint32)(guint64)target) - 7;
 183         }
 184         else if ((code [0] == 0xe8)) {
 185                 /* call <DISP> */
 186                 gint64 disp = (guint8*)target - (guint8*)code;
 187                 g_assert (amd64_is_imm32 (disp));
 188                 x86_patch (code, (unsigned char*)target);
 189         }
 190         else
 191                 x86_patch (code, (unsigned char*)target);
 192 }
 193
 194 void
 195 mono_amd64_patch (unsigned char* code, gpointer target)
 196 {
 197         amd64_patch (code, target);
 198 }
 199
 200 typedef enum {
 201         ArgInIReg,
 202         ArgInFloatSSEReg,
 203         ArgInDoubleSSEReg,
 204         ArgOnStack,
 205         ArgValuetypeInReg,
 206         ArgNone /* only in pair_storage */
 207 } ArgStorage;
 208
 209 typedef struct {
 210         gint16 offset;
 211         gint8  reg;
 212         ArgStorage storage;
 213
 214         /* Only if storage == ArgValuetypeInReg */
 215         ArgStorage pair_storage [2];
 216         gint8 pair_regs [2];
 217 } ArgInfo;
 218
 219 typedef struct {
 220         int nargs;
 221         guint32 stack_usage;
 222         guint32 reg_usage;
 223         guint32 freg_usage;
 224         gboolean need_stack_align;
 225         ArgInfo ret;
 226         ArgInfo sig_cookie;
 227         ArgInfo args [1];
 228 } CallInfo;
 229
 230 #define DEBUG(a) if (cfg->verbose_level > 1) a
 231
 232 #define NEW_ICONST(cfg,dest,val) do {   \
 233                 (dest) = mono_mempool_alloc0 ((cfg)->mempool, sizeof (MonoInst));       \
 234                 (dest)->opcode = OP_ICONST;     \
 235                 (dest)->inst_c0 = (val);        \
 236                 (dest)->type = STACK_I4;        \
 237         } while (0)
 238
 239 #define PARAM_REGS 6
 240
 241 static AMD64_Reg_No param_regs [] = { AMD64_RDI, AMD64_RSI, AMD64_RDX, AMD64_RCX, AMD64_R8, AMD64_R9 };
 242
 243 static AMD64_Reg_No return_regs [] = { AMD64_RAX, AMD64_RDX };
 244
 245 static void inline
 246 add_general (guint32 *gr, guint32 *stack_size, ArgInfo *ainfo)
 247 {
 248     ainfo->offset = *stack_size;
 249
 250     if (*gr >= PARAM_REGS) {
 251                 ainfo->storage = ArgOnStack;
 252                 (*stack_size) += sizeof (gpointer);
 253     }
 254     else {
 255                 ainfo->storage = ArgInIReg;
 256                 ainfo->reg = param_regs [*gr];
 257                 (*gr) ++;
 258     }
 259 }
 260
 261 #define FLOAT_PARAM_REGS 8
 262
 263 static void inline
 264 add_float (guint32 *gr, guint32 *stack_size, ArgInfo *ainfo, gboolean is_double)
 265 {
 266     ainfo->offset = *stack_size;
 267
 268     if (*gr >= FLOAT_PARAM_REGS) {
 269                 ainfo->storage = ArgOnStack;
 270                 (*stack_size) += sizeof (gpointer);
 271     }
 272     else {
 273                 /* A double register */
 274                 if (is_double)
 275                         ainfo->storage = ArgInDoubleSSEReg;
 276                 else
 277                         ainfo->storage = ArgInFloatSSEReg;
 278                 ainfo->reg = *gr;
 279                 (*gr) += 1;
 280     }
 281 }
 282
 283 typedef enum ArgumentClass {
 284         ARG_CLASS_NO_CLASS,
 285         ARG_CLASS_MEMORY,
 286         ARG_CLASS_INTEGER,
 287         ARG_CLASS_SSE
 288 } ArgumentClass;
 289
 290 static ArgumentClass
 291 merge_argument_class_from_type (MonoType *type, ArgumentClass class1)
 292 {
 293         ArgumentClass class2 = ARG_CLASS_NO_CLASS;
 294         MonoType *ptype;
 295
 296         ptype = mono_type_get_underlying_type (type);
 297         switch (ptype->type) {
 298         case MONO_TYPE_BOOLEAN:
 299         case MONO_TYPE_CHAR:
 300         case MONO_TYPE_I1:
 301         case MONO_TYPE_U1:
 302         case MONO_TYPE_I2:
 303         case MONO_TYPE_U2:
 304         case MONO_TYPE_I4:
 305         case MONO_TYPE_U4:
 306         case MONO_TYPE_I:
 307         case MONO_TYPE_U:
 308         case MONO_TYPE_STRING:
 309         case MONO_TYPE_OBJECT:
 310         case MONO_TYPE_CLASS:
 311         case MONO_TYPE_SZARRAY:
 312         case MONO_TYPE_PTR:
 313         case MONO_TYPE_FNPTR:
 314         case MONO_TYPE_ARRAY:
 315         case MONO_TYPE_I8:
 316         case MONO_TYPE_U8:
 317                 class2 = ARG_CLASS_INTEGER;
 318                 break;
 319         case MONO_TYPE_R4:
 320         case MONO_TYPE_R8:
 321                 class2 = ARG_CLASS_SSE;
 322                 break;
 323
 324         case MONO_TYPE_TYPEDBYREF:
 325                 g_assert_not_reached ();
 326
 327         case MONO_TYPE_GENERICINST:
 328                 if (!mono_type_generic_inst_is_valuetype (ptype)) {
 329                         class2 = ARG_CLASS_INTEGER;
 330                         break;
 331                 }
 332                 /* fall through */
 333         case MONO_TYPE_VALUETYPE: {
 334                 MonoMarshalType *info = mono_marshal_load_type_info (ptype->data.klass);
 335                 int i;
 336
 337                 for (i = 0; i < info->num_fields; ++i) {
 338                         class2 = class1;
 339                         class2 = merge_argument_class_from_type (info->fields [i].field->type, class2);
 340                 }
 341                 break;
 342         }
 343         default:
 344                 g_assert_not_reached ();
 345         }
 346
 347         /* Merge */
 348         if (class1 == class2)
 349                 ;
 350         else if (class1 == ARG_CLASS_NO_CLASS)
 351                 class1 = class2;
 352         else if ((class1 == ARG_CLASS_MEMORY) || (class2 == ARG_CLASS_MEMORY))
 353                 class1 = ARG_CLASS_MEMORY;
 354         else if ((class1 == ARG_CLASS_INTEGER) || (class2 == ARG_CLASS_INTEGER))
 355                 class1 = ARG_CLASS_INTEGER;
 356         else
 357                 class1 = ARG_CLASS_SSE;
 358
 359         return class1;
 360 }
 361
 362 static void
 363 add_valuetype (MonoGenericSharingContext *gsctx, MonoMethodSignature *sig, ArgInfo *ainfo, MonoType *type,
 364                gboolean is_return,
 365                guint32 *gr, guint32 *fr, guint32 *stack_size)
 366 {
 367         guint32 size, quad, nquads, i;
 368         ArgumentClass args [2];
 369         MonoMarshalType *info;
 370         MonoClass *klass;
 371
 372         klass = mono_class_from_mono_type (type);
 373         if (sig->pinvoke)
 374                 size = mono_type_native_stack_size (&klass->byval_arg, NULL);
 375         else
 376                 size = mini_type_stack_size (gsctx, &klass->byval_arg, NULL);
 377
 378         if (!sig->pinvoke || (size == 0) || (size > 16)) {
 379                 /* Allways pass in memory */
 380                 ainfo->offset = *stack_size;
 381                 *stack_size += ALIGN_TO (size, 8);
 382                 ainfo->storage = ArgOnStack;
 383
 384                 return;
 385         }
 386
 387         /* FIXME: Handle structs smaller than 8 bytes */
 388         //if ((size % 8) != 0)
 389         //      NOT_IMPLEMENTED;
 390
 391         if (size > 8)
 392                 nquads = 2;
 393         else
 394                 nquads = 1;
 395
 396         /*
 397          * Implement the algorithm from section 3.2.3 of the X86_64 ABI.
 398          * The X87 and SSEUP stuff is left out since there are no such types in
 399          * the CLR.
 400          */
 401         info = mono_marshal_load_type_info (klass);
 402         g_assert (info);
 403         if (info->native_size > 16) {
 404                 ainfo->offset = *stack_size;
 405                 *stack_size += ALIGN_TO (info->native_size, 8);
 406                 ainfo->storage = ArgOnStack;
 407
 408                 return;
 409         }
 410
 411         args [0] = ARG_CLASS_NO_CLASS;
 412         args [1] = ARG_CLASS_NO_CLASS;
 413         for (quad = 0; quad < nquads; ++quad) {
 414                 int size;
 415                 guint32 align;
 416                 ArgumentClass class1;
 417
 418                 class1 = ARG_CLASS_NO_CLASS;
 419                 for (i = 0; i < info->num_fields; ++i) {
 420                         size = mono_marshal_type_size (info->fields [i].field->type,
 421                                                                                    info->fields [i].mspec,
 422                                                                                    &align, TRUE, klass->unicode);
 423                         if ((info->fields [i].offset < 8) && (info->fields [i].offset + size) > 8) {
 424                                 /* Unaligned field */
 425                                 NOT_IMPLEMENTED;
 426                         }
 427
 428                         /* Skip fields in other quad */
 429                         if ((quad == 0) && (info->fields [i].offset >= 8))
 430                                 continue;
 431                         if ((quad == 1) && (info->fields [i].offset < 8))
 432                                 continue;
 433
 434                         class1 = merge_argument_class_from_type (info->fields [i].field->type, class1);
 435                 }
 436                 g_assert (class1 != ARG_CLASS_NO_CLASS);
 437                 args [quad] = class1;
 438         }
 439
 440         /* Post merger cleanup */
 441         if ((args [0] == ARG_CLASS_MEMORY) || (args [1] == ARG_CLASS_MEMORY))
 442                 args [0] = args [1] = ARG_CLASS_MEMORY;
 443
 444         /* Allocate registers */
 445         {
 446                 int orig_gr = *gr;
 447                 int orig_fr = *fr;
 448
 449                 ainfo->storage = ArgValuetypeInReg;
 450                 ainfo->pair_storage [0] = ainfo->pair_storage [1] = ArgNone;
 451                 for (quad = 0; quad < nquads; ++quad) {
 452                         switch (args [quad]) {
 453                         case ARG_CLASS_INTEGER:
 454                                 if (*gr >= PARAM_REGS)
 455                                         args [quad] = ARG_CLASS_MEMORY;
 456                                 else {
 457                                         ainfo->pair_storage [quad] = ArgInIReg;
 458                                         if (is_return)
 459                                                 ainfo->pair_regs [quad] = return_regs [*gr];
 460                                         else
 461                                                 ainfo->pair_regs [quad] = param_regs [*gr];
 462                                         (*gr) ++;
 463                                 }
 464                                 break;
 465                         case ARG_CLASS_SSE:
 466                                 if (*fr >= FLOAT_PARAM_REGS)
 467                                         args [quad] = ARG_CLASS_MEMORY;
 468                                 else {
 469                                         ainfo->pair_storage [quad] = ArgInDoubleSSEReg;
 470                                         ainfo->pair_regs [quad] = *fr;
 471                                         (*fr) ++;
 472                                 }
 473                                 break;
 474                         case ARG_CLASS_MEMORY:
 475                                 break;
 476                         default:
 477                                 g_assert_not_reached ();
 478                         }
 479                 }
 480
 481                 if ((args [0] == ARG_CLASS_MEMORY) || (args [1] == ARG_CLASS_MEMORY)) {
 482                         /* Revert possible register assignments */
 483                         *gr = orig_gr;
 484                         *fr = orig_fr;
 485
 486                         ainfo->offset = *stack_size;
 487                         *stack_size += ALIGN_TO (info->native_size, 8);
 488                         ainfo->storage = ArgOnStack;
 489                 }
 490         }
 491 }
 492
 493 /*
 494  * get_call_info:
 495  *
 496  *  Obtain information about a call according to the calling convention.
 497  * For AMD64, see the "System V ABI, x86-64 Architecture Processor Supplement
 498  * Draft Version 0.23" document for more information.
 499  */
 500 static CallInfo*
 501 get_call_info (MonoCompile *cfg, MonoMemPool *mp, MonoMethodSignature *sig, gboolean is_pinvoke)
 502 {
 503         guint32 i, gr, fr;
 504         MonoType *ret_type;
 505         int n = sig->hasthis + sig->param_count;
 506         guint32 stack_size = 0;
 507         CallInfo *cinfo;
 508         MonoGenericSharingContext *gsctx = cfg ? cfg->generic_sharing_context : NULL;
 509
 510         if (mp)
 511                 cinfo = mono_mempool_alloc0 (mp, sizeof (CallInfo) + (sizeof (ArgInfo) * n));
 512         else
 513                 cinfo = g_malloc0 (sizeof (CallInfo) + (sizeof (ArgInfo) * n));
 514
 515         gr = 0;
 516         fr = 0;
 517
 518         /* return value */
 519         {
 520                 ret_type = mono_type_get_underlying_type (sig->ret);
 521                 ret_type = mini_get_basic_type_from_generic (gsctx, ret_type);
 522                 switch (ret_type->type) {
 523                 case MONO_TYPE_BOOLEAN:
 524                 case MONO_TYPE_I1:
 525                 case MONO_TYPE_U1:
 526                 case MONO_TYPE_I2:
 527                 case MONO_TYPE_U2:
 528                 case MONO_TYPE_CHAR:
 529                 case MONO_TYPE_I4:
 530                 case MONO_TYPE_U4:
 531                 case MONO_TYPE_I:
 532                 case MONO_TYPE_U:
 533                 case MONO_TYPE_PTR:
 534                 case MONO_TYPE_FNPTR:
 535                 case MONO_TYPE_CLASS:
 536                 case MONO_TYPE_OBJECT:
 537                 case MONO_TYPE_SZARRAY:
 538                 case MONO_TYPE_ARRAY:
 539                 case MONO_TYPE_STRING:
 540                         cinfo->ret.storage = ArgInIReg;
 541                         cinfo->ret.reg = AMD64_RAX;
 542                         break;
 543                 case MONO_TYPE_U8:
 544                 case MONO_TYPE_I8:
 545                         cinfo->ret.storage = ArgInIReg;
 546                         cinfo->ret.reg = AMD64_RAX;
 547                         break;
 548                 case MONO_TYPE_R4:
 549                         cinfo->ret.storage = ArgInFloatSSEReg;
 550                         cinfo->ret.reg = AMD64_XMM0;
 551                         break;
 552                 case MONO_TYPE_R8:
 553                         cinfo->ret.storage = ArgInDoubleSSEReg;
 554                         cinfo->ret.reg = AMD64_XMM0;
 555                         break;
 556                 case MONO_TYPE_GENERICINST:
 557                         if (!mono_type_generic_inst_is_valuetype (sig->ret)) {
 558                                 cinfo->ret.storage = ArgInIReg;
 559                                 cinfo->ret.reg = AMD64_RAX;
 560                                 break;
 561                         }
 562                         /* fall through */
 563                 case MONO_TYPE_VALUETYPE: {
 564                         guint32 tmp_gr = 0, tmp_fr = 0, tmp_stacksize = 0;
 565
 566                         add_valuetype (gsctx, sig, &cinfo->ret, sig->ret, TRUE, &tmp_gr, &tmp_fr, &tmp_stacksize);
 567                         if (cinfo->ret.storage == ArgOnStack)
 568                                 /* The caller passes the address where the value is stored */
 569                                 add_general (&gr, &stack_size, &cinfo->ret);
 570                         break;
 571                 }
 572                 case MONO_TYPE_TYPEDBYREF:
 573                         /* Same as a valuetype with size 24 */
 574                         add_general (&gr, &stack_size, &cinfo->ret);
 575                         ;
 576                         break;
 577                 case MONO_TYPE_VOID:
 578                         break;
 579                 default:
 580                         g_error ("Can't handle as return value 0x%x", sig->ret->type);
 581                 }
 582         }
 583
 584         /* this */
 585         if (sig->hasthis)
 586                 add_general (&gr, &stack_size, cinfo->args + 0);
 587
 588         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n == 0)) {
 589                 gr = PARAM_REGS;
 590                 fr = FLOAT_PARAM_REGS;
 591
 592                 /* Emit the signature cookie just before the implicit arguments */
 593                 add_general (&gr, &stack_size, &cinfo->sig_cookie);
 594         }
 595
 596         for (i = 0; i < sig->param_count; ++i) {
 597                 ArgInfo *ainfo = &cinfo->args [sig->hasthis + i];
 598                 MonoType *ptype;
 599
 600                 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
 601                         /* We allways pass the sig cookie on the stack for simplicity */
 602                         /*
 603                          * Prevent implicit arguments + the sig cookie from being passed
 604                          * in registers.
 605                          */
 606                         gr = PARAM_REGS;
 607                         fr = FLOAT_PARAM_REGS;
 608
 609                         /* Emit the signature cookie just before the implicit arguments */
 610                         add_general (&gr, &stack_size, &cinfo->sig_cookie);
 611                 }
 612
 613                 if (sig->params [i]->byref) {
 614                         add_general (&gr, &stack_size, ainfo);
 615                         continue;
 616                 }
 617                 ptype = mono_type_get_underlying_type (sig->params [i]);
 618                 ptype = mini_get_basic_type_from_generic (gsctx, ptype);
 619                 switch (ptype->type) {
 620                 case MONO_TYPE_BOOLEAN:
 621                 case MONO_TYPE_I1:
 622                 case MONO_TYPE_U1:
 623                         add_general (&gr, &stack_size, ainfo);
 624                         break;
 625                 case MONO_TYPE_I2:
 626                 case MONO_TYPE_U2:
 627                 case MONO_TYPE_CHAR:
 628                         add_general (&gr, &stack_size, ainfo);
 629                         break;
 630                 case MONO_TYPE_I4:
 631                 case MONO_TYPE_U4:
 632                         add_general (&gr, &stack_size, ainfo);
 633                         break;
 634                 case MONO_TYPE_I:
 635                 case MONO_TYPE_U:
 636                 case MONO_TYPE_PTR:
 637                 case MONO_TYPE_FNPTR:
 638                 case MONO_TYPE_CLASS:
 639                 case MONO_TYPE_OBJECT:
 640                 case MONO_TYPE_STRING:
 641                 case MONO_TYPE_SZARRAY:
 642                 case MONO_TYPE_ARRAY:
 643                         add_general (&gr, &stack_size, ainfo);
 644                         break;
 645                 case MONO_TYPE_GENERICINST:
 646                         if (!mono_type_generic_inst_is_valuetype (ptype)) {
 647                                 add_general (&gr, &stack_size, ainfo);
 648                                 break;
 649                         }
 650                         /* fall through */
 651                 case MONO_TYPE_VALUETYPE:
 652                         add_valuetype (gsctx, sig, ainfo, sig->params [i], FALSE, &gr, &fr, &stack_size);
 653                         break;
 654                 case MONO_TYPE_TYPEDBYREF:
 655                         stack_size += sizeof (MonoTypedRef);
 656                         ainfo->storage = ArgOnStack;
 657                         break;
 658                 case MONO_TYPE_U8:
 659                 case MONO_TYPE_I8:
 660                         add_general (&gr, &stack_size, ainfo);
 661                         break;
 662                 case MONO_TYPE_R4:
 663                         add_float (&fr, &stack_size, ainfo, FALSE);
 664                         break;
 665                 case MONO_TYPE_R8:
 666                         add_float (&fr, &stack_size, ainfo, TRUE);
 667                         break;
 668                 default:
 669                         g_assert_not_reached ();
 670                 }
 671         }
 672
 673         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n > 0) && (sig->sentinelpos == sig->param_count)) {
 674                 gr = PARAM_REGS;
 675                 fr = FLOAT_PARAM_REGS;
 676
 677                 /* Emit the signature cookie just before the implicit arguments */
 678                 add_general (&gr, &stack_size, &cinfo->sig_cookie);
 679         }
 680
 681         if (stack_size & 0x8) {
 682                 /* The AMD64 ABI requires each stack frame to be 16 byte aligned */
 683                 cinfo->need_stack_align = TRUE;
 684                 stack_size += 8;
 685         }
 686
 687         cinfo->stack_usage = stack_size;
 688         cinfo->reg_usage = gr;
 689         cinfo->freg_usage = fr;
 690         return cinfo;
 691 }
 692
 693 /*
 694  * mono_arch_get_argument_info:
 695  * @csig:  a method signature
 696  * @param_count: the number of parameters to consider
 697  * @arg_info: an array to store the result infos
 698  *
 699  * Gathers information on parameters such as size, alignment and
 700  * padding. arg_info should be large enought to hold param_count + 1 entries.
 701  *
 702  * Returns the size of the argument area on the stack.
 703  */
 704 int
 705 mono_arch_get_argument_info (MonoMethodSignature *csig, int param_count, MonoJitArgumentInfo *arg_info)
 706 {
 707         int k;
 708         CallInfo *cinfo = get_call_info (NULL, NULL, csig, FALSE);
 709         guint32 args_size = cinfo->stack_usage;
 710
 711         /* The arguments are saved to a stack area in mono_arch_instrument_prolog */
 712         if (csig->hasthis) {
 713                 arg_info [0].offset = 0;
 714         }
 715
 716         for (k = 0; k < param_count; k++) {
 717                 arg_info [k + 1].offset = ((k + csig->hasthis) * 8);
 718                 /* FIXME: */
 719                 arg_info [k + 1].size = 0;
 720         }
 721
 722         g_free (cinfo);
 723
 724         return args_size;
 725 }
 726
 727 static int
 728 cpuid (int id, int* p_eax, int* p_ebx, int* p_ecx, int* p_edx)
 729 {
 730         return 0;
 731 }
 732
 733 /*
 734  * Initialize the cpu to execute managed code.
 735  */
 736 void
 737 mono_arch_cpu_init (void)
 738 {
 739 #ifndef _MSC_VER
 740         guint16 fpcw;
 741
 742         /* spec compliance requires running with double precision */
 743         __asm__  __volatile__ ("fnstcw %0\n": "=m" (fpcw));
 744         fpcw &= ~X86_FPCW_PRECC_MASK;
 745         fpcw |= X86_FPCW_PREC_DOUBLE;
 746         __asm__  __volatile__ ("fldcw %0\n": : "m" (fpcw));
 747         __asm__  __volatile__ ("fnstcw %0\n": "=m" (fpcw));
 748 #else
 749         _control87 (_PC_53, MCW_PC);
 750 #endif
 751 }
 752
 753 /*
 754  * Initialize architecture specific code.
 755  */
 756 void
 757 mono_arch_init (void)
 758 {
 759         InitializeCriticalSection (&mini_arch_mutex);
 760 }
 761
 762 /*
 763  * Cleanup architecture specific code.
 764  */
 765 void
 766 mono_arch_cleanup (void)
 767 {
 768         DeleteCriticalSection (&mini_arch_mutex);
 769 }
 770
 771 /*
 772  * This function returns the optimizations supported on this cpu.
 773  */
 774 guint32
 775 mono_arch_cpu_optimizazions (guint32 *exclude_mask)
 776 {
 777         int eax, ebx, ecx, edx;
 778         guint32 opts = 0;
 779
 780         /* FIXME: AMD64 */
 781
 782         *exclude_mask = 0;
 783         /* Feature Flags function, flags returned in EDX. */
 784         if (cpuid (1, &eax, &ebx, &ecx, &edx)) {
 785                 if (edx & (1 << 15)) {
 786                         opts |= MONO_OPT_CMOV;
 787                         if (edx & 1)
 788                                 opts |= MONO_OPT_FCMOV;
 789                         else
 790                                 *exclude_mask |= MONO_OPT_FCMOV;
 791                 } else
 792                         *exclude_mask |= MONO_OPT_CMOV;
 793         }
 794         return opts;
 795 }
 796
 797 gboolean
 798 mono_amd64_is_sse2 (void)
 799 {
 800         return use_sse2;
 801 }
 802
 803 GList *
 804 mono_arch_get_allocatable_int_vars (MonoCompile *cfg)
 805 {
 806         GList *vars = NULL;
 807         int i;
 808
 809         for (i = 0; i < cfg->num_varinfo; i++) {
 810                 MonoInst *ins = cfg->varinfo [i];
 811                 MonoMethodVar *vmv = MONO_VARINFO (cfg, i);
 812
 813                 /* unused vars */
 814                 if (vmv->range.first_use.abs_pos >= vmv->range.last_use.abs_pos)
 815                         continue;
 816
 817                 if ((ins->flags & (MONO_INST_IS_DEAD|MONO_INST_VOLATILE|MONO_INST_INDIRECT)) ||
 818                     (ins->opcode != OP_LOCAL && ins->opcode != OP_ARG))
 819                         continue;
 820
 821                 if (mono_is_regsize_var (ins->inst_vtype)) {
 822                         g_assert (MONO_VARINFO (cfg, i)->reg == -1);
 823                         g_assert (i == vmv->idx);
 824                         vars = g_list_prepend (vars, vmv);
 825                 }
 826         }
 827
 828         vars = mono_varlist_sort (cfg, vars, 0);
 829
 830         return vars;
 831 }
 832
 833 /**
 834  * mono_arch_compute_omit_fp:
 835  *
 836  *   Determine whenever the frame pointer can be eliminated.
 837  */
 838 static void
 839 mono_arch_compute_omit_fp (MonoCompile *cfg)
 840 {
 841         MonoMethodSignature *sig;
 842         MonoMethodHeader *header;
 843         int i, locals_size;
 844         CallInfo *cinfo;
 845
 846         if (cfg->arch.omit_fp_computed)
 847                 return;
 848
 849         header = mono_method_get_header (cfg->method);
 850
 851         sig = mono_method_signature (cfg->method);
 852
 853         if (!cfg->arch.cinfo)
 854                 cfg->arch.cinfo = get_call_info (cfg, cfg->mempool, sig, FALSE);
 855         cinfo = cfg->arch.cinfo;
 856
 857         /*
 858          * FIXME: Remove some of the restrictions.
 859          */
 860         cfg->arch.omit_fp = TRUE;
 861         cfg->arch.omit_fp_computed = TRUE;
 862
 863         /* Temporarily disable this when running in the debugger until we have support
 864          * for this in the debugger. */
 865         if (mono_debug_using_mono_debugger ())
 866                 cfg->arch.omit_fp = FALSE;
 867
 868         if (!debug_omit_fp ())
 869                 cfg->arch.omit_fp = FALSE;
 870         /*
 871         if (cfg->method->save_lmf)
 872                 cfg->arch.omit_fp = FALSE;
 873         */
 874         if (cfg->flags & MONO_CFG_HAS_ALLOCA)
 875                 cfg->arch.omit_fp = FALSE;
 876         if (header->num_clauses)
 877                 cfg->arch.omit_fp = FALSE;
 878         if (cfg->param_area)
 879                 cfg->arch.omit_fp = FALSE;
 880         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG))
 881                 cfg->arch.omit_fp = FALSE;
 882         if ((mono_jit_trace_calls != NULL && mono_trace_eval (cfg->method)) ||
 883                 (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE))
 884                 cfg->arch.omit_fp = FALSE;
 885         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
 886                 ArgInfo *ainfo = &cinfo->args [i];
 887
 888                 if (ainfo->storage == ArgOnStack) {
 889                         /*
 890                          * The stack offset can only be determined when the frame
 891                          * size is known.
 892                          */
 893                         cfg->arch.omit_fp = FALSE;
 894                 }
 895         }
 896
 897         locals_size = 0;
 898         for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
 899                 MonoInst *ins = cfg->varinfo [i];
 900                 int ialign;
 901
 902                 locals_size += mono_type_size (ins->inst_vtype, &ialign);
 903         }
 904
 905         if ((cfg->num_varinfo > 10000) || (locals_size >= (1 << 15))) {
 906                 /* Avoid hitting the stack_alloc_size < (1 << 16) assertion in emit_epilog () */
 907                 cfg->arch.omit_fp = FALSE;
 908         }
 909 }
 910
 911 GList *
 912 mono_arch_get_global_int_regs (MonoCompile *cfg)
 913 {
 914         GList *regs = NULL;
 915
 916         mono_arch_compute_omit_fp (cfg);
 917
 918         if (cfg->arch.omit_fp)
 919                 regs = g_list_prepend (regs, (gpointer)AMD64_RBP);
 920
 921         /* We use the callee saved registers for global allocation */
 922         regs = g_list_prepend (regs, (gpointer)AMD64_RBX);
 923         regs = g_list_prepend (regs, (gpointer)AMD64_R12);
 924         regs = g_list_prepend (regs, (gpointer)AMD64_R13);
 925         regs = g_list_prepend (regs, (gpointer)AMD64_R14);
 926         regs = g_list_prepend (regs, (gpointer)AMD64_R15);
 927
 928         return regs;
 929 }
 930
 931 /*
 932  * mono_arch_regalloc_cost:
 933  *
 934  *  Return the cost, in number of memory references, of the action of
 935  * allocating the variable VMV into a register during global register
 936  * allocation.
 937  */
 938 guint32
 939 mono_arch_regalloc_cost (MonoCompile *cfg, MonoMethodVar *vmv)
 940 {
 941         MonoInst *ins = cfg->varinfo [vmv->idx];
 942
 943         if (cfg->method->save_lmf)
 944                 /* The register is already saved */
 945                 /* substract 1 for the invisible store in the prolog */
 946                 return (ins->opcode == OP_ARG) ? 0 : 1;
 947         else
 948                 /* push+pop */
 949                 return (ins->opcode == OP_ARG) ? 1 : 2;
 950 }
 951
 952 void
 953 mono_arch_allocate_vars (MonoCompile *cfg)
 954 {
 955         MonoMethodSignature *sig;
 956         MonoMethodHeader *header;
 957         MonoInst *inst;
 958         int i, offset;
 959         guint32 locals_stack_size, locals_stack_align;
 960         gint32 *offsets;
 961         CallInfo *cinfo;
 962
 963         header = mono_method_get_header (cfg->method);
 964
 965         sig = mono_method_signature (cfg->method);
 966
 967         cinfo = cfg->arch.cinfo;
 968
 969         mono_arch_compute_omit_fp (cfg);
 970
 971         /*
 972          * We use the ABI calling conventions for managed code as well.
 973          * Exception: valuetypes are never passed or returned in registers.
 974          */
 975
 976         if (cfg->arch.omit_fp) {
 977                 cfg->flags |= MONO_CFG_HAS_SPILLUP;
 978                 cfg->frame_reg = AMD64_RSP;
 979                 offset = 0;
 980         } else {
 981                 /* Locals are allocated backwards from %fp */
 982                 cfg->frame_reg = AMD64_RBP;
 983                 offset = 0;
 984         }
 985
 986         cfg->arch.reg_save_area_offset = offset;
 987
 988         /* Reserve space for caller saved registers */
 989         for (i = 0; i < AMD64_NREG; ++i)
 990                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
 991                         offset += sizeof (gpointer);
 992                 }
 993
 994         if (cfg->method->save_lmf) {
 995                 /* Reserve stack space for saving LMF + argument regs */
 996                 guint32 size = sizeof (MonoLMF);
 997
 998                 if (lmf_addr_tls_offset == -1)
 999                         /* Need to save argument regs too */
1000                         size += (AMD64_NREG * 8) + (8 * 8);
1001
1002                 if (cfg->arch.omit_fp) {
1003                         cfg->arch.lmf_offset = offset;
1004                         offset += size;
1005                 }
1006                 else {
1007                         offset += size;
1008                         cfg->arch.lmf_offset = -offset;
1009                 }
1010         }
1011
1012         if (sig->ret->type != MONO_TYPE_VOID) {
1013                 switch (cinfo->ret.storage) {
1014                 case ArgInIReg:
1015                 case ArgInFloatSSEReg:
1016                 case ArgInDoubleSSEReg:
1017                         if ((MONO_TYPE_ISSTRUCT (sig->ret) && !mono_class_from_mono_type (sig->ret)->enumtype) || (sig->ret->type == MONO_TYPE_TYPEDBYREF)) {
1018                                 /* The register is volatile */
1019                                 cfg->ret->opcode = OP_REGOFFSET;
1020                                 cfg->ret->inst_basereg = cfg->frame_reg;
1021                                 if (cfg->arch.omit_fp) {
1022                                         cfg->ret->inst_offset = offset;
1023                                         offset += 8;
1024                                 } else {
1025                                         offset += 8;
1026                                         cfg->ret->inst_offset = -offset;
1027                                 }
1028                         }
1029                         else {
1030                                 cfg->ret->opcode = OP_REGVAR;
1031                                 cfg->ret->inst_c0 = cinfo->ret.reg;
1032                         }
1033                         break;
1034                 case ArgValuetypeInReg:
1035                         /* Allocate a local to hold the result, the epilog will copy it to the correct place */
1036                         g_assert (!cfg->arch.omit_fp);
1037                         offset += 16;
1038                         cfg->ret->opcode = OP_REGOFFSET;
1039                         cfg->ret->inst_basereg = cfg->frame_reg;
1040                         cfg->ret->inst_offset = - offset;
1041                         break;
1042                 default:
1043                         g_assert_not_reached ();
1044                 }
1045                 cfg->ret->dreg = cfg->ret->inst_c0;
1046         }
1047
1048         /* Allocate locals */
1049         offsets = mono_allocate_stack_slots_full (cfg, cfg->arch.omit_fp ? FALSE: TRUE, &locals_stack_size, &locals_stack_align);
1050         if (locals_stack_align) {
1051                 offset += (locals_stack_align - 1);
1052                 offset &= ~(locals_stack_align - 1);
1053         }
1054         for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
1055                 if (offsets [i] != -1) {
1056                         MonoInst *inst = cfg->varinfo [i];
1057                         inst->opcode = OP_REGOFFSET;
1058                         inst->inst_basereg = cfg->frame_reg;
1059                         if (cfg->arch.omit_fp)
1060                                 inst->inst_offset = (offset + offsets [i]);
1061                         else
1062                                 inst->inst_offset = - (offset + offsets [i]);
1063                         //printf ("allocated local %d to ", i); mono_print_tree_nl (inst);
1064                 }
1065         }
1066         offset += locals_stack_size;
1067
1068         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG)) {
1069                 g_assert (!cfg->arch.omit_fp);
1070                 g_assert (cinfo->sig_cookie.storage == ArgOnStack);
1071                 cfg->sig_cookie = cinfo->sig_cookie.offset + ARGS_OFFSET;
1072         }
1073
1074         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
1075                 inst = cfg->args [i];
1076                 if (inst->opcode != OP_REGVAR) {
1077                         ArgInfo *ainfo = &cinfo->args [i];
1078                         gboolean inreg = TRUE;
1079                         MonoType *arg_type;
1080
1081                         if (sig->hasthis && (i == 0))
1082                                 arg_type = &mono_defaults.object_class->byval_arg;
1083                         else
1084                                 arg_type = sig->params [i - sig->hasthis];
1085
1086                         /* FIXME: Allocate volatile arguments to registers */
1087                         if (inst->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT))
1088                                 inreg = FALSE;
1089
1090                         /*
1091                          * Under AMD64, all registers used to pass arguments to functions
1092                          * are volatile across calls.
1093                          * FIXME: Optimize this.
1094                          */
1095                         if ((ainfo->storage == ArgInIReg) || (ainfo->storage == ArgInFloatSSEReg) || (ainfo->storage == ArgInDoubleSSEReg) || (ainfo->storage == ArgValuetypeInReg))
1096                                 inreg = FALSE;
1097
1098                         inst->opcode = OP_REGOFFSET;
1099
1100                         switch (ainfo->storage) {
1101                         case ArgInIReg:
1102                         case ArgInFloatSSEReg:
1103                         case ArgInDoubleSSEReg:
1104                                 inst->opcode = OP_REGVAR;
1105                                 inst->dreg = ainfo->reg;
1106                                 break;
1107                         case ArgOnStack:
1108                                 g_assert (!cfg->arch.omit_fp);
1109                                 inst->opcode = OP_REGOFFSET;
1110                                 inst->inst_basereg = cfg->frame_reg;
1111                                 inst->inst_offset = ainfo->offset + ARGS_OFFSET;
1112                                 break;
1113                         case ArgValuetypeInReg:
1114                                 break;
1115                         default:
1116                                 NOT_IMPLEMENTED;
1117                         }
1118
1119                         if (!inreg && (ainfo->storage != ArgOnStack)) {
1120                                 inst->opcode = OP_REGOFFSET;
1121                                 inst->inst_basereg = cfg->frame_reg;
1122                                 /* These arguments are saved to the stack in the prolog */
1123                                 if (cfg->arch.omit_fp) {
1124                                         inst->inst_offset = offset;
1125                                         offset += (ainfo->storage == ArgValuetypeInReg) ? 2 * sizeof (gpointer) : sizeof (gpointer);
1126                                 } else {
1127                                         offset += (ainfo->storage == ArgValuetypeInReg) ? 2 * sizeof (gpointer) : sizeof (gpointer);
1128                                         inst->inst_offset = - offset;
1129                                 }
1130                         }
1131                 }
1132         }
1133
1134         cfg->stack_offset = offset;
1135 }
1136
1137 void
1138 mono_arch_create_vars (MonoCompile *cfg)
1139 {
1140         MonoMethodSignature *sig;
1141         CallInfo *cinfo;
1142
1143         sig = mono_method_signature (cfg->method);
1144
1145         if (!cfg->arch.cinfo)
1146                 cfg->arch.cinfo = get_call_info (cfg, cfg->mempool, sig, FALSE);
1147         cinfo = cfg->arch.cinfo;
1148
1149         if (cinfo->ret.storage == ArgValuetypeInReg)
1150                 cfg->ret_var_is_local = TRUE;
1151 }
1152
1153 static void
1154 add_outarg_reg (MonoCompile *cfg, MonoCallInst *call, MonoInst *arg, ArgStorage storage, int reg, MonoInst *tree)
1155 {
1156         switch (storage) {
1157         case ArgInIReg:
1158                 arg->opcode = OP_OUTARG_REG;
1159                 arg->inst_left = tree;
1160                 arg->inst_call = call;
1161                 arg->backend.reg3 = reg;
1162                 break;
1163         case ArgInFloatSSEReg:
1164                 arg->opcode = OP_AMD64_OUTARG_XMMREG_R4;
1165                 arg->inst_left = tree;
1166                 arg->inst_call = call;
1167                 arg->backend.reg3 = reg;
1168                 break;
1169         case ArgInDoubleSSEReg:
1170                 arg->opcode = OP_AMD64_OUTARG_XMMREG_R8;
1171                 arg->inst_left = tree;
1172                 arg->inst_call = call;
1173                 arg->backend.reg3 = reg;
1174                 break;
1175         default:
1176                 g_assert_not_reached ();
1177         }
1178 }
1179
1180 /* Fixme: we need an alignment solution for enter_method and mono_arch_call_opcode,
1181  * currently alignment in mono_arch_call_opcode is computed without arch_get_argument_info
1182  */
1183
1184 static int
1185 arg_storage_to_ldind (ArgStorage storage)
1186 {
1187         switch (storage) {
1188         case ArgInIReg:
1189                 return CEE_LDIND_I;
1190         case ArgInDoubleSSEReg:
1191                 return CEE_LDIND_R8;
1192         case ArgInFloatSSEReg:
1193                 return CEE_LDIND_R4;
1194         default:
1195                 g_assert_not_reached ();
1196         }
1197
1198         return -1;
1199 }
1200
1201 static void
1202 emit_sig_cookie (MonoCompile *cfg, MonoCallInst *call, CallInfo *cinfo)
1203 {
1204         MonoInst *arg;
1205         MonoMethodSignature *tmp_sig;
1206         MonoInst *sig_arg;
1207
1208         /* FIXME: Add support for signature tokens to AOT */
1209         cfg->disable_aot = TRUE;
1210
1211         g_assert (cinfo->sig_cookie.storage == ArgOnStack);
1212
1213         /*
1214          * mono_ArgIterator_Setup assumes the signature cookie is
1215          * passed first and all the arguments which were before it are
1216          * passed on the stack after the signature. So compensate by
1217          * passing a different signature.
1218          */
1219         tmp_sig = mono_metadata_signature_dup (call->signature);
1220         tmp_sig->param_count -= call->signature->sentinelpos;
1221         tmp_sig->sentinelpos = 0;
1222         memcpy (tmp_sig->params, call->signature->params + call->signature->sentinelpos, tmp_sig->param_count * sizeof (MonoType*));
1223
1224         MONO_INST_NEW (cfg, sig_arg, OP_ICONST);
1225         sig_arg->inst_p0 = tmp_sig;
1226
1227         MONO_INST_NEW (cfg, arg, OP_OUTARG);
1228         arg->inst_left = sig_arg;
1229         arg->type = STACK_PTR;
1230
1231         /* prepend, so they get reversed */
1232         arg->next = call->out_args;
1233         call->out_args = arg;
1234 }
1235
1236 /*
1237  * take the arguments and generate the arch-specific
1238  * instructions to properly call the function in call.
1239  * This includes pushing, moving arguments to the right register
1240  * etc.
1241  * Issue: who does the spilling if needed, and when?
1242  */
1243 MonoCallInst*
1244 mono_arch_call_opcode (MonoCompile *cfg, MonoBasicBlock* bb, MonoCallInst *call, int is_virtual) {
1245         MonoInst *arg, *in;
1246         MonoMethodSignature *sig;
1247         int i, n, stack_size;
1248         CallInfo *cinfo;
1249         ArgInfo *ainfo;
1250
1251         stack_size = 0;
1252
1253         sig = call->signature;
1254         n = sig->param_count + sig->hasthis;
1255
1256         cinfo = get_call_info (cfg, cfg->mempool, sig, sig->pinvoke);
1257
1258         for (i = 0; i < n; ++i) {
1259                 ainfo = cinfo->args + i;
1260
1261                 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
1262                         /* Emit the signature cookie just before the implicit arguments */
1263                         emit_sig_cookie (cfg, call, cinfo);
1264                 }
1265
1266                 if (is_virtual && i == 0) {
1267                         /* the argument will be attached to the call instruction */
1268                         in = call->args [i];
1269                 } else {
1270                         MONO_INST_NEW (cfg, arg, OP_OUTARG);
1271                         in = call->args [i];
1272                         arg->cil_code = in->cil_code;
1273                         arg->inst_left = in;
1274                         arg->type = in->type;
1275                         /* prepend, so they get reversed */
1276                         arg->next = call->out_args;
1277                         call->out_args = arg;
1278
1279                         if ((i >= sig->hasthis) && (MONO_TYPE_ISSTRUCT(sig->params [i - sig->hasthis]))) {
1280                                 guint32 align;
1281                                 guint32 size;
1282
1283                                 if (sig->params [i - sig->hasthis]->type == MONO_TYPE_TYPEDBYREF) {
1284                                         size = sizeof (MonoTypedRef);
1285                                         align = sizeof (gpointer);
1286                                 }
1287                                 else
1288                                 if (sig->pinvoke)
1289                                         size = mono_type_native_stack_size (&in->klass->byval_arg, &align);
1290                                 else {
1291                                         /*
1292                                          * Other backends use mini_type_stack_size (), but that
1293                                          * aligns the size to 8, which is larger than the size of
1294                                          * the source, leading to reads of invalid memory if the
1295                                          * source is at the end of address space.
1296                                          */
1297                                         size = mono_class_value_size (in->klass, &align);
1298                                 }
1299                                 if (ainfo->storage == ArgValuetypeInReg) {
1300                                         if (ainfo->pair_storage [1] == ArgNone) {
1301                                                 MonoInst *load;
1302
1303                                                 /* Simpler case */
1304
1305                                                 MONO_INST_NEW (cfg, load, arg_storage_to_ldind (ainfo->pair_storage [0]));
1306                                                 load->inst_left = in;
1307
1308                                                 add_outarg_reg (cfg, call, arg, ainfo->pair_storage [0], ainfo->pair_regs [0], load);
1309                                         }
1310                                         else {
1311                                                 /* Trees can't be shared so make a copy */
1312                                                 MonoInst *vtaddr = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1313                                                 MonoInst *load, *load2, *offset_ins;
1314
1315                                                 /* Reg1 */
1316                                                 MONO_INST_NEW (cfg, load, CEE_LDIND_I);
1317                                                 load->ssa_op = MONO_SSA_LOAD;
1318                                                 load->inst_i0 = (cfg)->varinfo [vtaddr->inst_c0];
1319
1320                                                 NEW_ICONST (cfg, offset_ins, 0);
1321                                                 MONO_INST_NEW (cfg, load2, CEE_ADD);
1322                                                 load2->inst_left = load;
1323                                                 load2->inst_right = offset_ins;
1324
1325                                                 MONO_INST_NEW (cfg, load, arg_storage_to_ldind (ainfo->pair_storage [0]));
1326                                                 load->inst_left = load2;
1327
1328                                                 add_outarg_reg (cfg, call, arg, ainfo->pair_storage [0], ainfo->pair_regs [0], load);
1329
1330                                                 /* Reg2 */
1331                                                 MONO_INST_NEW (cfg, load, CEE_LDIND_I);
1332                                                 load->ssa_op = MONO_SSA_LOAD;
1333                                                 load->inst_i0 = (cfg)->varinfo [vtaddr->inst_c0];
1334
1335                                                 NEW_ICONST (cfg, offset_ins, 8);
1336                                                 MONO_INST_NEW (cfg, load2, CEE_ADD);
1337                                                 load2->inst_left = load;
1338                                                 load2->inst_right = offset_ins;
1339
1340                                                 MONO_INST_NEW (cfg, load, arg_storage_to_ldind (ainfo->pair_storage [1]));
1341                                                 load->inst_left = load2;
1342
1343                                                 MONO_INST_NEW (cfg, arg, OP_OUTARG);
1344                                                 arg->cil_code = in->cil_code;
1345                                                 arg->type = in->type;
1346                                                 /* prepend, so they get reversed */
1347                                                 arg->next = call->out_args;
1348                                                 call->out_args = arg;
1349
1350                                                 add_outarg_reg (cfg, call, arg, ainfo->pair_storage [1], ainfo->pair_regs [1], load);
1351
1352                                                 /* Prepend a copy inst */
1353                                                 MONO_INST_NEW (cfg, arg, CEE_STIND_I);
1354                                                 arg->cil_code = in->cil_code;
1355                                                 arg->ssa_op = MONO_SSA_STORE;
1356                                                 arg->inst_left = vtaddr;
1357                                                 arg->inst_right = in;
1358                                                 arg->type = in->type;
1359
1360                                                 /* prepend, so they get reversed */
1361                                                 arg->next = call->out_args;
1362                                                 call->out_args = arg;
1363                                         }
1364                                 }
1365                                 else {
1366                                         arg->opcode = OP_OUTARG_VT;
1367                                         arg->klass = in->klass;
1368                                         arg->backend.is_pinvoke = sig->pinvoke;
1369                                         arg->inst_imm = size;
1370                                 }
1371                         }
1372                         else {
1373                                 switch (ainfo->storage) {
1374                                 case ArgInIReg:
1375                                         add_outarg_reg (cfg, call, arg, ainfo->storage, ainfo->reg, in);
1376                                         break;
1377                                 case ArgInFloatSSEReg:
1378                                 case ArgInDoubleSSEReg:
1379                                         add_outarg_reg (cfg, call, arg, ainfo->storage, ainfo->reg, in);
1380                                         break;
1381                                 case ArgOnStack:
1382                                         arg->opcode = OP_OUTARG;
1383                                         if (!sig->params [i - sig->hasthis]->byref) {
1384                                                 if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R4)
1385                                                         arg->opcode = OP_OUTARG_R4;
1386                                                 else
1387                                                         if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R8)
1388                                                                 arg->opcode = OP_OUTARG_R8;
1389                                         }
1390                                         break;
1391                                 default:
1392                                         g_assert_not_reached ();
1393                                 }
1394                         }
1395                 }
1396         }
1397
1398         /* Handle the case where there are no implicit arguments */
1399         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n == sig->sentinelpos)) {
1400                 emit_sig_cookie (cfg, call, cinfo);
1401         }
1402
1403         if (cinfo->need_stack_align) {
1404                 MONO_INST_NEW (cfg, arg, OP_AMD64_OUTARG_ALIGN_STACK);
1405                 /* prepend, so they get reversed */
1406                 arg->next = call->out_args;
1407                 call->out_args = arg;
1408         }
1409
1410         call->stack_usage = cinfo->stack_usage;
1411         cfg->param_area = MAX (cfg->param_area, call->stack_usage);
1412         cfg->flags |= MONO_CFG_HAS_CALLS;
1413
1414         return call;
1415 }
1416
1417 #define EMIT_COND_BRANCH(ins,cond,sign) \
1418 if (ins->flags & MONO_INST_BRLABEL) { \
1419         if (ins->inst_i0->inst_c0) { \
1420                 x86_branch (code, cond, cfg->native_code + ins->inst_i0->inst_c0, sign); \
1421         } else { \
1422                 mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_LABEL, ins->inst_i0); \
1423                 if ((cfg->opt & MONO_OPT_BRANCH) && \
1424                     x86_is_imm8 (ins->inst_i0->inst_c1 - cpos)) \
1425                         x86_branch8 (code, cond, 0, sign); \
1426                 else \
1427                         x86_branch32 (code, cond, 0, sign); \
1428         } \
1429 } else { \
1430         if (ins->inst_true_bb->native_offset) { \
1431                 x86_branch (code, cond, cfg->native_code + ins->inst_true_bb->native_offset, sign); \
1432         } else { \
1433                 mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_BB, ins->inst_true_bb); \
1434                 if ((cfg->opt & MONO_OPT_BRANCH) && \
1435                     x86_is_imm8 (ins->inst_true_bb->max_offset - cpos)) \
1436                         x86_branch8 (code, cond, 0, sign); \
1437                 else \
1438                         x86_branch32 (code, cond, 0, sign); \
1439         } \
1440 }
1441
1442 /* emit an exception if condition is fail */
1443 #define EMIT_COND_SYSTEM_EXCEPTION(cond,signed,exc_name)            \
1444         do {                                                        \
1445                 MonoInst *tins = mono_branch_optimize_exception_target (cfg, bb, exc_name); \
1446                 if (tins == NULL) {                                                                             \
1447                         mono_add_patch_info (cfg, code - cfg->native_code,   \
1448                                         MONO_PATCH_INFO_EXC, exc_name);  \
1449                         x86_branch32 (code, cond, 0, signed);               \
1450                 } else {        \
1451                         EMIT_COND_BRANCH (tins, cond, signed);  \
1452                 }                       \
1453         } while (0);
1454
1455 #define EMIT_FPCOMPARE(code) do { \
1456         amd64_fcompp (code); \
1457         amd64_fnstsw (code); \
1458 } while (0);
1459
1460 #define EMIT_SSE2_FPFUNC(code, op, dreg, sreg1) do { \
1461     amd64_movsd_membase_reg (code, AMD64_RSP, -8, (sreg1)); \
1462         amd64_fld_membase (code, AMD64_RSP, -8, TRUE); \
1463         amd64_ ##op (code); \
1464         amd64_fst_membase (code, AMD64_RSP, -8, TRUE, TRUE); \
1465         amd64_movsd_reg_membase (code, (dreg), AMD64_RSP, -8); \
1466 } while (0);
1467
1468 static guint8*
1469 emit_call_body (MonoCompile *cfg, guint8 *code, guint32 patch_type, gconstpointer data)
1470 {
1471         mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
1472
1473         /*
1474          * FIXME: Add support for thunks
1475          */
1476         {
1477                 gboolean near_call = FALSE;
1478
1479                 /*
1480                  * Indirect calls are expensive so try to make a near call if possible.
1481                  * The caller memory is allocated by the code manager so it is
1482                  * guaranteed to be at a 32 bit offset.
1483                  */
1484
1485                 if (patch_type != MONO_PATCH_INFO_ABS) {
1486                         /* The target is in memory allocated using the code manager */
1487                         near_call = TRUE;
1488
1489                         if ((patch_type == MONO_PATCH_INFO_METHOD) || (patch_type == MONO_PATCH_INFO_METHOD_JUMP)) {
1490                                 if (((MonoMethod*)data)->klass->image->assembly->aot_module)
1491                                         /* The callee might be an AOT method */
1492                                         near_call = FALSE;
1493                         }
1494
1495                         if (patch_type == MONO_PATCH_INFO_INTERNAL_METHOD) {
1496                                 /*
1497                                  * The call might go directly to a native function without
1498                                  * the wrapper.
1499                                  */
1500                                 MonoJitICallInfo *mi = mono_find_jit_icall_by_name (data);
1501                                 if (mi) {
1502                                         gconstpointer target = mono_icall_get_wrapper (mi);
1503                                         if ((((guint64)target) >> 32) != 0)
1504                                                 near_call = FALSE;
1505                                 }
1506                         }
1507                 }
1508                 else {
1509                         if (mono_find_class_init_trampoline_by_addr (data))
1510                                 near_call = TRUE;
1511                         else {
1512                                 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (data);
1513                                 if (info) {
1514                                         if ((cfg->method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) &&
1515                                                 strstr (cfg->method->name, info->name)) {
1516                                                 /* A call to the wrapped function */
1517                                                 if ((((guint64)data) >> 32) == 0)
1518                                                         near_call = TRUE;
1519                                         }
1520                                         else if (info->func == info->wrapper) {
1521                                                 /* No wrapper */
1522                                                 if ((((guint64)info->func) >> 32) == 0)
1523                                                         near_call = TRUE;
1524                                         }
1525                                         else {
1526                                                 /* See the comment in mono_codegen () */
1527                                                 if ((info->name [0] != 'v') || (strstr (info->name, "ves_array_new_va_") == NULL && strstr (info->name, "ves_array_element_address_") == NULL))
1528                                                         near_call = TRUE;
1529                                         }
1530                                 }
1531                                 else if ((((guint64)data) >> 32) == 0)
1532                                         near_call = TRUE;
1533                         }
1534                 }
1535
1536                 if (cfg->method->dynamic)
1537                         /* These methods are allocated using malloc */
1538                         near_call = FALSE;
1539
1540                 if (cfg->compile_aot)
1541                         near_call = TRUE;
1542
1543 #ifdef MONO_ARCH_NOMAP32BIT
1544                 near_call = FALSE;
1545 #endif
1546
1547                 if (near_call) {
1548                         amd64_call_code (code, 0);
1549                 }
1550                 else {
1551                         amd64_set_reg_template (code, GP_SCRATCH_REG);
1552                         amd64_call_reg (code, GP_SCRATCH_REG);
1553                 }
1554         }
1555
1556         return code;
1557 }
1558
1559 static inline guint8*
1560 emit_call (MonoCompile *cfg, guint8 *code, guint32 patch_type, gconstpointer data)
1561 {
1562         mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
1563
1564         return emit_call_body (cfg, code, patch_type, data);
1565 }
1566
1567 static inline int
1568 store_membase_imm_to_store_membase_reg (int opcode)
1569 {
1570         switch (opcode) {
1571         case OP_STORE_MEMBASE_IMM:
1572                 return OP_STORE_MEMBASE_REG;
1573         case OP_STOREI4_MEMBASE_IMM:
1574                 return OP_STOREI4_MEMBASE_REG;
1575         case OP_STOREI8_MEMBASE_IMM:
1576                 return OP_STOREI8_MEMBASE_REG;
1577         }
1578
1579         return -1;
1580 }
1581
1582 #define INST_IGNORES_CFLAGS(opcode) (!(((opcode) == OP_ADC) || ((opcode) == OP_ADC_IMM) || ((opcode) == OP_IADC) || ((opcode) == OP_IADC_IMM) || ((opcode) == OP_SBB) || ((opcode) == OP_SBB_IMM) || ((opcode) == OP_ISBB) || ((opcode) == OP_ISBB_IMM)))
1583
1584 /*
1585  * peephole_pass_1:
1586  *
1587  *   Perform peephole opts which should/can be performed before local regalloc
1588  */
1589 static void
1590 peephole_pass_1 (MonoCompile *cfg, MonoBasicBlock *bb)
1591 {
1592         MonoInst *ins, *last_ins = NULL;
1593         ins = bb->code;
1594
1595         while (ins) {
1596
1597                 switch (ins->opcode) {
1598                 case OP_ADD_IMM:
1599                 case OP_IADD_IMM:
1600                 case OP_LADD_IMM:
1601                         if ((ins->sreg1 < MONO_MAX_IREGS) && (ins->dreg >= MONO_MAX_IREGS) && (ins->inst_imm > 0)) {
1602                                 /*
1603                                  * X86_LEA is like ADD, but doesn't have the
1604                                  * sreg1==dreg restriction. inst_imm > 0 is needed since LEA sign-extends
1605                                  * its operand to 64 bit.
1606                                  */
1607                                 ins->opcode = OP_X86_LEA_MEMBASE;
1608                                 ins->inst_basereg = ins->sreg1;
1609                                 /* Fall through */
1610                         }
1611                         else
1612                                 break;
1613                 case CEE_XOR:
1614                         if ((ins->sreg1 == ins->sreg2) && (ins->sreg1 == ins->dreg)) {
1615                                 MonoInst *ins2;
1616
1617                                 /*
1618                                  * Replace STORE_MEMBASE_IMM 0 with STORE_MEMBASE_REG since
1619                                  * the latter has length 2-3 instead of 6 (reverse constant
1620                                  * propagation). These instruction sequences are very common
1621                                  * in the initlocals bblock.
1622                                  */
1623                                 for (ins2 = ins->next; ins2; ins2 = ins2->next) {
1624                                         if (((ins2->opcode == OP_STORE_MEMBASE_IMM) || (ins2->opcode == OP_STOREI4_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_IMM) || (ins2->opcode == OP_STORE_MEMBASE_IMM)) && (ins2->inst_imm == 0)) {
1625                                                 ins2->opcode = store_membase_imm_to_store_membase_reg (ins2->opcode);
1626                                                 ins2->sreg1 = ins->dreg;
1627                                         } else if ((ins2->opcode == OP_STOREI1_MEMBASE_IMM) || (ins2->opcode == OP_STOREI2_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_REG) || (ins2->opcode == OP_STORE_MEMBASE_REG)) {
1628                                                 /* Continue */
1629                                         } else if (((ins2->opcode == OP_ICONST) || (ins2->opcode == OP_I8CONST)) && (ins2->dreg == ins->dreg) && (ins2->inst_c0 == 0)) {
1630                                                 NULLIFY_INS (ins2);
1631                                                 /* Continue */
1632                                         } else {
1633                                                 break;
1634                                         }
1635                                 }
1636                         }
1637                         break;
1638                 case OP_COMPARE_IMM:
1639                         /* OP_COMPARE_IMM (reg, 0)
1640                          * -->
1641                          * OP_AMD64_TEST_NULL (reg)
1642                          */
1643                         if (!ins->inst_imm)
1644                                 ins->opcode = OP_AMD64_TEST_NULL;
1645                         break;
1646                 case OP_ICOMPARE_IMM:
1647                         if (!ins->inst_imm)
1648                                 ins->opcode = OP_X86_TEST_NULL;
1649                         break;
1650                 case OP_AMD64_ICOMPARE_MEMBASE_IMM:
1651                         /*
1652                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
1653                          * OP_X86_COMPARE_MEMBASE_IMM offset(basereg), imm
1654                          * -->
1655                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
1656                          * OP_COMPARE_IMM reg, imm
1657                          *
1658                          * Note: if imm = 0 then OP_COMPARE_IMM replaced with OP_X86_TEST_NULL
1659                          */
1660                         if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_REG) &&
1661                             ins->inst_basereg == last_ins->inst_destbasereg &&
1662                             ins->inst_offset == last_ins->inst_offset) {
1663                                         ins->opcode = OP_ICOMPARE_IMM;
1664                                         ins->sreg1 = last_ins->sreg1;
1665
1666                                         /* check if we can remove cmp reg,0 with test null */
1667                                         if (!ins->inst_imm)
1668                                                 ins->opcode = OP_X86_TEST_NULL;
1669                                 }
1670
1671                         break;
1672                 case OP_LOAD_MEMBASE:
1673                 case OP_LOADI4_MEMBASE:
1674                         /*
1675                          * Note: if reg1 = reg2 the load op is removed
1676                          *
1677                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1678                          * OP_LOAD_MEMBASE offset(basereg), reg2
1679                          * -->
1680                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1681                          * OP_MOVE reg1, reg2
1682                          */
1683                         if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_REG
1684                                          || last_ins->opcode == OP_STORE_MEMBASE_REG) &&
1685                             ins->inst_basereg == last_ins->inst_destbasereg &&
1686                             ins->inst_offset == last_ins->inst_offset) {
1687                                 if (ins->dreg == last_ins->sreg1) {
1688                                         last_ins->next = ins->next;
1689                                         ins = ins->next;
1690                                         continue;
1691                                 } else {
1692                                         //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1693                                         ins->opcode = OP_MOVE;
1694                                         ins->sreg1 = last_ins->sreg1;
1695                                 }
1696
1697                         /*
1698                          * Note: reg1 must be different from the basereg in the second load
1699                          * Note: if reg1 = reg2 is equal then second load is removed
1700                          *
1701                          * OP_LOAD_MEMBASE offset(basereg), reg1
1702                          * OP_LOAD_MEMBASE offset(basereg), reg2
1703                          * -->
1704                          * OP_LOAD_MEMBASE offset(basereg), reg1
1705                          * OP_MOVE reg1, reg2
1706                          */
1707                         } if (last_ins && (last_ins->opcode == OP_LOADI4_MEMBASE
1708                                            || last_ins->opcode == OP_LOAD_MEMBASE) &&
1709                               ins->inst_basereg != last_ins->dreg &&
1710                               ins->inst_basereg == last_ins->inst_basereg &&
1711                               ins->inst_offset == last_ins->inst_offset) {
1712
1713                                 if (ins->dreg == last_ins->dreg) {
1714                                         last_ins->next = ins->next;
1715                                         ins = ins->next;
1716                                         continue;
1717                                 } else {
1718                                         ins->opcode = OP_MOVE;
1719                                         ins->sreg1 = last_ins->dreg;
1720                                 }
1721
1722                                 //g_assert_not_reached ();
1723
1724 #if 0
1725                         /*
1726                          * OP_STORE_MEMBASE_IMM imm, offset(basereg)
1727                          * OP_LOAD_MEMBASE offset(basereg), reg
1728                          * -->
1729                          * OP_STORE_MEMBASE_IMM imm, offset(basereg)
1730                          * OP_ICONST reg, imm
1731                          */
1732                         } else if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_IMM
1733                                                 || last_ins->opcode == OP_STORE_MEMBASE_IMM) &&
1734                                    ins->inst_basereg == last_ins->inst_destbasereg &&
1735                                    ins->inst_offset == last_ins->inst_offset) {
1736                                 //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1737                                 ins->opcode = OP_ICONST;
1738                                 ins->inst_c0 = last_ins->inst_imm;
1739                                 g_assert_not_reached (); // check this rule
1740 #endif
1741                         }
1742                         break;
1743                 case OP_LOADI1_MEMBASE:
1744                         /*
1745                          * Note: if reg1 = reg2 the load op is removed
1746                          *
1747                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1748                          * OP_LOAD_MEMBASE offset(basereg), reg2
1749                          * -->
1750                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1751                          * OP_MOVE reg1, reg2
1752                          */
1753                         if (last_ins && (last_ins->opcode == OP_STOREI1_MEMBASE_REG) &&
1754                                         ins->inst_basereg == last_ins->inst_destbasereg &&
1755                                         ins->inst_offset == last_ins->inst_offset) {
1756                                 if (ins->dreg == last_ins->sreg1) {
1757                                         last_ins->next = ins->next;
1758                                         ins = ins->next;
1759                                         continue;
1760                                 } else {
1761                                         //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1762                                         ins->opcode = OP_MOVE;
1763                                         ins->sreg1 = last_ins->sreg1;
1764                                 }
1765                         }
1766                         break;
1767                 case OP_LOADI2_MEMBASE:
1768                         /*
1769                          * Note: if reg1 = reg2 the load op is removed
1770                          *
1771                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1772                          * OP_LOAD_MEMBASE offset(basereg), reg2
1773                          * -->
1774                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1775                          * OP_MOVE reg1, reg2
1776                          */
1777                         if (last_ins && (last_ins->opcode == OP_STOREI2_MEMBASE_REG) &&
1778                                         ins->inst_basereg == last_ins->inst_destbasereg &&
1779                                         ins->inst_offset == last_ins->inst_offset) {
1780                                 if (ins->dreg == last_ins->sreg1) {
1781                                         last_ins->next = ins->next;
1782                                         ins = ins->next;
1783                                         continue;
1784                                 } else {
1785                                         //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1786                                         ins->opcode = OP_MOVE;
1787                                         ins->sreg1 = last_ins->sreg1;
1788                                 }
1789                         }
1790                         break;
1791                 case CEE_CONV_I4:
1792                 case CEE_CONV_U4:
1793                 case OP_MOVE:
1794                 case OP_FMOVE:
1795                         /*
1796                          * Removes:
1797                          *
1798                          * OP_MOVE reg, reg
1799                          */
1800                         if (ins->dreg == ins->sreg1) {
1801                                 if (last_ins)
1802                                         last_ins->next = ins->next;
1803                                 else
1804                                         bb->code = ins->next;
1805                                 ins = ins->next;
1806                                 continue;
1807                         }
1808                         /*
1809                          * Removes:
1810                          *
1811                          * OP_MOVE sreg, dreg
1812                          * OP_MOVE dreg, sreg
1813                          */
1814                         if (last_ins && last_ins->opcode == OP_MOVE &&
1815                             ins->sreg1 == last_ins->dreg &&
1816                             ins->dreg == last_ins->sreg1) {
1817                                 last_ins->next = ins->next;
1818                                 ins = ins->next;
1819                                 continue;
1820                         }
1821                         break;
1822                 }
1823                 last_ins = ins;
1824                 ins = ins->next;
1825         }
1826         bb->last_ins = last_ins;
1827 }
1828
1829 static void
1830 peephole_pass (MonoCompile *cfg, MonoBasicBlock *bb)
1831 {
1832         MonoInst *ins, *last_ins = NULL;
1833         ins = bb->code;
1834
1835         while (ins) {
1836
1837                 switch (ins->opcode) {
1838                 case OP_ICONST:
1839                 case OP_I8CONST:
1840                         /* reg = 0 -> XOR (reg, reg) */
1841                         /* XOR sets cflags on x86, so we cant do it always */
1842                         if (ins->inst_c0 == 0 && (!ins->next || (ins->next && INST_IGNORES_CFLAGS (ins->next->opcode)))) {
1843                                 ins->opcode = OP_LXOR;
1844                                 ins->sreg1 = ins->dreg;
1845                                 ins->sreg2 = ins->dreg;
1846                                 /* Fall through */
1847                         }
1848                         else
1849                                 break;
1850                 case CEE_XOR:
1851                 case OP_LXOR:
1852                         if ((ins->sreg1 == ins->sreg2) && (ins->sreg1 == ins->dreg)) {
1853                                 MonoInst *ins2;
1854
1855                                 /*
1856                                  * Replace STORE_MEMBASE_IMM 0 with STORE_MEMBASE_REG since
1857                                  * the latter has length 2-3 instead of 6 (reverse constant
1858                                  * propagation). These instruction sequences are very common
1859                                  * in the initlocals bblock.
1860                                  */
1861                                 for (ins2 = ins->next; ins2; ins2 = ins2->next) {
1862                                         if (((ins2->opcode == OP_STORE_MEMBASE_IMM) || (ins2->opcode == OP_STOREI4_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_IMM) || (ins2->opcode == OP_STORE_MEMBASE_IMM)) && (ins2->inst_imm == 0)) {
1863                                                 ins2->opcode = store_membase_imm_to_store_membase_reg (ins2->opcode);
1864                                                 ins2->sreg1 = ins->dreg;
1865                                         } else if ((ins2->opcode == OP_STOREI1_MEMBASE_IMM) || (ins2->opcode == OP_STOREI2_MEMBASE_IMM) || (ins2->opcode == OP_STOREI8_MEMBASE_REG) || (ins2->opcode == OP_STORE_MEMBASE_REG)) {
1866                                                 /* Continue */
1867                                         } else if (((ins2->opcode == OP_ICONST) || (ins2->opcode == OP_I8CONST)) && (ins2->dreg == ins->dreg) && (ins2->inst_c0 == 0)) {
1868                                                 NULLIFY_INS (ins2);
1869                                                 /* Continue */
1870                                         } else {
1871                                                 break;
1872                                         }
1873                                 }
1874                         }
1875                         break;
1876                 case OP_IADD_IMM:
1877                         if ((ins->inst_imm == 1) && (ins->dreg == ins->sreg1))
1878                                 ins->opcode = OP_X86_INC_REG;
1879                         break;
1880                 case OP_ISUB_IMM:
1881                         if ((ins->inst_imm == 1) && (ins->dreg == ins->sreg1))
1882                                 ins->opcode = OP_X86_DEC_REG;
1883                         break;
1884                 case OP_MUL_IMM:
1885                         /* remove unnecessary multiplication with 1 */
1886                         if (ins->inst_imm == 1) {
1887                                 if (ins->dreg != ins->sreg1) {
1888                                         ins->opcode = OP_MOVE;
1889                                 } else {
1890                                         last_ins->next = ins->next;
1891                                         ins = ins->next;
1892                                         continue;
1893                                 }
1894                         }
1895                         break;
1896                 case OP_COMPARE_IMM:
1897                         /* OP_COMPARE_IMM (reg, 0)
1898                          * -->
1899                          * OP_AMD64_TEST_NULL (reg)
1900                          */
1901                         if (!ins->inst_imm)
1902                                 ins->opcode = OP_AMD64_TEST_NULL;
1903                         break;
1904                 case OP_ICOMPARE_IMM:
1905                         if (!ins->inst_imm)
1906                                 ins->opcode = OP_X86_TEST_NULL;
1907                         break;
1908                 case OP_AMD64_ICOMPARE_MEMBASE_IMM:
1909                         /*
1910                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
1911                          * OP_X86_COMPARE_MEMBASE_IMM offset(basereg), imm
1912                          * -->
1913                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
1914                          * OP_COMPARE_IMM reg, imm
1915                          *
1916                          * Note: if imm = 0 then OP_COMPARE_IMM replaced with OP_X86_TEST_NULL
1917                          */
1918                         if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_REG) &&
1919                             ins->inst_basereg == last_ins->inst_destbasereg &&
1920                             ins->inst_offset == last_ins->inst_offset) {
1921                                         ins->opcode = OP_ICOMPARE_IMM;
1922                                         ins->sreg1 = last_ins->sreg1;
1923
1924                                         /* check if we can remove cmp reg,0 with test null */
1925                                         if (!ins->inst_imm)
1926                                                 ins->opcode = OP_X86_TEST_NULL;
1927                                 }
1928
1929                         break;
1930                 case OP_LOAD_MEMBASE:
1931                 case OP_LOADI4_MEMBASE:
1932                         /*
1933                          * Note: if reg1 = reg2 the load op is removed
1934                          *
1935                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1936                          * OP_LOAD_MEMBASE offset(basereg), reg2
1937                          * -->
1938                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1939                          * OP_MOVE reg1, reg2
1940                          */
1941                         if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_REG
1942                                          || last_ins->opcode == OP_STORE_MEMBASE_REG) &&
1943                             ins->inst_basereg == last_ins->inst_destbasereg &&
1944                             ins->inst_offset == last_ins->inst_offset) {
1945                                 if (ins->dreg == last_ins->sreg1) {
1946                                         last_ins->next = ins->next;
1947                                         ins = ins->next;
1948                                         continue;
1949                                 } else {
1950                                         //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1951                                         ins->opcode = OP_MOVE;
1952                                         ins->sreg1 = last_ins->sreg1;
1953                                 }
1954
1955                         /*
1956                          * Note: reg1 must be different from the basereg in the second load
1957                          * Note: if reg1 = reg2 is equal then second load is removed
1958                          *
1959                          * OP_LOAD_MEMBASE offset(basereg), reg1
1960                          * OP_LOAD_MEMBASE offset(basereg), reg2
1961                          * -->
1962                          * OP_LOAD_MEMBASE offset(basereg), reg1
1963                          * OP_MOVE reg1, reg2
1964                          */
1965                         } if (last_ins && (last_ins->opcode == OP_LOADI4_MEMBASE
1966                                            || last_ins->opcode == OP_LOAD_MEMBASE) &&
1967                               ins->inst_basereg != last_ins->dreg &&
1968                               ins->inst_basereg == last_ins->inst_basereg &&
1969                               ins->inst_offset == last_ins->inst_offset) {
1970
1971                                 if (ins->dreg == last_ins->dreg) {
1972                                         last_ins->next = ins->next;
1973                                         ins = ins->next;
1974                                         continue;
1975                                 } else {
1976                                         ins->opcode = OP_MOVE;
1977                                         ins->sreg1 = last_ins->dreg;
1978                                 }
1979
1980                                 //g_assert_not_reached ();
1981
1982 #if 0
1983                         /*
1984                          * OP_STORE_MEMBASE_IMM imm, offset(basereg)
1985                          * OP_LOAD_MEMBASE offset(basereg), reg
1986                          * -->
1987                          * OP_STORE_MEMBASE_IMM imm, offset(basereg)
1988                          * OP_ICONST reg, imm
1989                          */
1990                         } else if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_IMM
1991                                                 || last_ins->opcode == OP_STORE_MEMBASE_IMM) &&
1992                                    ins->inst_basereg == last_ins->inst_destbasereg &&
1993                                    ins->inst_offset == last_ins->inst_offset) {
1994                                 //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1995                                 ins->opcode = OP_ICONST;
1996                                 ins->inst_c0 = last_ins->inst_imm;
1997                                 g_assert_not_reached (); // check this rule
1998 #endif
1999                         }
2000                         break;
2001                 case OP_LOADI1_MEMBASE:
2002                         /*
2003                          * Note: if reg1 = reg2 the load op is removed
2004                          *
2005                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
2006                          * OP_LOAD_MEMBASE offset(basereg), reg2
2007                          * -->
2008                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
2009                          * OP_MOVE reg1, reg2
2010                          */
2011                         if (last_ins && (last_ins->opcode == OP_STOREI1_MEMBASE_REG) &&
2012                                         ins->inst_basereg == last_ins->inst_destbasereg &&
2013                                         ins->inst_offset == last_ins->inst_offset) {
2014                                 if (ins->dreg == last_ins->sreg1) {
2015                                         last_ins->next = ins->next;
2016                                         ins = ins->next;
2017                                         continue;
2018                                 } else {
2019                                         //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
2020                                         ins->opcode = OP_MOVE;
2021                                         ins->sreg1 = last_ins->sreg1;
2022                                 }
2023                         }
2024                         break;
2025                 case OP_LOADI2_MEMBASE:
2026                         /*
2027                          * Note: if reg1 = reg2 the load op is removed
2028                          *
2029                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
2030                          * OP_LOAD_MEMBASE offset(basereg), reg2
2031                          * -->
2032                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
2033                          * OP_MOVE reg1, reg2
2034                          */
2035                         if (last_ins && (last_ins->opcode == OP_STOREI2_MEMBASE_REG) &&
2036                                         ins->inst_basereg == last_ins->inst_destbasereg &&
2037                                         ins->inst_offset == last_ins->inst_offset) {
2038                                 if (ins->dreg == last_ins->sreg1) {
2039                                         last_ins->next = ins->next;
2040                                         ins = ins->next;
2041                                         continue;
2042                                 } else {
2043                                         //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
2044                                         ins->opcode = OP_MOVE;
2045                                         ins->sreg1 = last_ins->sreg1;
2046                                 }
2047                         }
2048                         break;
2049                 case CEE_CONV_I4:
2050                 case CEE_CONV_U4:
2051                 case OP_MOVE:
2052                 case OP_FMOVE:
2053                         /*
2054                          * Removes:
2055                          *
2056                          * OP_MOVE reg, reg
2057                          */
2058                         if (ins->dreg == ins->sreg1) {
2059                                 if (last_ins)
2060                                         last_ins->next = ins->next;
2061                                 else
2062                                         bb->code = ins->next;
2063                                 ins = ins->next;
2064                                 continue;
2065                         }
2066                         /*
2067                          * Removes:
2068                          *
2069                          * OP_MOVE sreg, dreg
2070                          * OP_MOVE dreg, sreg
2071                          */
2072                         if (last_ins && last_ins->opcode == OP_MOVE &&
2073                             ins->sreg1 == last_ins->dreg &&
2074                             ins->dreg == last_ins->sreg1) {
2075                                 last_ins->next = ins->next;
2076                                 ins = ins->next;
2077                                 continue;
2078                         }
2079                         break;
2080                 }
2081                 last_ins = ins;
2082                 ins = ins->next;
2083         }
2084         bb->last_ins = last_ins;
2085 }
2086
2087 static void
2088 insert_after_ins (MonoBasicBlock *bb, MonoInst *ins, MonoInst *to_insert)
2089 {
2090         if (ins == NULL) {
2091                 ins = bb->code;
2092                 bb->code = to_insert;
2093                 to_insert->next = ins;
2094         }
2095         else {
2096                 to_insert->next = ins->next;
2097                 ins->next = to_insert;
2098         }
2099 }
2100
2101 #define NEW_INS(cfg,dest,op) do {       \
2102                 (dest) = mono_mempool_alloc0 ((cfg)->mempool, sizeof (MonoInst));       \
2103                 (dest)->opcode = (op);  \
2104         insert_after_ins (bb, last_ins, (dest)); \
2105         } while (0)
2106
2107 /*
2108  * mono_arch_lowering_pass:
2109  *
2110  *  Converts complex opcodes into simpler ones so that each IR instruction
2111  * corresponds to one machine instruction.
2112  */
2113 static void
2114 mono_arch_lowering_pass (MonoCompile *cfg, MonoBasicBlock *bb)
2115 {
2116         MonoInst *ins, *temp, *last_ins = NULL;
2117         ins = bb->code;
2118
2119         if (bb->max_vreg > cfg->rs->next_vreg)
2120                 cfg->rs->next_vreg = bb->max_vreg;
2121
2122         /*
2123          * FIXME: Need to add more instructions, but the current machine
2124          * description can't model some parts of the composite instructions like
2125          * cdq.
2126          */
2127         while (ins) {
2128                 switch (ins->opcode) {
2129                 case OP_DIV_IMM:
2130                 case OP_REM_IMM:
2131                 case OP_IDIV_IMM:
2132                 case OP_IREM_IMM:
2133                         NEW_INS (cfg, temp, OP_ICONST);
2134                         temp->inst_c0 = ins->inst_imm;
2135                         temp->dreg = mono_regstate_next_int (cfg->rs);
2136                         switch (ins->opcode) {
2137                         case OP_DIV_IMM:
2138                                 ins->opcode = OP_LDIV;
2139                                 break;
2140                         case OP_REM_IMM:
2141                                 ins->opcode = OP_LREM;
2142                                 break;
2143                         case OP_IDIV_IMM:
2144                                 ins->opcode = OP_IDIV;
2145                                 break;
2146                         case OP_IREM_IMM:
2147                                 ins->opcode = OP_IREM;
2148                                 break;
2149                         }
2150                         ins->sreg2 = temp->dreg;
2151                         break;
2152                 case OP_COMPARE_IMM:
2153                         if (!amd64_is_imm32 (ins->inst_imm)) {
2154                                 NEW_INS (cfg, temp, OP_I8CONST);
2155                                 temp->inst_c0 = ins->inst_imm;
2156                                 temp->dreg = mono_regstate_next_int (cfg->rs);
2157                                 ins->opcode = OP_COMPARE;
2158                                 ins->sreg2 = temp->dreg;
2159                         }
2160                         break;
2161                 case OP_LOAD_MEMBASE:
2162                 case OP_LOADI8_MEMBASE:
2163                         if (!amd64_is_imm32 (ins->inst_offset)) {
2164                                 NEW_INS (cfg, temp, OP_I8CONST);
2165                                 temp->inst_c0 = ins->inst_offset;
2166                                 temp->dreg = mono_regstate_next_int (cfg->rs);
2167                                 ins->opcode = OP_AMD64_LOADI8_MEMINDEX;
2168                                 ins->inst_indexreg = temp->dreg;
2169                         }
2170                         break;
2171                 case OP_STORE_MEMBASE_IMM:
2172                 case OP_STOREI8_MEMBASE_IMM:
2173                         if (!amd64_is_imm32 (ins->inst_imm)) {
2174                                 NEW_INS (cfg, temp, OP_I8CONST);
2175                                 temp->inst_c0 = ins->inst_imm;
2176                                 temp->dreg = mono_regstate_next_int (cfg->rs);
2177                                 ins->opcode = OP_STOREI8_MEMBASE_REG;
2178                                 ins->sreg1 = temp->dreg;
2179                         }
2180                         break;
2181                 default:
2182                         break;
2183                 }
2184                 last_ins = ins;
2185                 ins = ins->next;
2186         }
2187         bb->last_ins = last_ins;
2188
2189         bb->max_vreg = cfg->rs->next_vreg;
2190 }
2191
2192 static const int
2193 branch_cc_table [] = {
2194         X86_CC_EQ, X86_CC_GE, X86_CC_GT, X86_CC_LE, X86_CC_LT,
2195         X86_CC_NE, X86_CC_GE, X86_CC_GT, X86_CC_LE, X86_CC_LT,
2196         X86_CC_O, X86_CC_NO, X86_CC_C, X86_CC_NC
2197 };
2198
2199 /* Maps CMP_... constants to X86_CC_... constants */
2200 static const int
2201 cc_table [] = {
2202         X86_CC_EQ, X86_CC_NE, X86_CC_LE, X86_CC_GE, X86_CC_LT, X86_CC_GT,
2203         X86_CC_LE, X86_CC_GE, X86_CC_LT, X86_CC_GT
2204 };
2205
2206 static const int
2207 cc_signed_table [] = {
2208         TRUE, TRUE, TRUE, TRUE, TRUE, TRUE,
2209         FALSE, FALSE, FALSE, FALSE
2210 };
2211
2212 /*#include "cprop.c"*/
2213
2214 /*
2215  * Local register allocation.
2216  * We first scan the list of instructions and we save the liveness info of
2217  * each register (when the register is first used, when it's value is set etc.).
2218  * We also reverse the list of instructions (in the InstList list) because assigning
2219  * registers backwards allows for more tricks to be used.
2220  */
2221 void
2222 mono_arch_local_regalloc (MonoCompile *cfg, MonoBasicBlock *bb)
2223 {
2224         if (!bb->code)
2225                 return;
2226
2227         mono_arch_lowering_pass (cfg, bb);
2228
2229         if (cfg->opt & MONO_OPT_PEEPHOLE)
2230                 peephole_pass_1 (cfg, bb);
2231
2232         mono_local_regalloc (cfg, bb);
2233 }
2234
2235 static unsigned char*
2236 emit_float_to_int (MonoCompile *cfg, guchar *code, int dreg, int sreg, int size, gboolean is_signed)
2237 {
2238         if (use_sse2) {
2239                 amd64_sse_cvttsd2si_reg_reg (code, dreg, sreg);
2240         }
2241         else {
2242                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 16);
2243                 x86_fnstcw_membase(code, AMD64_RSP, 0);
2244                 amd64_mov_reg_membase (code, dreg, AMD64_RSP, 0, 2);
2245                 amd64_alu_reg_imm (code, X86_OR, dreg, 0xc00);
2246                 amd64_mov_membase_reg (code, AMD64_RSP, 2, dreg, 2);
2247                 amd64_fldcw_membase (code, AMD64_RSP, 2);
2248                 amd64_push_reg (code, AMD64_RAX); // SP = SP - 8
2249                 amd64_fist_pop_membase (code, AMD64_RSP, 0, size == 8);
2250                 amd64_pop_reg (code, dreg);
2251                 amd64_fldcw_membase (code, AMD64_RSP, 0);
2252                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 16);
2253         }
2254
2255         if (size == 1)
2256                 amd64_widen_reg (code, dreg, dreg, is_signed, FALSE);
2257         else if (size == 2)
2258                 amd64_widen_reg (code, dreg, dreg, is_signed, TRUE);
2259         return code;
2260 }
2261
2262 static unsigned char*
2263 mono_emit_stack_alloc (guchar *code, MonoInst* tree)
2264 {
2265         int sreg = tree->sreg1;
2266         int need_touch = FALSE;
2267
2268 #if defined(PLATFORM_WIN32) || defined(MONO_ARCH_SIGSEGV_ON_ALTSTACK)
2269         if (!tree->flags & MONO_INST_INIT)
2270                 need_touch = TRUE;
2271 #endif
2272
2273         if (need_touch) {
2274                 guint8* br[5];
2275
2276                 /*
2277                  * Under Windows:
2278                  * If requested stack size is larger than one page,
2279                  * perform stack-touch operation
2280                  */
2281                 /*
2282                  * Generate stack probe code.
2283                  * Under Windows, it is necessary to allocate one page at a time,
2284                  * "touching" stack after each successful sub-allocation. This is
2285                  * because of the way stack growth is implemented - there is a
2286                  * guard page before the lowest stack page that is currently commited.
2287                  * Stack normally grows sequentially so OS traps access to the
2288                  * guard page and commits more pages when needed.
2289                  */
2290                 amd64_test_reg_imm (code, sreg, ~0xFFF);
2291                 br[0] = code; x86_branch8 (code, X86_CC_Z, 0, FALSE);
2292
2293                 br[2] = code; /* loop */
2294                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 0x1000);
2295                 amd64_test_membase_reg (code, AMD64_RSP, 0, AMD64_RSP);
2296                 amd64_alu_reg_imm (code, X86_SUB, sreg, 0x1000);
2297                 amd64_alu_reg_imm (code, X86_CMP, sreg, 0x1000);
2298                 br[3] = code; x86_branch8 (code, X86_CC_AE, 0, FALSE);
2299                 amd64_patch (br[3], br[2]);
2300                 amd64_test_reg_reg (code, sreg, sreg);
2301                 br[4] = code; x86_branch8 (code, X86_CC_Z, 0, FALSE);
2302                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, sreg);
2303
2304                 br[1] = code; x86_jump8 (code, 0);
2305
2306                 amd64_patch (br[0], code);
2307                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, sreg);
2308                 amd64_patch (br[1], code);
2309                 amd64_patch (br[4], code);
2310         }
2311         else
2312                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, tree->sreg1);
2313
2314         if (tree->flags & MONO_INST_INIT) {
2315                 int offset = 0;
2316                 if (tree->dreg != AMD64_RAX && sreg != AMD64_RAX) {
2317                         amd64_push_reg (code, AMD64_RAX);
2318                         offset += 8;
2319                 }
2320                 if (tree->dreg != AMD64_RCX && sreg != AMD64_RCX) {
2321                         amd64_push_reg (code, AMD64_RCX);
2322                         offset += 8;
2323                 }
2324                 if (tree->dreg != AMD64_RDI && sreg != AMD64_RDI) {
2325                         amd64_push_reg (code, AMD64_RDI);
2326                         offset += 8;
2327                 }
2328
2329                 amd64_shift_reg_imm (code, X86_SHR, sreg, 3);
2330                 if (sreg != AMD64_RCX)
2331                         amd64_mov_reg_reg (code, AMD64_RCX, sreg, 8);
2332                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
2333
2334                 amd64_lea_membase (code, AMD64_RDI, AMD64_RSP, offset);
2335                 amd64_cld (code);
2336                 amd64_prefix (code, X86_REP_PREFIX);
2337                 amd64_stosl (code);
2338
2339                 if (tree->dreg != AMD64_RDI && sreg != AMD64_RDI)
2340                         amd64_pop_reg (code, AMD64_RDI);
2341                 if (tree->dreg != AMD64_RCX && sreg != AMD64_RCX)
2342                         amd64_pop_reg (code, AMD64_RCX);
2343                 if (tree->dreg != AMD64_RAX && sreg != AMD64_RAX)
2344                         amd64_pop_reg (code, AMD64_RAX);
2345         }
2346         return code;
2347 }
2348
2349 static guint8*
2350 emit_move_return_value (MonoCompile *cfg, MonoInst *ins, guint8 *code)
2351 {
2352         CallInfo *cinfo;
2353         guint32 quad;
2354
2355         /* Move return value to the target register */
2356         /* FIXME: do this in the local reg allocator */
2357         switch (ins->opcode) {
2358         case CEE_CALL:
2359         case OP_CALL_REG:
2360         case OP_CALL_MEMBASE:
2361         case OP_LCALL:
2362         case OP_LCALL_REG:
2363         case OP_LCALL_MEMBASE:
2364                 g_assert (ins->dreg == AMD64_RAX);
2365                 break;
2366         case OP_FCALL:
2367         case OP_FCALL_REG:
2368         case OP_FCALL_MEMBASE:
2369                 if (((MonoCallInst*)ins)->signature->ret->type == MONO_TYPE_R4) {
2370                         if (use_sse2)
2371                                 amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, AMD64_XMM0);
2372                         else {
2373                                 /* FIXME: optimize this */
2374                                 amd64_movss_membase_reg (code, AMD64_RSP, -8, AMD64_XMM0);
2375                                 amd64_fld_membase (code, AMD64_RSP, -8, FALSE);
2376                         }
2377                 }
2378                 else {
2379                         if (use_sse2) {
2380                                 if (ins->dreg != AMD64_XMM0)
2381                                         amd64_sse_movsd_reg_reg (code, ins->dreg, AMD64_XMM0);
2382                         }
2383                         else {
2384                                 /* FIXME: optimize this */
2385                                 amd64_movsd_membase_reg (code, AMD64_RSP, -8, AMD64_XMM0);
2386                                 amd64_fld_membase (code, AMD64_RSP, -8, TRUE);
2387                         }
2388                 }
2389                 break;
2390         case OP_VCALL:
2391         case OP_VCALL_REG:
2392         case OP_VCALL_MEMBASE:
2393                 cinfo = get_call_info (cfg, cfg->mempool, ((MonoCallInst*)ins)->signature, FALSE);
2394                 if (cinfo->ret.storage == ArgValuetypeInReg) {
2395                         /* Pop the destination address from the stack */
2396                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
2397                         amd64_pop_reg (code, AMD64_RCX);
2398
2399                         for (quad = 0; quad < 2; quad ++) {
2400                                 switch (cinfo->ret.pair_storage [quad]) {
2401                                 case ArgInIReg:
2402                                         amd64_mov_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad], 8);
2403                                         break;
2404                                 case ArgInFloatSSEReg:
2405                                         amd64_movss_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad]);
2406                                         break;
2407                                 case ArgInDoubleSSEReg:
2408                                         amd64_movsd_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad]);
2409                                         break;
2410                                 case ArgNone:
2411                                         break;
2412                                 default:
2413                                         NOT_IMPLEMENTED;
2414                                 }
2415                         }
2416                 }
2417                 break;
2418         }
2419
2420         return code;
2421 }
2422
2423 /*
2424  * emit_tls_get:
2425  * @code: buffer to store code to
2426  * @dreg: hard register where to place the result
2427  * @tls_offset: offset info
2428  *
2429  * emit_tls_get emits in @code the native code that puts in the dreg register
2430  * the item in the thread local storage identified by tls_offset.
2431  *
2432  * Returns: a pointer to the end of the stored code
2433  */
2434 static guint8*
2435 emit_tls_get (guint8* code, int dreg, int tls_offset)
2436 {
2437         if (optimize_for_xen) {
2438                 x86_prefix (code, X86_FS_PREFIX);
2439                 amd64_mov_reg_mem (code, dreg, 0, 8);
2440                 amd64_mov_reg_membase (code, dreg, dreg, tls_offset, 8);
2441         } else {
2442                 x86_prefix (code, X86_FS_PREFIX);
2443                 amd64_mov_reg_mem (code, dreg, tls_offset, 8);
2444         }
2445         return code;
2446 }
2447
2448 /*
2449  * emit_load_volatile_arguments:
2450  *
2451  *  Load volatile arguments from the stack to the original input registers.
2452  * Required before a tail call.
2453  */
2454 static guint8*
2455 emit_load_volatile_arguments (MonoCompile *cfg, guint8 *code)
2456 {
2457         MonoMethod *method = cfg->method;
2458         MonoMethodSignature *sig;
2459         MonoInst *inst;
2460         CallInfo *cinfo;
2461         guint32 i;
2462
2463         /* FIXME: Generate intermediate code instead */
2464
2465         sig = mono_method_signature (method);
2466
2467         cinfo = cfg->arch.cinfo;
2468
2469         /* This is the opposite of the code in emit_prolog */
2470
2471         if (sig->ret->type != MONO_TYPE_VOID) {
2472                 if ((cinfo->ret.storage == ArgInIReg) && (cfg->ret->opcode != OP_REGVAR)) {
2473                         amd64_mov_reg_membase (code, cinfo->ret.reg, cfg->ret->inst_basereg, cfg->ret->inst_offset, 8);
2474                 }
2475         }
2476
2477         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
2478                 ArgInfo *ainfo = cinfo->args + i;
2479                 MonoType *arg_type;
2480                 inst = cfg->args [i];
2481
2482                 if (sig->hasthis && (i == 0))
2483                         arg_type = &mono_defaults.object_class->byval_arg;
2484                 else
2485                         arg_type = sig->params [i - sig->hasthis];
2486
2487                 if (inst->opcode != OP_REGVAR) {
2488                         switch (ainfo->storage) {
2489                         case ArgInIReg: {
2490                                 guint32 size = 8;
2491
2492                                 /* FIXME: I1 etc */
2493                                 amd64_mov_reg_membase (code, ainfo->reg, inst->inst_basereg, inst->inst_offset, size);
2494                                 break;
2495                         }
2496                         case ArgInFloatSSEReg:
2497                                 amd64_movss_reg_membase (code, ainfo->reg, inst->inst_basereg, inst->inst_offset);
2498                                 break;
2499                         case ArgInDoubleSSEReg:
2500                                 amd64_movsd_reg_membase (code, ainfo->reg, inst->inst_basereg, inst->inst_offset);
2501                                 break;
2502                         default:
2503                                 break;
2504                         }
2505                 }
2506                 else {
2507                         g_assert (ainfo->storage == ArgInIReg);
2508
2509                         amd64_mov_reg_reg (code, ainfo->reg, inst->dreg, 8);
2510                 }
2511         }
2512
2513         return code;
2514 }
2515
2516 #define REAL_PRINT_REG(text,reg) \
2517 mono_assert (reg >= 0); \
2518 amd64_push_reg (code, AMD64_RAX); \
2519 amd64_push_reg (code, AMD64_RDX); \
2520 amd64_push_reg (code, AMD64_RCX); \
2521 amd64_push_reg (code, reg); \
2522 amd64_push_imm (code, reg); \
2523 amd64_push_imm (code, text " %d %p\n"); \
2524 amd64_mov_reg_imm (code, AMD64_RAX, printf); \
2525 amd64_call_reg (code, AMD64_RAX); \
2526 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 3*4); \
2527 amd64_pop_reg (code, AMD64_RCX); \
2528 amd64_pop_reg (code, AMD64_RDX); \
2529 amd64_pop_reg (code, AMD64_RAX);
2530
2531 /* benchmark and set based on cpu */
2532 #define LOOP_ALIGNMENT 8
2533 #define bb_is_loop_start(bb) ((bb)->loop_body_start && (bb)->nesting)
2534
2535 void
2536 mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
2537 {
2538         MonoInst *ins;
2539         MonoCallInst *call;
2540         guint offset;
2541         guint8 *code = cfg->native_code + cfg->code_len;
2542         MonoInst *last_ins = NULL;
2543         guint last_offset = 0;
2544         int max_len, cpos;
2545
2546         if (cfg->opt & MONO_OPT_PEEPHOLE)
2547                 peephole_pass (cfg, bb);
2548
2549         if (cfg->opt & MONO_OPT_LOOP) {
2550                 int pad, align = LOOP_ALIGNMENT;
2551                 /* set alignment depending on cpu */
2552                 if (bb_is_loop_start (bb) && (pad = (cfg->code_len & (align - 1)))) {
2553                         pad = align - pad;
2554                         /*g_print ("adding %d pad at %x to loop in %s\n", pad, cfg->code_len, cfg->method->name);*/
2555                         amd64_padding (code, pad);
2556                         cfg->code_len += pad;
2557                         bb->native_offset = cfg->code_len;
2558                 }
2559         }
2560
2561         if (cfg->verbose_level > 2)
2562                 g_print ("Basic block %d starting at offset 0x%x\n", bb->block_num, bb->native_offset);
2563
2564         cpos = bb->max_offset;
2565
2566         if (cfg->prof_options & MONO_PROFILE_COVERAGE) {
2567                 MonoProfileCoverageInfo *cov = cfg->coverage_info;
2568                 g_assert (!cfg->compile_aot);
2569                 cpos += 6;
2570
2571                 cov->data [bb->dfn].cil_code = bb->cil_code;
2572                 amd64_mov_reg_imm (code, AMD64_R11, (guint64)&cov->data [bb->dfn].count);
2573                 /* this is not thread save, but good enough */
2574                 amd64_inc_membase (code, AMD64_R11, 0);
2575         }
2576
2577         offset = code - cfg->native_code;
2578
2579         mono_debug_open_block (cfg, bb, offset);
2580
2581         ins = bb->code;
2582         while (ins) {
2583                 offset = code - cfg->native_code;
2584
2585                 max_len = ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
2586
2587                 if (offset > (cfg->code_size - max_len - 16)) {
2588                         cfg->code_size *= 2;
2589                         cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
2590                         code = cfg->native_code + offset;
2591                         mono_jit_stats.code_reallocs++;
2592                 }
2593
2594                 mono_debug_record_line_number (cfg, ins, offset);
2595
2596                 switch (ins->opcode) {
2597                 case OP_BIGMUL:
2598                         amd64_mul_reg (code, ins->sreg2, TRUE);
2599                         break;
2600                 case OP_BIGMUL_UN:
2601                         amd64_mul_reg (code, ins->sreg2, FALSE);
2602                         break;
2603                 case OP_X86_SETEQ_MEMBASE:
2604                         amd64_set_membase (code, X86_CC_EQ, ins->inst_basereg, ins->inst_offset, TRUE);
2605                         break;
2606                 case OP_STOREI1_MEMBASE_IMM:
2607                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 1);
2608                         break;
2609                 case OP_STOREI2_MEMBASE_IMM:
2610                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 2);
2611                         break;
2612                 case OP_STOREI4_MEMBASE_IMM:
2613                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 4);
2614                         break;
2615                 case OP_STOREI1_MEMBASE_REG:
2616                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 1);
2617                         break;
2618                 case OP_STOREI2_MEMBASE_REG:
2619                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 2);
2620                         break;
2621                 case OP_STORE_MEMBASE_REG:
2622                 case OP_STOREI8_MEMBASE_REG:
2623                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 8);
2624                         break;
2625                 case OP_STOREI4_MEMBASE_REG:
2626                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 4);
2627                         break;
2628                 case OP_STORE_MEMBASE_IMM:
2629                 case OP_STOREI8_MEMBASE_IMM:
2630                         g_assert (amd64_is_imm32 (ins->inst_imm));
2631                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 8);
2632                         break;
2633                 case CEE_LDIND_I:
2634                         amd64_mov_reg_mem (code, ins->dreg, (gssize)ins->inst_p0, sizeof (gpointer));
2635                         break;
2636                 case CEE_LDIND_I4:
2637                         amd64_mov_reg_mem (code, ins->dreg, (gssize)ins->inst_p0, 4);
2638                         break;
2639                 case CEE_LDIND_U4:
2640                         amd64_mov_reg_mem (code, ins->dreg, (gssize)ins->inst_p0, 4);
2641                         break;
2642                 case OP_LOADU4_MEM:
2643                         amd64_mov_reg_imm (code, ins->dreg, ins->inst_p0);
2644                         amd64_mov_reg_membase (code, ins->dreg, ins->dreg, 0, 4);
2645                         break;
2646                 case OP_LOAD_MEMBASE:
2647                 case OP_LOADI8_MEMBASE:
2648                         g_assert (amd64_is_imm32 (ins->inst_offset));
2649                         amd64_mov_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, sizeof (gpointer));
2650                         break;
2651                 case OP_LOADI4_MEMBASE:
2652                         amd64_movsxd_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
2653                         break;
2654                 case OP_LOADU4_MEMBASE:
2655                         amd64_mov_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, 4);
2656                         break;
2657                 case OP_LOADU1_MEMBASE:
2658                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, FALSE, FALSE);
2659                         break;
2660                 case OP_LOADI1_MEMBASE:
2661                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, TRUE, FALSE);
2662                         break;
2663                 case OP_LOADU2_MEMBASE:
2664                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, FALSE, TRUE);
2665                         break;
2666                 case OP_LOADI2_MEMBASE:
2667                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, TRUE, TRUE);
2668                         break;
2669                 case OP_AMD64_LOADI8_MEMINDEX:
2670                         amd64_mov_reg_memindex_size (code, ins->dreg, ins->inst_basereg, 0, ins->inst_indexreg, 0, 8);
2671                         break;
2672                 case CEE_CONV_I1:
2673                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, FALSE);
2674                         break;
2675                 case CEE_CONV_I2:
2676                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, TRUE);
2677                         break;
2678                 case CEE_CONV_U1:
2679                         amd64_widen_reg (code, ins->dreg, ins->sreg1, FALSE, FALSE);
2680                         break;
2681                 case CEE_CONV_U2:
2682                         amd64_widen_reg (code, ins->dreg, ins->sreg1, FALSE, TRUE);
2683                         break;
2684                 case CEE_CONV_U8:
2685                 case CEE_CONV_U:
2686                         /* Clean out the upper word */
2687                         amd64_mov_reg_reg_size (code, ins->dreg, ins->sreg1, 4);
2688                         break;
2689                 case CEE_CONV_I8:
2690                 case CEE_CONV_I:
2691                         amd64_movsxd_reg_reg (code, ins->dreg, ins->sreg1);
2692                         break;
2693                 case OP_COMPARE:
2694                 case OP_LCOMPARE:
2695                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
2696                         break;
2697                 case OP_COMPARE_IMM:
2698                         g_assert (amd64_is_imm32 (ins->inst_imm));
2699                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, ins->inst_imm);
2700                         break;
2701                 case OP_X86_COMPARE_REG_MEMBASE:
2702                         amd64_alu_reg_membase (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset);
2703                         break;
2704                 case OP_X86_TEST_NULL:
2705                         amd64_test_reg_reg_size (code, ins->sreg1, ins->sreg1, 4);
2706                         break;
2707                 case OP_AMD64_TEST_NULL:
2708                         amd64_test_reg_reg (code, ins->sreg1, ins->sreg1);
2709                         break;
2710                 case OP_X86_ADD_MEMBASE_IMM:
2711                         /* FIXME: Make a 64 version too */
2712                         amd64_alu_membase_imm_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
2713                         break;
2714                 case OP_X86_ADD_MEMBASE:
2715                         amd64_alu_reg_membase_size (code, X86_ADD, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
2716                         break;
2717                 case OP_X86_SUB_MEMBASE_IMM:
2718                         g_assert (amd64_is_imm32 (ins->inst_imm));
2719                         amd64_alu_membase_imm_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
2720                         break;
2721                 case OP_X86_SUB_MEMBASE:
2722                         amd64_alu_reg_membase_size (code, X86_SUB, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
2723                         break;
2724                 case OP_X86_INC_MEMBASE:
2725                         amd64_inc_membase_size (code, ins->inst_basereg, ins->inst_offset, 4);
2726                         break;
2727                 case OP_X86_INC_REG:
2728                         amd64_inc_reg_size (code, ins->dreg, 4);
2729                         break;
2730                 case OP_X86_DEC_MEMBASE:
2731                         amd64_dec_membase_size (code, ins->inst_basereg, ins->inst_offset, 4);
2732                         break;
2733                 case OP_X86_DEC_REG:
2734                         amd64_dec_reg_size (code, ins->dreg, 4);
2735                         break;
2736                 case OP_X86_MUL_MEMBASE:
2737                         amd64_imul_reg_membase_size (code, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
2738                         break;
2739                 case OP_AMD64_ICOMPARE_MEMBASE_REG:
2740                         amd64_alu_membase_reg_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
2741                         break;
2742                 case OP_AMD64_ICOMPARE_MEMBASE_IMM:
2743                         amd64_alu_membase_imm_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
2744                         break;
2745                 case OP_AMD64_ICOMPARE_REG_MEMBASE:
2746                         amd64_alu_reg_membase_size (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
2747                         break;
2748                 case OP_BREAK:
2749                         amd64_breakpoint (code);
2750                         break;
2751                 case OP_ADDCC:
2752                 case CEE_ADD:
2753                 case OP_LADD:
2754                         amd64_alu_reg_reg (code, X86_ADD, ins->sreg1, ins->sreg2);
2755                         break;
2756                 case OP_ADC:
2757                         amd64_alu_reg_reg (code, X86_ADC, ins->sreg1, ins->sreg2);
2758                         break;
2759                 case OP_ADD_IMM:
2760                         g_assert (amd64_is_imm32 (ins->inst_imm));
2761                         amd64_alu_reg_imm (code, X86_ADD, ins->dreg, ins->inst_imm);
2762                         break;
2763                 case OP_ADC_IMM:
2764                         g_assert (amd64_is_imm32 (ins->inst_imm));
2765                         amd64_alu_reg_imm (code, X86_ADC, ins->dreg, ins->inst_imm);
2766                         break;
2767                 case OP_SUBCC:
2768                 case CEE_SUB:
2769                         amd64_alu_reg_reg (code, X86_SUB, ins->sreg1, ins->sreg2);
2770                         break;
2771                 case OP_SBB:
2772                         amd64_alu_reg_reg (code, X86_SBB, ins->sreg1, ins->sreg2);
2773                         break;
2774                 case OP_SUB_IMM:
2775                         g_assert (amd64_is_imm32 (ins->inst_imm));
2776                         amd64_alu_reg_imm (code, X86_SUB, ins->dreg, ins->inst_imm);
2777                         break;
2778                 case OP_SBB_IMM:
2779                         g_assert (amd64_is_imm32 (ins->inst_imm));
2780                         amd64_alu_reg_imm (code, X86_SBB, ins->dreg, ins->inst_imm);
2781                         break;
2782                 case CEE_AND:
2783                         amd64_alu_reg_reg (code, X86_AND, ins->sreg1, ins->sreg2);
2784                         break;
2785                 case OP_AND_IMM:
2786                         g_assert (amd64_is_imm32 (ins->inst_imm));
2787                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ins->inst_imm);
2788                         break;
2789                 case CEE_MUL:
2790                 case OP_LMUL:
2791                         amd64_imul_reg_reg (code, ins->sreg1, ins->sreg2);
2792                         break;
2793                 case OP_MUL_IMM:
2794                 case OP_LMUL_IMM:
2795                 case OP_IMUL_IMM: {
2796                         guint32 size = (ins->opcode == OP_IMUL_IMM) ? 4 : 8;
2797
2798                         switch (ins->inst_imm) {
2799                         case 2:
2800                                 /* MOV r1, r2 */
2801                                 /* ADD r1, r1 */
2802                                 if (ins->dreg != ins->sreg1)
2803                                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, size);
2804                                 amd64_alu_reg_reg (code, X86_ADD, ins->dreg, ins->dreg);
2805                                 break;
2806                         case 3:
2807                                 /* LEA r1, [r2 + r2*2] */
2808                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 1);
2809                                 break;
2810                         case 5:
2811                                 /* LEA r1, [r2 + r2*4] */
2812                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
2813                                 break;
2814                         case 6:
2815                                 /* LEA r1, [r2 + r2*2] */
2816                                 /* ADD r1, r1          */
2817                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 1);
2818                                 amd64_alu_reg_reg (code, X86_ADD, ins->dreg, ins->dreg);
2819                                 break;
2820                         case 9:
2821                                 /* LEA r1, [r2 + r2*8] */
2822                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 3);
2823                                 break;
2824                         case 10:
2825                                 /* LEA r1, [r2 + r2*4] */
2826                                 /* ADD r1, r1          */
2827                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
2828                                 amd64_alu_reg_reg (code, X86_ADD, ins->dreg, ins->dreg);
2829                                 break;
2830                         case 12:
2831                                 /* LEA r1, [r2 + r2*2] */
2832                                 /* SHL r1, 2           */
2833                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 1);
2834                                 amd64_shift_reg_imm (code, X86_SHL, ins->dreg, 2);
2835                                 break;
2836                         case 25:
2837                                 /* LEA r1, [r2 + r2*4] */
2838                                 /* LEA r1, [r1 + r1*4] */
2839                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
2840                                 amd64_lea_memindex (code, ins->dreg, ins->dreg, 0, ins->dreg, 2);
2841                                 break;
2842                         case 100:
2843                                 /* LEA r1, [r2 + r2*4] */
2844                                 /* SHL r1, 2           */
2845                                 /* LEA r1, [r1 + r1*4] */
2846                                 amd64_lea_memindex (code, ins->dreg, ins->sreg1, 0, ins->sreg1, 2);
2847                                 amd64_shift_reg_imm (code, X86_SHL, ins->dreg, 2);
2848                                 amd64_lea_memindex (code, ins->dreg, ins->dreg, 0, ins->dreg, 2);
2849                                 break;
2850                         default:
2851                                 amd64_imul_reg_reg_imm_size (code, ins->dreg, ins->sreg1, ins->inst_imm, size);
2852                                 break;
2853                         }
2854                         break;
2855                 }
2856                 case CEE_DIV:
2857                 case OP_LDIV:
2858                 case CEE_REM:
2859                 case OP_LREM:
2860                         /* Regalloc magic makes the div/rem cases the same */
2861                         if (ins->sreg2 == AMD64_RDX) {
2862                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
2863                                 amd64_cdq (code);
2864                                 amd64_div_membase (code, AMD64_RSP, -8, TRUE);
2865                         } else {
2866                                 amd64_cdq (code);
2867                                 amd64_div_reg (code, ins->sreg2, TRUE);
2868                         }
2869                         break;
2870                 case CEE_DIV_UN:
2871                 case OP_LDIV_UN:
2872                 case CEE_REM_UN:
2873                 case OP_LREM_UN:
2874                         if (ins->sreg2 == AMD64_RDX) {
2875                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
2876                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
2877                                 amd64_div_membase (code, AMD64_RSP, -8, FALSE);
2878                         } else {
2879                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
2880                                 amd64_div_reg (code, ins->sreg2, FALSE);
2881                         }
2882                         break;
2883                 case OP_IDIV:
2884                 case OP_IREM:
2885                         if (ins->sreg2 == AMD64_RDX) {
2886                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
2887                                 amd64_cdq_size (code, 4);
2888                                 amd64_div_membase_size (code, AMD64_RSP, -8, TRUE, 4);
2889                         } else {
2890                                 amd64_cdq_size (code, 4);
2891                                 amd64_div_reg_size (code, ins->sreg2, TRUE, 4);
2892                         }
2893                         break;
2894                 case OP_IDIV_UN:
2895                 case OP_IREM_UN:
2896                         if (ins->sreg2 == AMD64_RDX) {
2897                                 amd64_mov_membase_reg (code, AMD64_RSP, -8, AMD64_RDX, 8);
2898                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
2899                                 amd64_div_membase_size (code, AMD64_RSP, -8, FALSE, 4);
2900                         } else {
2901                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
2902                                 amd64_div_reg_size (code, ins->sreg2, FALSE, 4);
2903                         }
2904                         break;
2905                 case OP_LMUL_OVF:
2906                         amd64_imul_reg_reg (code, ins->sreg1, ins->sreg2);
2907                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
2908                         break;
2909                 case CEE_OR:
2910                         amd64_alu_reg_reg (code, X86_OR, ins->sreg1, ins->sreg2);
2911                         break;
2912                 case OP_OR_IMM
2913 :                       g_assert (amd64_is_imm32 (ins->inst_imm));
2914                         amd64_alu_reg_imm (code, X86_OR, ins->sreg1, ins->inst_imm);
2915                         break;
2916                 case CEE_XOR:
2917                 case OP_LXOR:
2918                         amd64_alu_reg_reg (code, X86_XOR, ins->sreg1, ins->sreg2);
2919                         break;
2920                 case OP_XOR_IMM:
2921                         g_assert (amd64_is_imm32 (ins->inst_imm));
2922                         amd64_alu_reg_imm (code, X86_XOR, ins->sreg1, ins->inst_imm);
2923                         break;
2924                 case CEE_SHL:
2925                 case OP_LSHL:
2926                         g_assert (ins->sreg2 == AMD64_RCX);
2927                         amd64_shift_reg (code, X86_SHL, ins->dreg);
2928                         break;
2929                 case CEE_SHR:
2930                 case OP_LSHR:
2931                         g_assert (ins->sreg2 == AMD64_RCX);
2932                         amd64_shift_reg (code, X86_SAR, ins->dreg);
2933                         break;
2934                 case OP_SHR_IMM:
2935                         g_assert (amd64_is_imm32 (ins->inst_imm));
2936                         amd64_shift_reg_imm_size (code, X86_SAR, ins->dreg, ins->inst_imm, 4);
2937                         break;
2938                 case OP_LSHR_IMM:
2939                         g_assert (amd64_is_imm32 (ins->inst_imm));
2940                         amd64_shift_reg_imm (code, X86_SAR, ins->dreg, ins->inst_imm);
2941                         break;
2942                 case OP_SHR_UN_IMM:
2943                         g_assert (amd64_is_imm32 (ins->inst_imm));
2944                         amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, ins->inst_imm, 4);
2945                         break;
2946                 case OP_LSHR_UN_IMM:
2947                         g_assert (amd64_is_imm32 (ins->inst_imm));
2948                         amd64_shift_reg_imm (code, X86_SHR, ins->dreg, ins->inst_imm);
2949                         break;
2950                 case CEE_SHR_UN:
2951                         g_assert (ins->sreg2 == AMD64_RCX);
2952                         amd64_shift_reg_size (code, X86_SHR, ins->dreg, 4);
2953                         break;
2954                 case OP_LSHR_UN:
2955                         g_assert (ins->sreg2 == AMD64_RCX);
2956                         amd64_shift_reg (code, X86_SHR, ins->dreg);
2957                         break;
2958                 case OP_SHL_IMM:
2959                         g_assert (amd64_is_imm32 (ins->inst_imm));
2960                         amd64_shift_reg_imm_size (code, X86_SHL, ins->dreg, ins->inst_imm, 4);
2961                         break;
2962                 case OP_LSHL_IMM:
2963                         g_assert (amd64_is_imm32 (ins->inst_imm));
2964                         amd64_shift_reg_imm (code, X86_SHL, ins->dreg, ins->inst_imm);
2965                         break;
2966
2967                 case OP_IADDCC:
2968                 case OP_IADD:
2969                         amd64_alu_reg_reg_size (code, X86_ADD, ins->sreg1, ins->sreg2, 4);
2970                         break;
2971                 case OP_IADC:
2972                         amd64_alu_reg_reg_size (code, X86_ADC, ins->sreg1, ins->sreg2, 4);
2973                         break;
2974                 case OP_IADD_IMM:
2975                         amd64_alu_reg_imm_size (code, X86_ADD, ins->dreg, ins->inst_imm, 4);
2976                         break;
2977                 case OP_IADC_IMM:
2978                         amd64_alu_reg_imm_size (code, X86_ADC, ins->dreg, ins->inst_imm, 4);
2979                         break;
2980                 case OP_ISUBCC:
2981                 case OP_ISUB:
2982                         amd64_alu_reg_reg_size (code, X86_SUB, ins->sreg1, ins->sreg2, 4);
2983                         break;
2984                 case OP_ISBB:
2985                         amd64_alu_reg_reg_size (code, X86_SBB, ins->sreg1, ins->sreg2, 4);
2986                         break;
2987                 case OP_ISUB_IMM:
2988                         amd64_alu_reg_imm_size (code, X86_SUB, ins->dreg, ins->inst_imm, 4);
2989                         break;
2990                 case OP_ISBB_IMM:
2991                         amd64_alu_reg_imm_size (code, X86_SBB, ins->dreg, ins->inst_imm, 4);
2992                         break;
2993                 case OP_IAND:
2994                         amd64_alu_reg_reg_size (code, X86_AND, ins->sreg1, ins->sreg2, 4);
2995                         break;
2996                 case OP_IAND_IMM:
2997                         amd64_alu_reg_imm_size (code, X86_AND, ins->sreg1, ins->inst_imm, 4);
2998                         break;
2999                 case OP_IOR:
3000                         amd64_alu_reg_reg_size (code, X86_OR, ins->sreg1, ins->sreg2, 4);
3001                         break;
3002                 case OP_IOR_IMM:
3003                         amd64_alu_reg_imm_size (code, X86_OR, ins->sreg1, ins->inst_imm, 4);
3004                         break;
3005                 case OP_IXOR:
3006                         amd64_alu_reg_reg_size (code, X86_XOR, ins->sreg1, ins->sreg2, 4);
3007                         break;
3008                 case OP_IXOR_IMM:
3009                         amd64_alu_reg_imm_size (code, X86_XOR, ins->sreg1, ins->inst_imm, 4);
3010                         break;
3011                 case OP_INEG:
3012                         amd64_neg_reg_size (code, ins->sreg1, 4);
3013                         break;
3014                 case OP_INOT:
3015                         amd64_not_reg_size (code, ins->sreg1, 4);
3016                         break;
3017                 case OP_ISHL:
3018                         g_assert (ins->sreg2 == AMD64_RCX);
3019                         amd64_shift_reg_size (code, X86_SHL, ins->dreg, 4);
3020                         break;
3021                 case OP_ISHR:
3022                         g_assert (ins->sreg2 == AMD64_RCX);
3023                         amd64_shift_reg_size (code, X86_SAR, ins->dreg, 4);
3024                         break;
3025                 case OP_ISHR_IMM:
3026                         amd64_shift_reg_imm_size (code, X86_SAR, ins->dreg, ins->inst_imm, 4);
3027                         break;
3028                 case OP_ISHR_UN_IMM:
3029                         amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, ins->inst_imm, 4);
3030                         break;
3031                 case OP_ISHR_UN:
3032                         g_assert (ins->sreg2 == AMD64_RCX);
3033                         amd64_shift_reg_size (code, X86_SHR, ins->dreg, 4);
3034                         break;
3035                 case OP_ISHL_IMM:
3036                         amd64_shift_reg_imm_size (code, X86_SHL, ins->dreg, ins->inst_imm, 4);
3037                         break;
3038                 case OP_IMUL:
3039                         amd64_imul_reg_reg_size (code, ins->sreg1, ins->sreg2, 4);
3040                         break;
3041                 case OP_IMUL_OVF:
3042                         amd64_imul_reg_reg_size (code, ins->sreg1, ins->sreg2, 4);
3043                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
3044                         break;
3045                 case OP_IMUL_OVF_UN:
3046                 case OP_LMUL_OVF_UN: {
3047                         /* the mul operation and the exception check should most likely be split */
3048                         int non_eax_reg, saved_eax = FALSE, saved_edx = FALSE;
3049                         int size = (ins->opcode == OP_IMUL_OVF_UN) ? 4 : 8;
3050                         /*g_assert (ins->sreg2 == X86_EAX);
3051                         g_assert (ins->dreg == X86_EAX);*/
3052                         if (ins->sreg2 == X86_EAX) {
3053                                 non_eax_reg = ins->sreg1;
3054                         } else if (ins->sreg1 == X86_EAX) {
3055                                 non_eax_reg = ins->sreg2;
3056                         } else {
3057                                 /* no need to save since we're going to store to it anyway */
3058                                 if (ins->dreg != X86_EAX) {
3059                                         saved_eax = TRUE;
3060                                         amd64_push_reg (code, X86_EAX);
3061                                 }
3062                                 amd64_mov_reg_reg (code, X86_EAX, ins->sreg1, size);
3063                                 non_eax_reg = ins->sreg2;
3064                         }
3065                         if (ins->dreg == X86_EDX) {
3066                                 if (!saved_eax) {
3067                                         saved_eax = TRUE;
3068                                         amd64_push_reg (code, X86_EAX);
3069                                 }
3070                         } else {
3071                                 saved_edx = TRUE;
3072                                 amd64_push_reg (code, X86_EDX);
3073                         }
3074                         amd64_mul_reg_size (code, non_eax_reg, FALSE, size);
3075                         /* save before the check since pop and mov don't change the flags */
3076                         if (ins->dreg != X86_EAX)
3077                                 amd64_mov_reg_reg (code, ins->dreg, X86_EAX, size);
3078                         if (saved_edx)
3079                                 amd64_pop_reg (code, X86_EDX);
3080                         if (saved_eax)
3081                                 amd64_pop_reg (code, X86_EAX);
3082                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
3083                         break;
3084                 }
3085                 case OP_ICOMPARE:
3086                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
3087                         break;
3088                 case OP_ICOMPARE_IMM:
3089                         amd64_alu_reg_imm_size (code, X86_CMP, ins->sreg1, ins->inst_imm, 4);
3090                         break;
3091                 case OP_IBEQ:
3092                 case OP_IBLT:
3093                 case OP_IBGT:
3094                 case OP_IBGE:
3095                 case OP_IBLE:
3096                 case OP_IBNE_UN:
3097                 case OP_IBLT_UN:
3098                 case OP_IBGT_UN:
3099                 case OP_IBGE_UN:
3100                 case OP_IBLE_UN:
3101                         EMIT_COND_BRANCH (ins, cc_table [mono_opcode_to_cond (ins->opcode)], cc_signed_table [mono_opcode_to_cond (ins->opcode)]);
3102                         break;
3103                 case CEE_NOT:
3104                         amd64_not_reg (code, ins->sreg1);
3105                         break;
3106                 case CEE_NEG:
3107                         amd64_neg_reg (code, ins->sreg1);
3108                         break;
3109                 case OP_SEXT_I1:
3110                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, FALSE);
3111                         break;
3112                 case OP_SEXT_I2:
3113                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, TRUE);
3114                         break;
3115                 case OP_SEXT_I4:
3116                         amd64_movsxd_reg_reg (code, ins->dreg, ins->sreg1);
3117                         break;
3118                 case OP_ICONST:
3119                 case OP_I8CONST:
3120                         if ((((guint64)ins->inst_c0) >> 32) == 0)
3121                                 amd64_mov_reg_imm_size (code, ins->dreg, ins->inst_c0, 4);
3122                         else
3123                                 amd64_mov_reg_imm_size (code, ins->dreg, ins->inst_c0, 8);
3124                         break;
3125                 case OP_AOTCONST:
3126                         mono_add_patch_info (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
3127                         amd64_mov_reg_membase (code, ins->dreg, AMD64_RIP, 0, 8);
3128                         break;
3129                 case CEE_CONV_I4:
3130                 case CEE_CONV_U4:
3131                 case OP_MOVE:
3132                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, sizeof (gpointer));
3133                         break;
3134                 case OP_AMD64_SET_XMMREG_R4: {
3135                         if (use_sse2) {
3136                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg1);
3137                         }
3138                         else {
3139                                 amd64_fst_membase (code, AMD64_RSP, -8, FALSE, TRUE);
3140                                 /* ins->dreg is set to -1 by the reg allocator */
3141                                 amd64_movss_reg_membase (code, ins->backend.reg3, AMD64_RSP, -8);
3142                         }
3143                         break;
3144                 }
3145                 case OP_AMD64_SET_XMMREG_R8: {
3146                         if (use_sse2) {
3147                                 if (ins->dreg != ins->sreg1)
3148                                         amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
3149                         }
3150                         else {
3151                                 amd64_fst_membase (code, AMD64_RSP, -8, TRUE, TRUE);
3152                                 /* ins->dreg is set to -1 by the reg allocator */
3153                                 amd64_movsd_reg_membase (code, ins->backend.reg3, AMD64_RSP, -8);
3154                         }
3155                         break;
3156                 }
3157                 case OP_JMP: {
3158                         /*
3159                          * Note: this 'frame destruction' logic is useful for tail calls, too.
3160                          * Keep in sync with the code in emit_epilog.
3161                          */
3162                         int pos = 0, i;
3163
3164                         /* FIXME: no tracing support... */
3165                         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
3166                                 code = mono_arch_instrument_epilog (cfg, mono_profiler_method_leave, code, FALSE);
3167
3168                         g_assert (!cfg->method->save_lmf);
3169
3170                         code = emit_load_volatile_arguments (cfg, code);
3171
3172                         if (cfg->arch.omit_fp) {
3173                                 guint32 save_offset = 0;
3174                                 /* Pop callee-saved registers */
3175                                 for (i = 0; i < AMD64_NREG; ++i)
3176                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
3177                                                 amd64_mov_reg_membase (code, i, AMD64_RSP, save_offset, 8);
3178                                                 save_offset += 8;
3179                                         }
3180                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, cfg->arch.stack_alloc_size);
3181                         }
3182                         else {
3183                                 for (i = 0; i < AMD64_NREG; ++i)
3184                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i)))
3185                                                 pos -= sizeof (gpointer);
3186
3187                                 if (pos)
3188                                         amd64_lea_membase (code, AMD64_RSP, AMD64_RBP, pos);
3189
3190                                 /* Pop registers in reverse order */
3191                                 for (i = AMD64_NREG - 1; i > 0; --i)
3192                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
3193                                                 amd64_pop_reg (code, i);
3194                                         }
3195
3196                                 amd64_leave (code);
3197                         }
3198
3199                         offset = code - cfg->native_code;
3200                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_METHOD_JUMP, ins->inst_p0);
3201                         if (cfg->compile_aot)
3202                                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_RIP, 0, 8);
3203                         else
3204                                 amd64_set_reg_template (code, AMD64_R11);
3205                         amd64_jump_reg (code, AMD64_R11);
3206                         break;
3207                 }
3208                 case OP_CHECK_THIS:
3209                         /* ensure ins->sreg1 is not NULL */
3210                         amd64_alu_membase_imm (code, X86_CMP, ins->sreg1, 0, 0);
3211                         break;
3212                 case OP_ARGLIST: {
3213                         amd64_lea_membase (code, AMD64_R11, cfg->frame_reg, cfg->sig_cookie);
3214                         amd64_mov_membase_reg (code, ins->sreg1, 0, AMD64_R11, 8);
3215                         break;
3216                 }
3217                 case OP_FCALL:
3218                 case OP_LCALL:
3219                 case OP_VCALL:
3220                 case OP_VOIDCALL:
3221                 case CEE_CALL:
3222                         call = (MonoCallInst*)ins;
3223                         /*
3224                          * The AMD64 ABI forces callers to know about varargs.
3225                          */
3226                         if ((call->signature->call_convention == MONO_CALL_VARARG) && (call->signature->pinvoke))
3227                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
3228                         else if ((cfg->method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) && (cfg->method->klass->image != mono_defaults.corlib)) {
3229                                 /*
3230                                  * Since the unmanaged calling convention doesn't contain a
3231                                  * 'vararg' entry, we have to treat every pinvoke call as a
3232                                  * potential vararg call.
3233                                  */
3234                                 guint32 nregs, i;
3235                                 nregs = 0;
3236                                 for (i = 0; i < AMD64_XMM_NREG; ++i)
3237                                         if (call->used_fregs & (1 << i))
3238                                                 nregs ++;
3239                                 if (!nregs)
3240                                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
3241                                 else
3242                                         amd64_mov_reg_imm (code, AMD64_RAX, nregs);
3243                         }
3244
3245                         if (ins->flags & MONO_INST_HAS_METHOD)
3246                                 code = emit_call (cfg, code, MONO_PATCH_INFO_METHOD, call->method);
3247                         else
3248                                 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, call->fptr);
3249                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention))
3250                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
3251                         code = emit_move_return_value (cfg, ins, code);
3252                         break;
3253                 case OP_FCALL_REG:
3254                 case OP_LCALL_REG:
3255                 case OP_VCALL_REG:
3256                 case OP_VOIDCALL_REG:
3257                 case OP_CALL_REG:
3258                         call = (MonoCallInst*)ins;
3259
3260                         if (AMD64_IS_ARGUMENT_REG (ins->sreg1)) {
3261                                 amd64_mov_reg_reg (code, AMD64_R11, ins->sreg1, 8);
3262                                 ins->sreg1 = AMD64_R11;
3263                         }
3264
3265                         /*
3266                          * The AMD64 ABI forces callers to know about varargs.
3267                          */
3268                         if ((call->signature->call_convention == MONO_CALL_VARARG) && (call->signature->pinvoke)) {
3269                                 if (ins->sreg1 == AMD64_RAX) {
3270                                         amd64_mov_reg_reg (code, AMD64_R11, AMD64_RAX, 8);
3271                                         ins->sreg1 = AMD64_R11;
3272                                 }
3273                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
3274                         }
3275                         amd64_call_reg (code, ins->sreg1);
3276                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention))
3277                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
3278                         code = emit_move_return_value (cfg, ins, code);
3279                         break;
3280                 case OP_FCALL_MEMBASE:
3281                 case OP_LCALL_MEMBASE:
3282                 case OP_VCALL_MEMBASE:
3283                 case OP_VOIDCALL_MEMBASE:
3284                 case OP_CALL_MEMBASE:
3285                         call = (MonoCallInst*)ins;
3286
3287                         if (AMD64_IS_ARGUMENT_REG (ins->sreg1)) {
3288                                 /*
3289                                  * Can't use R11 because it is clobbered by the trampoline
3290                                  * code, and the reg value is needed by get_vcall_slot_addr.
3291                                  */
3292                                 amd64_mov_reg_reg (code, AMD64_RAX, ins->sreg1, 8);
3293                                 ins->sreg1 = AMD64_RAX;
3294                         }
3295
3296                         amd64_call_membase (code, ins->sreg1, ins->inst_offset);
3297                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention))
3298                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
3299                         code = emit_move_return_value (cfg, ins, code);
3300                         break;
3301                 case OP_OUTARG:
3302                 case OP_X86_PUSH:
3303                         amd64_push_reg (code, ins->sreg1);
3304                         break;
3305                 case OP_X86_PUSH_IMM:
3306                         g_assert (amd64_is_imm32 (ins->inst_imm));
3307                         amd64_push_imm (code, ins->inst_imm);
3308                         break;
3309                 case OP_X86_PUSH_MEMBASE:
3310                         amd64_push_membase (code, ins->inst_basereg, ins->inst_offset);
3311                         break;
3312                 case OP_X86_PUSH_OBJ:
3313                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, ins->inst_imm);
3314                         amd64_push_reg (code, AMD64_RDI);
3315                         amd64_push_reg (code, AMD64_RSI);
3316                         amd64_push_reg (code, AMD64_RCX);
3317                         if (ins->inst_offset)
3318                                 amd64_lea_membase (code, AMD64_RSI, ins->inst_basereg, ins->inst_offset);
3319                         else
3320                                 amd64_mov_reg_reg (code, AMD64_RSI, ins->inst_basereg, 8);
3321                         amd64_lea_membase (code, AMD64_RDI, AMD64_RSP, 3 * 8);
3322                         amd64_mov_reg_imm (code, AMD64_RCX, (ins->inst_imm >> 3));
3323                         amd64_cld (code);
3324                         amd64_prefix (code, X86_REP_PREFIX);
3325                         amd64_movsd (code);
3326                         amd64_pop_reg (code, AMD64_RCX);
3327                         amd64_pop_reg (code, AMD64_RSI);
3328                         amd64_pop_reg (code, AMD64_RDI);
3329                         break;
3330                 case OP_X86_LEA:
3331                         amd64_lea_memindex (code, ins->dreg, ins->sreg1, ins->inst_imm, ins->sreg2, ins->backend.shift_amount);
3332                         break;
3333                 case OP_X86_LEA_MEMBASE:
3334                         amd64_lea_membase (code, ins->dreg, ins->sreg1, ins->inst_imm);
3335                         break;
3336                 case OP_X86_XCHG:
3337                         amd64_xchg_reg_reg (code, ins->sreg1, ins->sreg2, 4);
3338                         break;
3339                 case OP_LOCALLOC:
3340                         /* keep alignment */
3341                         amd64_alu_reg_imm (code, X86_ADD, ins->sreg1, MONO_ARCH_FRAME_ALIGNMENT - 1);
3342                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ~(MONO_ARCH_FRAME_ALIGNMENT - 1));
3343                         code = mono_emit_stack_alloc (code, ins);
3344                         amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
3345                         break;
3346                 case CEE_RET:
3347                         amd64_ret (code);
3348                         break;
3349                 case OP_THROW: {
3350                         amd64_mov_reg_reg (code, AMD64_RDI, ins->sreg1, 8);
3351                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
3352                                              (gpointer)"mono_arch_throw_exception");
3353                         break;
3354                 }
3355                 case OP_RETHROW: {
3356                         amd64_mov_reg_reg (code, AMD64_RDI, ins->sreg1, 8);
3357                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
3358                                              (gpointer)"mono_arch_rethrow_exception");
3359                         break;
3360                 }
3361                 case OP_CALL_HANDLER:
3362                         /* Align stack */
3363                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
3364                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_BB, ins->inst_target_bb);
3365                         amd64_call_imm (code, 0);
3366                         /* Restore stack alignment */
3367                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
3368                         break;
3369
3370                 case OP_LABEL:
3371                         ins->inst_c0 = code - cfg->native_code;
3372                         break;
3373                 case OP_NOP:
3374                         break;
3375                 case OP_BR:
3376                         //g_print ("target: %p, next: %p, curr: %p, last: %p\n", ins->inst_target_bb, bb->next_bb, ins, bb->last_ins);
3377                         //if ((ins->inst_target_bb == bb->next_bb) && ins == bb->last_ins)
3378                         //break;
3379                         if (ins->flags & MONO_INST_BRLABEL) {
3380                                 if (ins->inst_i0->inst_c0) {
3381                                         amd64_jump_code (code, cfg->native_code + ins->inst_i0->inst_c0);
3382                                 } else {
3383                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_LABEL, ins->inst_i0);
3384                                         if ((cfg->opt & MONO_OPT_BRANCH) &&
3385                                             x86_is_imm8 (ins->inst_i0->inst_c1 - cpos))
3386                                                 x86_jump8 (code, 0);
3387                                         else
3388                                                 x86_jump32 (code, 0);
3389                                 }
3390                         } else {
3391                                 if (ins->inst_target_bb->native_offset) {
3392                                         amd64_jump_code (code, cfg->native_code + ins->inst_target_bb->native_offset);
3393                                 } else {
3394                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb);
3395                                         if ((cfg->opt & MONO_OPT_BRANCH) &&
3396                                             x86_is_imm8 (ins->inst_target_bb->max_offset - cpos))
3397                                                 x86_jump8 (code, 0);
3398                                         else
3399                                                 x86_jump32 (code, 0);
3400                                 }
3401                         }
3402                         break;
3403                 case OP_BR_REG:
3404                         amd64_jump_reg (code, ins->sreg1);
3405                         break;
3406                 case OP_CEQ:
3407                 case OP_ICEQ:
3408                 case OP_CLT:
3409                 case OP_ICLT:
3410                 case OP_CGT:
3411                 case OP_ICGT:
3412                 case OP_CLT_UN:
3413                 case OP_ICLT_UN:
3414                 case OP_CGT_UN:
3415                 case OP_ICGT_UN:
3416                         amd64_set_reg (code, cc_table [mono_opcode_to_cond (ins->opcode)], ins->dreg, cc_signed_table [mono_opcode_to_cond (ins->opcode)]);
3417                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
3418                         break;
3419                 case OP_COND_EXC_EQ:
3420                 case OP_COND_EXC_NE_UN:
3421                 case OP_COND_EXC_LT:
3422                 case OP_COND_EXC_LT_UN:
3423                 case OP_COND_EXC_GT:
3424                 case OP_COND_EXC_GT_UN:
3425                 case OP_COND_EXC_GE:
3426                 case OP_COND_EXC_GE_UN:
3427                 case OP_COND_EXC_LE:
3428                 case OP_COND_EXC_LE_UN:
3429                         EMIT_COND_SYSTEM_EXCEPTION (cc_table [mono_opcode_to_cond (ins->opcode)], cc_signed_table [mono_opcode_to_cond (ins->opcode)], ins->inst_p1);
3430                         break;
3431                 case OP_COND_EXC_OV:
3432                 case OP_COND_EXC_NO:
3433                 case OP_COND_EXC_C:
3434                 case OP_COND_EXC_NC:
3435                         EMIT_COND_SYSTEM_EXCEPTION (branch_cc_table [ins->opcode - OP_COND_EXC_EQ],
3436                                                     (ins->opcode < OP_COND_EXC_NE_UN), ins->inst_p1);
3437                         break;
3438                 case OP_COND_EXC_IOV:
3439                 case OP_COND_EXC_IC:
3440                         EMIT_COND_SYSTEM_EXCEPTION (branch_cc_table [ins->opcode - OP_COND_EXC_IEQ],
3441                                                     (ins->opcode < OP_COND_EXC_INE_UN), ins->inst_p1);
3442                         break;
3443                 case CEE_BEQ:
3444                 case CEE_BNE_UN:
3445                 case CEE_BLT:
3446                 case CEE_BLT_UN:
3447                 case CEE_BGT:
3448                 case CEE_BGT_UN:
3449                 case CEE_BGE:
3450                 case CEE_BGE_UN:
3451                 case CEE_BLE:
3452                 case CEE_BLE_UN:
3453                         EMIT_COND_BRANCH (ins, cc_table [mono_opcode_to_cond (ins->opcode)], cc_signed_table [mono_opcode_to_cond (ins->opcode)]);
3454                         break;
3455
3456                 /* floating point opcodes */
3457                 case OP_R8CONST: {
3458                         double d = *(double *)ins->inst_p0;
3459
3460                         if (use_sse2) {
3461                                 if ((d == 0.0) && (mono_signbit (d) == 0)) {
3462                                         amd64_sse_xorpd_reg_reg (code, ins->dreg, ins->dreg);
3463                                 }
3464                                 else {
3465                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, ins->inst_p0);
3466                                         amd64_sse_movsd_reg_membase (code, ins->dreg, AMD64_RIP, 0);
3467                                 }
3468                         }
3469                         else if ((d == 0.0) && (mono_signbit (d) == 0)) {
3470                                 amd64_fldz (code);
3471                         } else if (d == 1.0) {
3472                                 x86_fld1 (code);
3473                         } else {
3474                                 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, ins->inst_p0);
3475                                 amd64_fld_membase (code, AMD64_RIP, 0, TRUE);
3476                         }
3477                         break;
3478                 }
3479                 case OP_R4CONST: {
3480                         float f = *(float *)ins->inst_p0;
3481
3482                         if (use_sse2) {
3483                                 if ((f == 0.0) && (mono_signbit (f) == 0)) {
3484                                         amd64_sse_xorpd_reg_reg (code, ins->dreg, ins->dreg);
3485                                 }
3486                                 else {
3487                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R4, ins->inst_p0);
3488                                         amd64_sse_movss_reg_membase (code, ins->dreg, AMD64_RIP, 0);
3489                                         amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
3490                                 }
3491                         }
3492                         else if ((f == 0.0) && (mono_signbit (f) == 0)) {
3493                                 amd64_fldz (code);
3494                         } else if (f == 1.0) {
3495                                 x86_fld1 (code);
3496                         } else {
3497                                 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R4, ins->inst_p0);
3498                                 amd64_fld_membase (code, AMD64_RIP, 0, FALSE);
3499                         }
3500                         break;
3501                 }
3502                 case OP_STORER8_MEMBASE_REG:
3503                         if (use_sse2)
3504                                 amd64_sse_movsd_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1);
3505                         else
3506                                 amd64_fst_membase (code, ins->inst_destbasereg, ins->inst_offset, TRUE, TRUE);
3507                         break;
3508                 case OP_LOADR8_SPILL_MEMBASE:
3509                         if (use_sse2)
3510                                 g_assert_not_reached ();
3511                         amd64_fld_membase (code, ins->inst_basereg, ins->inst_offset, TRUE);
3512                         amd64_fxch (code, 1);
3513                         break;
3514                 case OP_LOADR8_MEMBASE:
3515                         if (use_sse2)
3516                                 amd64_sse_movsd_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
3517                         else
3518                                 amd64_fld_membase (code, ins->inst_basereg, ins->inst_offset, TRUE);
3519                         break;
3520                 case OP_STORER4_MEMBASE_REG:
3521                         if (use_sse2) {
3522                                 /* This requires a double->single conversion */
3523                                 amd64_sse_cvtsd2ss_reg_reg (code, AMD64_XMM15, ins->sreg1);
3524                                 amd64_sse_movss_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, AMD64_XMM15);
3525                         }
3526                         else
3527                                 amd64_fst_membase (code, ins->inst_destbasereg, ins->inst_offset, FALSE, TRUE);
3528                         break;
3529                 case OP_LOADR4_MEMBASE:
3530                         if (use_sse2) {
3531                                 amd64_sse_movss_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
3532                                 amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
3533                         }
3534                         else
3535                                 amd64_fld_membase (code, ins->inst_basereg, ins->inst_offset, FALSE);
3536                         break;
3537                 case CEE_CONV_R4: /* FIXME: change precision */
3538                 case CEE_CONV_R8:
3539                         if (use_sse2)
3540                                 amd64_sse_cvtsi2sd_reg_reg_size (code, ins->dreg, ins->sreg1, 4);
3541                         else {
3542                                 amd64_push_reg (code, ins->sreg1);
3543                                 amd64_fild_membase (code, AMD64_RSP, 0, FALSE);
3544                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
3545                         }
3546                         break;
3547                 case CEE_CONV_R_UN:
3548                         /* Emulated */
3549                         g_assert_not_reached ();
3550                         break;
3551                 case OP_LCONV_TO_R4: /* FIXME: change precision */
3552                 case OP_LCONV_TO_R8:
3553                         if (use_sse2)
3554                                 amd64_sse_cvtsi2sd_reg_reg (code, ins->dreg, ins->sreg1);
3555                         else {
3556                                 amd64_push_reg (code, ins->sreg1);
3557                                 amd64_fild_membase (code, AMD64_RSP, 0, TRUE);
3558                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
3559                         }
3560                         break;
3561                 case OP_X86_FP_LOAD_I8:
3562                         if (use_sse2)
3563                                 g_assert_not_reached ();
3564                         amd64_fild_membase (code, ins->inst_basereg, ins->inst_offset, TRUE);
3565                         break;
3566                 case OP_X86_FP_LOAD_I4:
3567                         if (use_sse2)
3568                                 g_assert_not_reached ();
3569                         amd64_fild_membase (code, ins->inst_basereg, ins->inst_offset, FALSE);
3570                         break;
3571                 case OP_FCONV_TO_I1:
3572                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 1, TRUE);
3573                         break;
3574                 case OP_FCONV_TO_U1:
3575                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 1, FALSE);
3576                         break;
3577                 case OP_FCONV_TO_I2:
3578                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 2, TRUE);
3579                         break;
3580                 case OP_FCONV_TO_U2:
3581                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 2, FALSE);
3582                         break;
3583                 case OP_FCONV_TO_I4:
3584                 case OP_FCONV_TO_I:
3585                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 4, TRUE);
3586                         break;
3587                 case OP_FCONV_TO_I8:
3588                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 8, TRUE);
3589                         break;
3590                 case OP_LCONV_TO_R_UN: {
3591                         static guint8 mn[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x40 };
3592                         guint8 *br;
3593
3594                         if (use_sse2)
3595                                 g_assert_not_reached ();
3596
3597                         /* load 64bit integer to FP stack */
3598                         amd64_push_imm (code, 0);
3599                         amd64_push_reg (code, ins->sreg2);
3600                         amd64_push_reg (code, ins->sreg1);
3601                         amd64_fild_membase (code, AMD64_RSP, 0, TRUE);
3602                         /* store as 80bit FP value */
3603                         x86_fst80_membase (code, AMD64_RSP, 0);
3604
3605                         /* test if lreg is negative */
3606                         amd64_test_reg_reg (code, ins->sreg2, ins->sreg2);
3607                         br = code; x86_branch8 (code, X86_CC_GEZ, 0, TRUE);
3608
3609                         /* add correction constant mn */
3610                         x86_fld80_mem (code, (gssize)mn);
3611                         x86_fld80_membase (code, AMD64_RSP, 0);
3612                         amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
3613                         x86_fst80_membase (code, AMD64_RSP, 0);
3614
3615                         amd64_patch (br, code);
3616
3617                         x86_fld80_membase (code, AMD64_RSP, 0);
3618                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 12);
3619
3620                         break;
3621                 }
3622                 case CEE_CONV_OVF_U4:
3623                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, 0);
3624                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_LT, TRUE, "OverflowException");
3625                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, 8);
3626                         break;
3627                 case CEE_CONV_OVF_I4_UN:
3628                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, 0x7fffffff);
3629                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_GT, FALSE, "OverflowException");
3630                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, 8);
3631                         break;
3632                 case OP_FMOVE:
3633                         if (use_sse2 && (ins->dreg != ins->sreg1))
3634                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
3635                         break;
3636                 case OP_FADD:
3637                         if (use_sse2)
3638                                 amd64_sse_addsd_reg_reg (code, ins->dreg, ins->sreg2);
3639                         else
3640                                 amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
3641                         break;
3642                 case OP_FSUB:
3643                         if (use_sse2)
3644                                 amd64_sse_subsd_reg_reg (code, ins->dreg, ins->sreg2);
3645                         else
3646                                 amd64_fp_op_reg (code, X86_FSUB, 1, TRUE);
3647                         break;
3648                 case OP_FMUL:
3649                         if (use_sse2)
3650                                 amd64_sse_mulsd_reg_reg (code, ins->dreg, ins->sreg2);
3651                         else
3652                                 amd64_fp_op_reg (code, X86_FMUL, 1, TRUE);
3653                         break;
3654                 case OP_FDIV:
3655                         if (use_sse2)
3656                                 amd64_sse_divsd_reg_reg (code, ins->dreg, ins->sreg2);
3657                         else
3658                                 amd64_fp_op_reg (code, X86_FDIV, 1, TRUE);
3659                         break;
3660                 case OP_FNEG:
3661                         if (use_sse2) {
3662                                 static double r8_0 = -0.0;
3663
3664                                 g_assert (ins->sreg1 == ins->dreg);
3665
3666                                 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, &r8_0);
3667                                 amd64_sse_xorpd_reg_membase (code, ins->dreg, AMD64_RIP, 0);
3668                         }
3669                         else
3670                                 amd64_fchs (code);
3671                         break;
3672                 case OP_SIN:
3673                         if (use_sse2) {
3674                                 EMIT_SSE2_FPFUNC (code, fsin, ins->dreg, ins->sreg1);
3675                         }
3676                         else {
3677                                 amd64_fsin (code);
3678                                 amd64_fldz (code);
3679                                 amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
3680                         }
3681                         break;
3682                 case OP_COS:
3683                         if (use_sse2) {
3684                                 EMIT_SSE2_FPFUNC (code, fcos, ins->dreg, ins->sreg1);
3685                         }
3686                         else {
3687                                 amd64_fcos (code);
3688                                 amd64_fldz (code);
3689                                 amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
3690                         }
3691                         break;
3692                 case OP_ABS:
3693                         if (use_sse2) {
3694                                 EMIT_SSE2_FPFUNC (code, fabs, ins->dreg, ins->sreg1);
3695                         }
3696                         else
3697                                 amd64_fabs (code);
3698                         break;
3699                 case OP_TAN: {
3700                         /*
3701                          * it really doesn't make sense to inline all this code,
3702                          * it's here just to show that things may not be as simple
3703                          * as they appear.
3704                          */
3705                         guchar *check_pos, *end_tan, *pop_jump;
3706                         if (use_sse2)
3707                                 g_assert_not_reached ();
3708                         amd64_push_reg (code, AMD64_RAX);
3709                         amd64_fptan (code);
3710                         amd64_fnstsw (code);
3711                         amd64_test_reg_imm (code, AMD64_RAX, X86_FP_C2);
3712                         check_pos = code;
3713                         x86_branch8 (code, X86_CC_NE, 0, FALSE);
3714                         amd64_fstp (code, 0); /* pop the 1.0 */
3715                         end_tan = code;
3716                         x86_jump8 (code, 0);
3717                         amd64_fldpi (code);
3718                         amd64_fp_op (code, X86_FADD, 0);
3719                         amd64_fxch (code, 1);
3720                         x86_fprem1 (code);
3721                         amd64_fstsw (code);
3722                         amd64_test_reg_imm (code, AMD64_RAX, X86_FP_C2);
3723                         pop_jump = code;
3724                         x86_branch8 (code, X86_CC_NE, 0, FALSE);
3725                         amd64_fstp (code, 1);
3726                         amd64_fptan (code);
3727                         amd64_patch (pop_jump, code);
3728                         amd64_fstp (code, 0); /* pop the 1.0 */
3729                         amd64_patch (check_pos, code);
3730                         amd64_patch (end_tan, code);
3731                         amd64_fldz (code);
3732                         amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
3733                         amd64_pop_reg (code, AMD64_RAX);
3734                         break;
3735                 }
3736                 case OP_ATAN:
3737                         if (use_sse2)
3738                                 g_assert_not_reached ();
3739                         x86_fld1 (code);
3740                         amd64_fpatan (code);
3741                         amd64_fldz (code);
3742                         amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
3743                         break;
3744                 case OP_SQRT:
3745                         if (use_sse2) {
3746                                 EMIT_SSE2_FPFUNC (code, fsqrt, ins->dreg, ins->sreg1);
3747                         }
3748                         else
3749                                 amd64_fsqrt (code);
3750                         break;
3751                 case OP_X86_FPOP:
3752                         if (!use_sse2)
3753                                 amd64_fstp (code, 0);
3754                         break;
3755                 case OP_FREM: {
3756                         guint8 *l1, *l2;
3757
3758                         if (use_sse2)
3759                                 g_assert_not_reached ();
3760                         amd64_push_reg (code, AMD64_RAX);
3761                         /* we need to exchange ST(0) with ST(1) */
3762                         amd64_fxch (code, 1);
3763
3764                         /* this requires a loop, because fprem somtimes
3765                          * returns a partial remainder */
3766                         l1 = code;
3767                         /* looks like MS is using fprem instead of the IEEE compatible fprem1 */
3768                         /* x86_fprem1 (code); */
3769                         amd64_fprem (code);
3770                         amd64_fnstsw (code);
3771                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_C2);
3772                         l2 = code + 2;
3773                         x86_branch8 (code, X86_CC_NE, l1 - l2, FALSE);
3774
3775                         /* pop result */
3776                         amd64_fstp (code, 1);
3777
3778                         amd64_pop_reg (code, AMD64_RAX);
3779                         break;
3780                 }
3781                 case OP_FCOMPARE:
3782                         if (use_sse2) {
3783                                 /*
3784                                  * The two arguments are swapped because the fbranch instructions
3785                                  * depend on this for the non-sse case to work.
3786                                  */
3787                                 amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
3788                                 break;
3789                         }
3790                         if (cfg->opt & MONO_OPT_FCMOV) {
3791                                 amd64_fcomip (code, 1);
3792                                 amd64_fstp (code, 0);
3793                                 break;
3794                         }
3795                         /* this overwrites EAX */
3796                         EMIT_FPCOMPARE(code);
3797                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_CC_MASK);
3798                         break;
3799                 case OP_FCEQ:
3800                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
3801                                 /* zeroing the register at the start results in
3802                                  * shorter and faster code (we can also remove the widening op)
3803                                  */
3804                                 guchar *unordered_check;
3805                                 amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
3806
3807                                 if (use_sse2)
3808                                         amd64_sse_comisd_reg_reg (code, ins->sreg1, ins->sreg2);
3809                                 else {
3810                                         amd64_fcomip (code, 1);
3811                                         amd64_fstp (code, 0);
3812                                 }
3813                                 unordered_check = code;
3814                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
3815                                 amd64_set_reg (code, X86_CC_EQ, ins->dreg, FALSE);
3816                                 amd64_patch (unordered_check, code);
3817                                 break;
3818                         }
3819                         if (ins->dreg != AMD64_RAX)
3820                                 amd64_push_reg (code, AMD64_RAX);
3821
3822                         EMIT_FPCOMPARE(code);
3823                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_CC_MASK);
3824                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0x4000);
3825                         amd64_set_reg (code, X86_CC_EQ, ins->dreg, TRUE);
3826                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
3827
3828                         if (ins->dreg != AMD64_RAX)
3829                                 amd64_pop_reg (code, AMD64_RAX);
3830                         break;
3831                 case OP_FCLT:
3832                 case OP_FCLT_UN:
3833                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
3834                                 /* zeroing the register at the start results in
3835                                  * shorter and faster code (we can also remove the widening op)
3836                                  */
3837                                 amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
3838                                 if (use_sse2)
3839                                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
3840                                 else {
3841                                         amd64_fcomip (code, 1);
3842                                         amd64_fstp (code, 0);
3843                                 }
3844                                 if (ins->opcode == OP_FCLT_UN) {
3845                                         guchar *unordered_check = code;
3846                                         guchar *jump_to_end;
3847                                         x86_branch8 (code, X86_CC_P, 0, FALSE);
3848                                         amd64_set_reg (code, X86_CC_GT, ins->dreg, FALSE);
3849                                         jump_to_end = code;
3850                                         x86_jump8 (code, 0);
3851                                         amd64_patch (unordered_check, code);
3852                                         amd64_inc_reg (code, ins->dreg);
3853                                         amd64_patch (jump_to_end, code);
3854                                 } else {
3855                                         amd64_set_reg (code, X86_CC_GT, ins->dreg, FALSE);
3856                                 }
3857                                 break;
3858                         }
3859                         if (ins->dreg != AMD64_RAX)
3860                                 amd64_push_reg (code, AMD64_RAX);
3861
3862                         EMIT_FPCOMPARE(code);
3863                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_CC_MASK);
3864                         if (ins->opcode == OP_FCLT_UN) {
3865                                 guchar *is_not_zero_check, *end_jump;
3866                                 is_not_zero_check = code;
3867                                 x86_branch8 (code, X86_CC_NZ, 0, TRUE);
3868                                 end_jump = code;
3869                                 x86_jump8 (code, 0);
3870                                 amd64_patch (is_not_zero_check, code);
3871                                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_CC_MASK);
3872
3873                                 amd64_patch (end_jump, code);
3874                         }
3875                         amd64_set_reg (code, X86_CC_EQ, ins->dreg, TRUE);
3876                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
3877
3878                         if (ins->dreg != AMD64_RAX)
3879                                 amd64_pop_reg (code, AMD64_RAX);
3880                         break;
3881                 case OP_FCGT:
3882                 case OP_FCGT_UN:
3883                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
3884                                 /* zeroing the register at the start results in
3885                                  * shorter and faster code (we can also remove the widening op)
3886                                  */
3887                                 guchar *unordered_check;
3888                                 amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
3889                                 if (use_sse2)
3890                                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
3891                                 else {
3892                                         amd64_fcomip (code, 1);
3893                                         amd64_fstp (code, 0);
3894                                 }
3895                                 if (ins->opcode == OP_FCGT) {
3896                                         unordered_check = code;
3897                                         x86_branch8 (code, X86_CC_P, 0, FALSE);
3898                                         amd64_set_reg (code, X86_CC_LT, ins->dreg, FALSE);
3899                                         amd64_patch (unordered_check, code);
3900                                 } else {
3901                                         amd64_set_reg (code, X86_CC_LT, ins->dreg, FALSE);
3902                                 }
3903                                 break;
3904                         }
3905                         if (ins->dreg != AMD64_RAX)
3906                                 amd64_push_reg (code, AMD64_RAX);
3907
3908                         EMIT_FPCOMPARE(code);
3909                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_CC_MASK);
3910                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
3911                         if (ins->opcode == OP_FCGT_UN) {
3912                                 guchar *is_not_zero_check, *end_jump;
3913                                 is_not_zero_check = code;
3914                                 x86_branch8 (code, X86_CC_NZ, 0, TRUE);
3915                                 end_jump = code;
3916                                 x86_jump8 (code, 0);
3917                                 amd64_patch (is_not_zero_check, code);
3918                                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_CC_MASK);
3919
3920                                 amd64_patch (end_jump, code);
3921                         }
3922                         amd64_set_reg (code, X86_CC_EQ, ins->dreg, TRUE);
3923                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
3924
3925                         if (ins->dreg != AMD64_RAX)
3926                                 amd64_pop_reg (code, AMD64_RAX);
3927                         break;
3928                 case OP_FCLT_MEMBASE:
3929                 case OP_FCGT_MEMBASE:
3930                 case OP_FCLT_UN_MEMBASE:
3931                 case OP_FCGT_UN_MEMBASE:
3932                 case OP_FCEQ_MEMBASE: {
3933                         guchar *unordered_check, *jump_to_end;
3934                         int x86_cond;
3935                         g_assert (use_sse2);
3936
3937                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
3938                         amd64_sse_comisd_reg_membase (code, ins->sreg1, ins->sreg2, ins->inst_offset);
3939
3940                         switch (ins->opcode) {
3941                         case OP_FCEQ_MEMBASE:
3942                                 x86_cond = X86_CC_EQ;
3943                                 break;
3944                         case OP_FCLT_MEMBASE:
3945                         case OP_FCLT_UN_MEMBASE:
3946                                 x86_cond = X86_CC_LT;
3947                                 break;
3948                         case OP_FCGT_MEMBASE:
3949                         case OP_FCGT_UN_MEMBASE:
3950                                 x86_cond = X86_CC_GT;
3951                                 break;
3952                         default:
3953                                 g_assert_not_reached ();
3954                         }
3955
3956                         unordered_check = code;
3957                         x86_branch8 (code, X86_CC_P, 0, FALSE);
3958                         amd64_set_reg (code, x86_cond, ins->dreg, FALSE);
3959
3960                         switch (ins->opcode) {
3961                         case OP_FCEQ_MEMBASE:
3962                         case OP_FCLT_MEMBASE:
3963                         case OP_FCGT_MEMBASE:
3964                                 amd64_patch (unordered_check, code);
3965                                 break;
3966                         case OP_FCLT_UN_MEMBASE:
3967                         case OP_FCGT_UN_MEMBASE:
3968                                 jump_to_end = code;
3969                                 x86_jump8 (code, 0);
3970                                 amd64_patch (unordered_check, code);
3971                                 amd64_inc_reg (code, ins->dreg);
3972                                 amd64_patch (jump_to_end, code);
3973                                 break;
3974                         default:
3975                                 break;
3976                         }
3977                         break;
3978                 }
3979                 case OP_FBEQ:
3980                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
3981                                 guchar *jump = code;
3982                                 x86_branch8 (code, X86_CC_P, 0, TRUE);
3983                                 EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
3984                                 amd64_patch (jump, code);
3985                                 break;
3986                         }
3987                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0x4000);
3988                         EMIT_COND_BRANCH (ins, X86_CC_EQ, TRUE);
3989                         break;
3990                 case OP_FBNE_UN:
3991                         /* Branch if C013 != 100 */
3992                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
3993                                 /* branch if !ZF or (PF|CF) */
3994                                 EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
3995                                 EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
3996                                 EMIT_COND_BRANCH (ins, X86_CC_B, FALSE);
3997                                 break;
3998                         }
3999                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C3);
4000                         EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
4001                         break;
4002                 case OP_FBLT:
4003                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4004                                 EMIT_COND_BRANCH (ins, X86_CC_GT, FALSE);
4005                                 break;
4006                         }
4007                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4008                         break;
4009                 case OP_FBLT_UN:
4010                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4011                                 EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
4012                                 EMIT_COND_BRANCH (ins, X86_CC_GT, FALSE);
4013                                 break;
4014                         }
4015                         if (ins->opcode == OP_FBLT_UN) {
4016                                 guchar *is_not_zero_check, *end_jump;
4017                                 is_not_zero_check = code;
4018                                 x86_branch8 (code, X86_CC_NZ, 0, TRUE);
4019                                 end_jump = code;
4020                                 x86_jump8 (code, 0);
4021                                 amd64_patch (is_not_zero_check, code);
4022                                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_CC_MASK);
4023
4024                                 amd64_patch (end_jump, code);
4025                         }
4026                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4027                         break;
4028                 case OP_FBGT:
4029                 case OP_FBGT_UN:
4030                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4031                                 if (ins->opcode == OP_FBGT) {
4032                                         guchar *br1;
4033
4034                                         /* skip branch if C1=1 */
4035                                         br1 = code;
4036                                         x86_branch8 (code, X86_CC_P, 0, FALSE);
4037                                         /* branch if (C0 | C3) = 1 */
4038                                         EMIT_COND_BRANCH (ins, X86_CC_LT, FALSE);
4039                                         amd64_patch (br1, code);
4040                                         break;
4041                                 } else {
4042                                         EMIT_COND_BRANCH (ins, X86_CC_LT, FALSE);
4043                                 }
4044                                 break;
4045                         }
4046                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4047                         if (ins->opcode == OP_FBGT_UN) {
4048                                 guchar *is_not_zero_check, *end_jump;
4049                                 is_not_zero_check = code;
4050                                 x86_branch8 (code, X86_CC_NZ, 0, TRUE);
4051                                 end_jump = code;
4052                                 x86_jump8 (code, 0);
4053                                 amd64_patch (is_not_zero_check, code);
4054                                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_CC_MASK);
4055
4056                                 amd64_patch (end_jump, code);
4057                         }
4058                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4059                         break;
4060                 case OP_FBGE:
4061                         /* Branch if C013 == 100 or 001 */
4062                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4063                                 guchar *br1;
4064
4065                                 /* skip branch if C1=1 */
4066                                 br1 = code;
4067                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
4068                                 /* branch if (C0 | C3) = 1 */
4069                                 EMIT_COND_BRANCH (ins, X86_CC_BE, FALSE);
4070                                 amd64_patch (br1, code);
4071                                 break;
4072                         }
4073                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4074                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4075                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C3);
4076                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4077                         break;
4078                 case OP_FBGE_UN:
4079                         /* Branch if C013 == 000 */
4080                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4081                                 EMIT_COND_BRANCH (ins, X86_CC_LE, FALSE);
4082                                 break;
4083                         }
4084                         EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
4085                         break;
4086                 case OP_FBLE:
4087                         /* Branch if C013=000 or 100 */
4088                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4089                                 guchar *br1;
4090
4091                                 /* skip branch if C1=1 */
4092                                 br1 = code;
4093                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
4094                                 /* branch if C0=0 */
4095                                 EMIT_COND_BRANCH (ins, X86_CC_NB, FALSE);
4096                                 amd64_patch (br1, code);
4097                                 break;
4098                         }
4099                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, (X86_FP_C0|X86_FP_C1));
4100                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0);
4101                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4102                         break;
4103                 case OP_FBLE_UN:
4104                         /* Branch if C013 != 001 */
4105                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4106                                 EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
4107                                 EMIT_COND_BRANCH (ins, X86_CC_GE, FALSE);
4108                                 break;
4109                         }
4110                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4111                         EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
4112                         break;
4113                 case OP_CKFINITE: {
4114                         if (use_sse2) {
4115                                 /* Transfer value to the fp stack */
4116                                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 16);
4117                                 amd64_movsd_membase_reg (code, AMD64_RSP, 0, ins->sreg1);
4118                                 amd64_fld_membase (code, AMD64_RSP, 0, TRUE);
4119                         }
4120                         amd64_push_reg (code, AMD64_RAX);
4121                         amd64_fxam (code);
4122                         amd64_fnstsw (code);
4123                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, 0x4100);
4124                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4125                         amd64_pop_reg (code, AMD64_RAX);
4126                         if (use_sse2) {
4127                                 amd64_fstp (code, 0);
4128                         }
4129                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_EQ, FALSE, "ArithmeticException");
4130                         if (use_sse2)
4131                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 16);
4132                         break;
4133                 }
4134                 case OP_TLS_GET: {
4135                         code = emit_tls_get (code, ins->dreg, ins->inst_offset);
4136                         break;
4137                 }
4138                 case OP_MEMORY_BARRIER: {
4139                         /* Not needed on amd64 */
4140                         break;
4141                 }
4142                 case OP_ATOMIC_ADD_I4:
4143                 case OP_ATOMIC_ADD_I8: {
4144                         int dreg = ins->dreg;
4145                         guint32 size = (ins->opcode == OP_ATOMIC_ADD_I4) ? 4 : 8;
4146
4147                         if (dreg == ins->inst_basereg)
4148                                 dreg = AMD64_R11;
4149
4150                         if (dreg != ins->sreg2)
4151                                 amd64_mov_reg_reg (code, ins->dreg, ins->sreg2, size);
4152
4153                         x86_prefix (code, X86_LOCK_PREFIX);
4154                         amd64_xadd_membase_reg (code, ins->inst_basereg, ins->inst_offset, dreg, size);
4155
4156                         if (dreg != ins->dreg)
4157                                 amd64_mov_reg_reg (code, ins->dreg, dreg, size);
4158
4159                         break;
4160                 }
4161                 case OP_ATOMIC_ADD_NEW_I4:
4162                 case OP_ATOMIC_ADD_NEW_I8: {
4163                         int dreg = ins->dreg;
4164                         guint32 size = (ins->opcode == OP_ATOMIC_ADD_NEW_I4) ? 4 : 8;
4165
4166                         if ((dreg == ins->sreg2) || (dreg == ins->inst_basereg))
4167                                 dreg = AMD64_R11;
4168
4169                         amd64_mov_reg_reg (code, dreg, ins->sreg2, size);
4170                         amd64_prefix (code, X86_LOCK_PREFIX);
4171                         amd64_xadd_membase_reg (code, ins->inst_basereg, ins->inst_offset, dreg, size);
4172                         /* dreg contains the old value, add with sreg2 value */
4173                         amd64_alu_reg_reg_size (code, X86_ADD, dreg, ins->sreg2, size);
4174
4175                         if (ins->dreg != dreg)
4176                                 amd64_mov_reg_reg (code, ins->dreg, dreg, size);
4177
4178                         break;
4179                 }
4180                 case OP_ATOMIC_EXCHANGE_I4:
4181                 case OP_ATOMIC_EXCHANGE_I8: {
4182                         guchar *br[2];
4183                         int sreg2 = ins->sreg2;
4184                         int breg = ins->inst_basereg;
4185                         guint32 size = (ins->opcode == OP_ATOMIC_EXCHANGE_I4) ? 4 : 8;
4186
4187                         /*
4188                          * See http://msdn.microsoft.com/msdnmag/issues/0700/Win32/ for
4189                          * an explanation of how this works.
4190                          */
4191
4192                         /* cmpxchg uses eax as comperand, need to make sure we can use it
4193                          * hack to overcome limits in x86 reg allocator
4194                          * (req: dreg == eax and sreg2 != eax and breg != eax)
4195                          */
4196                         /* The pushes invalidate rsp */
4197                         if ((breg == AMD64_RAX) || (breg == AMD64_RSP)) {
4198                                 amd64_mov_reg_reg (code, AMD64_R11, breg, 8);
4199                                 breg = AMD64_R11;
4200                         }
4201
4202                         if (ins->dreg != AMD64_RAX)
4203                                 amd64_push_reg (code, AMD64_RAX);
4204
4205                         /* We need the EAX reg for the cmpxchg */
4206                         if (ins->sreg2 == AMD64_RAX) {
4207                                 amd64_push_reg (code, AMD64_RDX);
4208                                 amd64_mov_reg_reg (code, AMD64_RDX, AMD64_RAX, size);
4209                                 sreg2 = AMD64_RDX;
4210                         }
4211
4212                         amd64_mov_reg_membase (code, AMD64_RAX, breg, ins->inst_offset, size);
4213
4214                         br [0] = code; amd64_prefix (code, X86_LOCK_PREFIX);
4215                         amd64_cmpxchg_membase_reg_size (code, breg, ins->inst_offset, sreg2, size);
4216                         br [1] = code; amd64_branch8 (code, X86_CC_NE, -1, FALSE);
4217                         amd64_patch (br [1], br [0]);
4218
4219                         if (ins->dreg != AMD64_RAX) {
4220                                 amd64_mov_reg_reg (code, ins->dreg, AMD64_RAX, size);
4221                                 amd64_pop_reg (code, AMD64_RAX);
4222                         }
4223
4224                         if (ins->sreg2 != sreg2)
4225                                 amd64_pop_reg (code, AMD64_RDX);
4226
4227                         break;
4228                 }
4229                 default:
4230                         g_warning ("unknown opcode %s in %s()\n", mono_inst_name (ins->opcode), __FUNCTION__);
4231                         g_assert_not_reached ();
4232                 }
4233
4234                 if ((code - cfg->native_code - offset) > max_len) {
4235                         g_warning ("wrong maximal instruction length of instruction %s (expected %d, got %ld)",
4236                                    mono_inst_name (ins->opcode), max_len, code - cfg->native_code - offset);
4237                         g_assert_not_reached ();
4238                 }
4239
4240                 cpos += max_len;
4241
4242                 last_ins = ins;
4243                 last_offset = offset;
4244
4245                 ins = ins->next;
4246         }
4247
4248         cfg->code_len = code - cfg->native_code;
4249 }
4250
4251 void
4252 mono_arch_register_lowlevel_calls (void)
4253 {
4254 }
4255
4256 void
4257 mono_arch_patch_code (MonoMethod *method, MonoDomain *domain, guint8 *code, MonoJumpInfo *ji, gboolean run_cctors)
4258 {
4259         MonoJumpInfo *patch_info;
4260         gboolean compile_aot = !run_cctors;
4261
4262         for (patch_info = ji; patch_info; patch_info = patch_info->next) {
4263                 unsigned char *ip = patch_info->ip.i + code;
4264                 const unsigned char *target;
4265
4266                 target = mono_resolve_patch_target (method, domain, code, patch_info, run_cctors);
4267
4268                 if (compile_aot) {
4269                         switch (patch_info->type) {
4270                         case MONO_PATCH_INFO_BB:
4271                         case MONO_PATCH_INFO_LABEL:
4272                                 break;
4273                         default:
4274                                 /* No need to patch these */
4275                                 continue;
4276                         }
4277                 }
4278
4279                 switch (patch_info->type) {
4280                 case MONO_PATCH_INFO_NONE:
4281                         continue;
4282                 case MONO_PATCH_INFO_METHOD_REL:
4283                 case MONO_PATCH_INFO_R8:
4284                 case MONO_PATCH_INFO_R4:
4285                         g_assert_not_reached ();
4286                         continue;
4287                 case MONO_PATCH_INFO_BB:
4288                         break;
4289                 default:
4290                         break;
4291                 }
4292
4293                 /*
4294                  * Debug code to help track down problems where the target of a near call is
4295                  * is not valid.
4296                  */
4297                 if (amd64_is_near_call (ip)) {
4298                         gint64 disp = (guint8*)target - (guint8*)ip;
4299
4300                         if (!amd64_is_imm32 (disp)) {
4301                                 printf ("TYPE: %d\n", patch_info->type);
4302                                 switch (patch_info->type) {
4303                                 case MONO_PATCH_INFO_INTERNAL_METHOD:
4304                                         printf ("V: %s\n", patch_info->data.name);
4305                                         break;
4306                                 case MONO_PATCH_INFO_METHOD_JUMP:
4307                                 case MONO_PATCH_INFO_METHOD:
4308                                         printf ("V: %s\n", patch_info->data.method->name);
4309                                         break;
4310                                 default:
4311                                         break;
4312                                 }
4313                         }
4314                 }
4315
4316                 amd64_patch (ip, (gpointer)target);
4317         }
4318 }
4319
4320 /*
4321  * This macro is used for testing whenever the unwinder works correctly at every point
4322  * where an async exception can happen.
4323  */
4324 /* This will generate a SIGSEGV at the given point in the code */
4325 #define async_exc_point(code) do { \
4326     if (mono_inject_async_exc_method && mono_method_desc_full_match (mono_inject_async_exc_method, cfg->method)) { \
4327          if (cfg->arch.async_point_count == mono_inject_async_exc_pos) \
4328              amd64_mov_reg_mem (code, AMD64_RAX, 0, 4); \
4329          cfg->arch.async_point_count ++; \
4330     } \
4331 } while (0)
4332
4333 guint8 *
4334 mono_arch_emit_prolog (MonoCompile *cfg)
4335 {
4336         MonoMethod *method = cfg->method;
4337         MonoBasicBlock *bb;
4338         MonoMethodSignature *sig;
4339         MonoInst *inst;
4340         int alloc_size, pos, max_offset, i, quad;
4341         guint8 *code;
4342         CallInfo *cinfo;
4343         gint32 lmf_offset = cfg->arch.lmf_offset;
4344
4345         cfg->code_size =  MAX (((MonoMethodNormal *)method)->header->code_size * 4, 1024);
4346
4347         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
4348                 cfg->code_size += 512;
4349
4350         code = cfg->native_code = g_malloc (cfg->code_size);
4351
4352         /* Amount of stack space allocated by register saving code */
4353         pos = 0;
4354
4355         /*
4356          * The prolog consists of the following parts:
4357          * FP present:
4358          * - push rbp, mov rbp, rsp
4359          * - save callee saved regs using pushes
4360          * - allocate frame
4361          * - save lmf if needed
4362          * FP not present:
4363          * - allocate frame
4364          * - save lmf if needed
4365          * - save callee saved regs using moves
4366          */
4367
4368         async_exc_point (code);
4369
4370         if (!cfg->arch.omit_fp) {
4371                 amd64_push_reg (code, AMD64_RBP);
4372                 async_exc_point (code);
4373                 amd64_mov_reg_reg (code, AMD64_RBP, AMD64_RSP, sizeof (gpointer));
4374                 async_exc_point (code);
4375         }
4376
4377         /* Save callee saved registers */
4378         if (!cfg->arch.omit_fp && !method->save_lmf) {
4379                 for (i = 0; i < AMD64_NREG; ++i)
4380                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
4381                                 amd64_push_reg (code, i);
4382                                 pos += sizeof (gpointer);
4383                                 async_exc_point (code);
4384                         }
4385         }
4386
4387         alloc_size = ALIGN_TO (cfg->stack_offset, MONO_ARCH_FRAME_ALIGNMENT);
4388
4389         alloc_size -= pos;
4390
4391         if (cfg->arch.omit_fp)
4392                 /*
4393                  * On enter, the stack is misaligned by the the pushing of the return
4394                  * address. It is either made aligned by the pushing of %rbp, or by
4395                  * this.
4396                  */
4397                 alloc_size += 8;
4398
4399         cfg->arch.stack_alloc_size = alloc_size;
4400
4401         /* Allocate stack frame */
4402         if (alloc_size) {
4403                 /* See mono_emit_stack_alloc */
4404 #if defined(PLATFORM_WIN32) || defined(MONO_ARCH_SIGSEGV_ON_ALTSTACK)
4405                 guint32 remaining_size = alloc_size;
4406                 while (remaining_size >= 0x1000) {
4407                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 0x1000);
4408                         async_exc_point (code);
4409                         amd64_test_membase_reg (code, AMD64_RSP, 0, AMD64_RSP);
4410                         remaining_size -= 0x1000;
4411                 }
4412                 if (remaining_size) {
4413                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, remaining_size);
4414                         async_exc_point (code);
4415                 }
4416 #else
4417                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, alloc_size);
4418                 async_exc_point (code);
4419 #endif
4420         }
4421
4422         /* Stack alignment check */
4423 #if 0
4424         {
4425                 amd64_mov_reg_reg (code, AMD64_RAX, AMD64_RSP, 8);
4426                 amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, 0xf);
4427                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0);
4428                 x86_branch8 (code, X86_CC_EQ, 2, FALSE);
4429                 amd64_breakpoint (code);
4430         }
4431 #endif
4432
4433         /* Save LMF */
4434         if (method->save_lmf) {
4435                 /* Save ip */
4436                 amd64_lea_membase (code, AMD64_R11, AMD64_RIP, 0);
4437                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rip), AMD64_R11, 8);
4438                 /* Save fp */
4439                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, ebp), AMD64_RBP, 8);
4440                 /* Save sp */
4441                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rsp), AMD64_RSP, 8);
4442                 /* Skip method (only needed for trampoline LMF frames) */
4443                 /* Save callee saved regs */
4444                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbx), AMD64_RBX, 8);
4445                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r12), AMD64_R12, 8);
4446                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r13), AMD64_R13, 8);
4447                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r14), AMD64_R14, 8);
4448                 amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r15), AMD64_R15, 8);
4449         }
4450
4451         /* Save callee saved registers */
4452         if (cfg->arch.omit_fp && !method->save_lmf) {
4453                 gint32 save_area_offset = 0;
4454
4455                 /* Save caller saved registers after sp is adjusted */
4456                 /* The registers are saved at the bottom of the frame */
4457                 /* FIXME: Optimize this so the regs are saved at the end of the frame in increasing order */
4458                 for (i = 0; i < AMD64_NREG; ++i)
4459                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
4460                                 amd64_mov_membase_reg (code, AMD64_RSP, save_area_offset, i, 8);
4461                                 save_area_offset += 8;
4462                                 async_exc_point (code);
4463                         }
4464         }
4465
4466         /* compute max_offset in order to use short forward jumps */
4467         max_offset = 0;
4468         if (cfg->opt & MONO_OPT_BRANCH) {
4469                 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4470                         MonoInst *ins = bb->code;
4471                         bb->max_offset = max_offset;
4472
4473                         if (cfg->prof_options & MONO_PROFILE_COVERAGE)
4474                                 max_offset += 6;
4475                         /* max alignment for loops */
4476                         if ((cfg->opt & MONO_OPT_LOOP) && bb_is_loop_start (bb))
4477                                 max_offset += LOOP_ALIGNMENT;
4478
4479                         while (ins) {
4480                                 if (ins->opcode == OP_LABEL)
4481                                         ins->inst_c1 = max_offset;
4482
4483                                 max_offset += ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
4484                                 ins = ins->next;
4485                         }
4486                 }
4487         }
4488
4489         sig = mono_method_signature (method);
4490         pos = 0;
4491
4492         cinfo = cfg->arch.cinfo;
4493
4494         if (sig->ret->type != MONO_TYPE_VOID) {
4495                 if ((cinfo->ret.storage == ArgInIReg) && (cfg->ret->opcode != OP_REGVAR)) {
4496                         /* Save volatile arguments to the stack */
4497                         amd64_mov_membase_reg (code, cfg->ret->inst_basereg, cfg->ret->inst_offset, cinfo->ret.reg, 8);
4498                 }
4499         }
4500
4501         /* Keep this in sync with emit_load_volatile_arguments */
4502         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
4503                 ArgInfo *ainfo = cinfo->args + i;
4504                 gint32 stack_offset;
4505                 MonoType *arg_type;
4506                 inst = cfg->args [i];
4507
4508                 if (sig->hasthis && (i == 0))
4509                         arg_type = &mono_defaults.object_class->byval_arg;
4510                 else
4511                         arg_type = sig->params [i - sig->hasthis];
4512
4513                 stack_offset = ainfo->offset + ARGS_OFFSET;
4514
4515                 /* Save volatile arguments to the stack */
4516                 if (inst->opcode != OP_REGVAR) {
4517                         switch (ainfo->storage) {
4518                         case ArgInIReg: {
4519                                 guint32 size = 8;
4520
4521                                 /* FIXME: I1 etc */
4522                                 /*
4523                                 if (stack_offset & 0x1)
4524                                         size = 1;
4525                                 else if (stack_offset & 0x2)
4526                                         size = 2;
4527                                 else if (stack_offset & 0x4)
4528                                         size = 4;
4529                                 else
4530                                         size = 8;
4531                                 */
4532                                 amd64_mov_membase_reg (code, inst->inst_basereg, inst->inst_offset, ainfo->reg, size);
4533                                 break;
4534                         }
4535                         case ArgInFloatSSEReg:
4536                                 amd64_movss_membase_reg (code, inst->inst_basereg, inst->inst_offset, ainfo->reg);
4537                                 break;
4538                         case ArgInDoubleSSEReg:
4539                                 amd64_movsd_membase_reg (code, inst->inst_basereg, inst->inst_offset, ainfo->reg);
4540                                 break;
4541                         case ArgValuetypeInReg:
4542                                 for (quad = 0; quad < 2; quad ++) {
4543                                         switch (ainfo->pair_storage [quad]) {
4544                                         case ArgInIReg:
4545                                                 amd64_mov_membase_reg (code, inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad], sizeof (gpointer));
4546                                                 break;
4547                                         case ArgInFloatSSEReg:
4548                                                 amd64_movss_membase_reg (code, inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad]);
4549                                                 break;
4550                                         case ArgInDoubleSSEReg:
4551                                                 amd64_movsd_membase_reg (code, inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad]);
4552                                                 break;
4553                                         case ArgNone:
4554                                                 break;
4555                                         default:
4556                                                 g_assert_not_reached ();
4557                                         }
4558                                 }
4559                                 break;
4560                         default:
4561                                 break;
4562                         }
4563                 }
4564
4565                 if (inst->opcode == OP_REGVAR) {
4566                         /* Argument allocated to (non-volatile) register */
4567                         switch (ainfo->storage) {
4568                         case ArgInIReg:
4569                                 amd64_mov_reg_reg (code, inst->dreg, ainfo->reg, 8);
4570                                 break;
4571                         case ArgOnStack:
4572                                 amd64_mov_reg_membase (code, inst->dreg, AMD64_RBP, ARGS_OFFSET + ainfo->offset, 8);
4573                                 break;
4574                         default:
4575                                 g_assert_not_reached ();
4576                         }
4577                 }
4578         }
4579
4580         /* Might need to attach the thread to the JIT */
4581         if (method->wrapper_type == MONO_WRAPPER_NATIVE_TO_MANAGED) {
4582                 guint64 domain = (guint64)cfg->domain;
4583
4584                 /*
4585                  * The call might clobber argument registers, but they are already
4586                  * saved to the stack/global regs.
4587                  */
4588                 if (lmf_addr_tls_offset != -1) {
4589                         guint8 *buf;
4590
4591                         code = emit_tls_get ( code, AMD64_RAX, lmf_addr_tls_offset);
4592                         amd64_test_reg_reg (code, AMD64_RAX, AMD64_RAX);
4593                         buf = code;
4594                         x86_branch8 (code, X86_CC_NE, 0, 0);
4595                         if ((domain >> 32) == 0)
4596                                 amd64_mov_reg_imm_size (code, AMD64_RDI, domain, 4);
4597                         else
4598                                 amd64_mov_reg_imm_size (code, AMD64_RDI, domain, 8);
4599                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, (gpointer)"mono_jit_thread_attach");
4600                         amd64_patch (buf, code);
4601                 } else {
4602                         g_assert (!cfg->compile_aot);
4603                         if ((domain >> 32) == 0)
4604                                 amd64_mov_reg_imm_size (code, AMD64_RDI, domain, 4);
4605                         else
4606                                 amd64_mov_reg_imm_size (code, AMD64_RDI, domain, 8);
4607                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, (gpointer)"mono_jit_thread_attach");
4608                 }
4609         }
4610
4611         if (method->save_lmf) {
4612                 if ((lmf_tls_offset != -1) && !optimize_for_xen) {
4613                         /*
4614                          * Optimized version which uses the mono_lmf TLS variable instead of indirection
4615                          * through the mono_lmf_addr TLS variable.
4616                          */
4617                         /* %rax = previous_lmf */
4618                         x86_prefix (code, X86_FS_PREFIX);
4619                         amd64_mov_reg_mem (code, AMD64_RAX, lmf_tls_offset, 8);
4620
4621                         /* Save previous_lmf */
4622                         amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), AMD64_RAX, 8);
4623                         /* Set new lmf */
4624                         amd64_lea_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset);
4625                         x86_prefix (code, X86_FS_PREFIX);
4626                         amd64_mov_mem_reg (code, lmf_tls_offset, AMD64_R11, 8);
4627                 } else {
4628                         if (lmf_addr_tls_offset != -1) {
4629                                 /* Load lmf quicky using the FS register */
4630                                 code = emit_tls_get (code, AMD64_RAX, lmf_addr_tls_offset);
4631                         }
4632                         else {
4633                                 /*
4634                                  * The call might clobber argument registers, but they are already
4635                                  * saved to the stack/global regs.
4636                                  */
4637                                 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4638                                                                   (gpointer)"mono_get_lmf_addr");
4639                         }
4640
4641                         /* Save lmf_addr */
4642                         amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, lmf_addr), AMD64_RAX, 8);
4643                         /* Save previous_lmf */
4644                         amd64_mov_reg_membase (code, AMD64_R11, AMD64_RAX, 0, 8);
4645                         amd64_mov_membase_reg (code, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), AMD64_R11, 8);
4646                         /* Set new lmf */
4647                         amd64_lea_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset);
4648                         amd64_mov_membase_reg (code, AMD64_RAX, 0, AMD64_R11, 8);
4649                 }
4650         }
4651
4652         if (mono_jit_trace_calls != NULL && mono_trace_eval (method))
4653                 code = mono_arch_instrument_prolog (cfg, mono_trace_enter_method, code, TRUE);
4654
4655         cfg->code_len = code - cfg->native_code;
4656
4657         g_assert (cfg->code_len < cfg->code_size);
4658
4659         return code;
4660 }
4661
4662 void
4663 mono_arch_emit_epilog (MonoCompile *cfg)
4664 {
4665         MonoMethod *method = cfg->method;
4666         int quad, pos, i;
4667         guint8 *code;
4668         int max_epilog_size = 16;
4669         CallInfo *cinfo;
4670         gint32 lmf_offset = cfg->arch.lmf_offset;
4671
4672         if (cfg->method->save_lmf)
4673                 max_epilog_size += 256;
4674
4675         if (mono_jit_trace_calls != NULL)
4676                 max_epilog_size += 50;
4677
4678         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
4679                 max_epilog_size += 50;
4680
4681         max_epilog_size += (AMD64_NREG * 2);
4682
4683         while (cfg->code_len + max_epilog_size > (cfg->code_size - 16)) {
4684                 cfg->code_size *= 2;
4685                 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
4686                 mono_jit_stats.code_reallocs++;
4687         }
4688
4689         code = cfg->native_code + cfg->code_len;
4690
4691         if (mono_jit_trace_calls != NULL && mono_trace_eval (method))
4692                 code = mono_arch_instrument_epilog (cfg, mono_trace_leave_method, code, TRUE);
4693
4694         /* the code restoring the registers must be kept in sync with OP_JMP */
4695         pos = 0;
4696
4697         if (method->save_lmf) {
4698                 if ((lmf_tls_offset != -1) && !optimize_for_xen) {
4699                         /*
4700                          * Optimized version which uses the mono_lmf TLS variable instead of indirection
4701                          * through the mono_lmf_addr TLS variable.
4702                          */
4703                         /* reg = previous_lmf */
4704                         amd64_mov_reg_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), 8);
4705                         x86_prefix (code, X86_FS_PREFIX);
4706                         amd64_mov_mem_reg (code, lmf_tls_offset, AMD64_R11, 8);
4707                 } else {
4708                         /* Restore previous lmf */
4709                         amd64_mov_reg_membase (code, AMD64_RCX, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), 8);
4710                         amd64_mov_reg_membase (code, AMD64_R11, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, lmf_addr), 8);
4711                         amd64_mov_membase_reg (code, AMD64_R11, 0, AMD64_RCX, 8);
4712                 }
4713
4714                 /* Restore caller saved regs */
4715                 if (cfg->used_int_regs & (1 << AMD64_RBP)) {
4716                         amd64_mov_reg_membase (code, AMD64_RBP, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, ebp), 8);
4717                 }
4718                 if (cfg->used_int_regs & (1 << AMD64_RBX)) {
4719                         amd64_mov_reg_membase (code, AMD64_RBX, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbx), 8);
4720                 }
4721                 if (cfg->used_int_regs & (1 << AMD64_R12)) {
4722                         amd64_mov_reg_membase (code, AMD64_R12, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r12), 8);
4723                 }
4724                 if (cfg->used_int_regs & (1 << AMD64_R13)) {
4725                         amd64_mov_reg_membase (code, AMD64_R13, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r13), 8);
4726                 }
4727                 if (cfg->used_int_regs & (1 << AMD64_R14)) {
4728                         amd64_mov_reg_membase (code, AMD64_R14, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r14), 8);
4729                 }
4730                 if (cfg->used_int_regs & (1 << AMD64_R15)) {
4731                         amd64_mov_reg_membase (code, AMD64_R15, cfg->frame_reg, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r15), 8);
4732                 }
4733         } else {
4734
4735                 if (cfg->arch.omit_fp) {
4736                         gint32 save_area_offset = 0;
4737
4738                         for (i = 0; i < AMD64_NREG; ++i)
4739                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
4740                                         amd64_mov_reg_membase (code, i, AMD64_RSP, save_area_offset, 8);
4741                                         save_area_offset += 8;
4742                                 }
4743                 }
4744                 else {
4745                         for (i = 0; i < AMD64_NREG; ++i)
4746                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i)))
4747                                         pos -= sizeof (gpointer);
4748
4749                         if (pos) {
4750                                 if (pos == - sizeof (gpointer)) {
4751                                         /* Only one register, so avoid lea */
4752                                         for (i = AMD64_NREG - 1; i > 0; --i)
4753                                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
4754                                                         amd64_mov_reg_membase (code, i, AMD64_RBP, pos, 8);
4755                                                 }
4756                                 }
4757                                 else {
4758                                         amd64_lea_membase (code, AMD64_RSP, AMD64_RBP, pos);
4759
4760                                         /* Pop registers in reverse order */
4761                                         for (i = AMD64_NREG - 1; i > 0; --i)
4762                                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
4763                                                         amd64_pop_reg (code, i);
4764                                                 }
4765                                 }
4766                         }
4767                 }
4768         }
4769
4770         /* Load returned vtypes into registers if needed */
4771         cinfo = cfg->arch.cinfo;
4772         if (cinfo->ret.storage == ArgValuetypeInReg) {
4773                 ArgInfo *ainfo = &cinfo->ret;
4774                 MonoInst *inst = cfg->ret;
4775
4776                 for (quad = 0; quad < 2; quad ++) {
4777                         switch (ainfo->pair_storage [quad]) {
4778                         case ArgInIReg:
4779                                 amd64_mov_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)), sizeof (gpointer));
4780                                 break;
4781                         case ArgInFloatSSEReg:
4782                                 amd64_movss_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)));
4783                                 break;
4784                         case ArgInDoubleSSEReg:
4785                                 amd64_movsd_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)));
4786                                 break;
4787                         case ArgNone:
4788                                 break;
4789                         default:
4790                                 g_assert_not_reached ();
4791                         }
4792                 }
4793         }
4794
4795         if (cfg->arch.omit_fp) {
4796                 if (cfg->arch.stack_alloc_size)
4797                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, cfg->arch.stack_alloc_size);
4798         } else {
4799                 amd64_leave (code);
4800         }
4801         async_exc_point (code);
4802         amd64_ret (code);
4803
4804         cfg->code_len = code - cfg->native_code;
4805
4806         g_assert (cfg->code_len < cfg->code_size);
4807
4808         if (cfg->arch.omit_fp) {
4809                 /*
4810                  * Encode the stack size into used_int_regs so the exception handler
4811                  * can access it.
4812                  */
4813                 g_assert (cfg->arch.stack_alloc_size < (1 << 16));
4814                 cfg->used_int_regs |= (1 << 31) | (cfg->arch.stack_alloc_size << 16);
4815         }
4816 }
4817
4818 void
4819 mono_arch_emit_exceptions (MonoCompile *cfg)
4820 {
4821         MonoJumpInfo *patch_info;
4822         int nthrows, i;
4823         guint8 *code;
4824         MonoClass *exc_classes [16];
4825         guint8 *exc_throw_start [16], *exc_throw_end [16];
4826         guint32 code_size = 0;
4827
4828         /* Compute needed space */
4829         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
4830                 if (patch_info->type == MONO_PATCH_INFO_EXC)
4831                         code_size += 40;
4832                 if (patch_info->type == MONO_PATCH_INFO_R8)
4833                         code_size += 8 + 15; /* sizeof (double) + alignment */
4834                 if (patch_info->type == MONO_PATCH_INFO_R4)
4835                         code_size += 4 + 15; /* sizeof (float) + alignment */
4836         }
4837
4838         while (cfg->code_len + code_size > (cfg->code_size - 16)) {
4839                 cfg->code_size *= 2;
4840                 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
4841                 mono_jit_stats.code_reallocs++;
4842         }
4843
4844         code = cfg->native_code + cfg->code_len;
4845
4846         /* add code to raise exceptions */
4847         nthrows = 0;
4848         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
4849                 switch (patch_info->type) {
4850                 case MONO_PATCH_INFO_EXC: {
4851                         MonoClass *exc_class;
4852                         guint8 *buf, *buf2;
4853                         guint32 throw_ip;
4854
4855                         amd64_patch (patch_info->ip.i + cfg->native_code, code);
4856
4857                         exc_class = mono_class_from_name (mono_defaults.corlib, "System", patch_info->data.name);
4858                         g_assert (exc_class);
4859                         throw_ip = patch_info->ip.i;
4860
4861                         //x86_breakpoint (code);
4862                         /* Find a throw sequence for the same exception class */
4863                         for (i = 0; i < nthrows; ++i)
4864                                 if (exc_classes [i] == exc_class)
4865                                         break;
4866                         if (i < nthrows) {
4867                                 amd64_mov_reg_imm (code, AMD64_RSI, (exc_throw_end [i] - cfg->native_code) - throw_ip);
4868                                 x86_jump_code (code, exc_throw_start [i]);
4869                                 patch_info->type = MONO_PATCH_INFO_NONE;
4870                         }
4871                         else {
4872                                 buf = code;
4873                                 amd64_mov_reg_imm_size (code, AMD64_RSI, 0xf0f0f0f0, 4);
4874                                 buf2 = code;
4875
4876                                 if (nthrows < 16) {
4877                                         exc_classes [nthrows] = exc_class;
4878                                         exc_throw_start [nthrows] = code;
4879                                 }
4880
4881                                 amd64_mov_reg_imm (code, AMD64_RDI, exc_class->type_token);
4882                                 patch_info->data.name = "mono_arch_throw_corlib_exception";
4883                                 patch_info->type = MONO_PATCH_INFO_INTERNAL_METHOD;
4884                                 patch_info->ip.i = code - cfg->native_code;
4885
4886                                 code = emit_call_body (cfg, code, patch_info->type, patch_info->data.name);
4887
4888                                 amd64_mov_reg_imm (buf, AMD64_RSI, (code - cfg->native_code) - throw_ip);
4889                                 while (buf < buf2)
4890                                         x86_nop (buf);
4891
4892                                 if (nthrows < 16) {
4893                                         exc_throw_end [nthrows] = code;
4894                                         nthrows ++;
4895                                 }
4896                         }
4897                         break;
4898                 }
4899                 default:
4900                         /* do nothing */
4901                         break;
4902                 }
4903         }
4904
4905         /* Handle relocations with RIP relative addressing */
4906         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
4907                 gboolean remove = FALSE;
4908
4909                 switch (patch_info->type) {
4910                 case MONO_PATCH_INFO_R8:
4911                 case MONO_PATCH_INFO_R4: {
4912                         guint8 *pos;
4913
4914                         if (use_sse2) {
4915                                 /* The SSE opcodes require a 16 byte alignment */
4916                                 code = (guint8*)ALIGN_TO (code, 16);
4917                         } else {
4918                                 code = (guint8*)ALIGN_TO (code, 8);
4919                         }
4920
4921                         pos = cfg->native_code + patch_info->ip.i;
4922
4923
4924                         if (use_sse2) {
4925                                 if (IS_REX (pos [1]))
4926                                         *(guint32*)(pos + 5) = (guint8*)code - pos - 9;
4927                                 else
4928                                         *(guint32*)(pos + 4) = (guint8*)code - pos - 8;
4929                         } else {
4930                                 *(guint32*)(pos + 3) = (guint8*)code - pos - 7;
4931                         }
4932
4933                         if (patch_info->type == MONO_PATCH_INFO_R8) {
4934                                 *(double*)code = *(double*)patch_info->data.target;
4935                                 code += sizeof (double);
4936                         } else {
4937                                 *(float*)code = *(float*)patch_info->data.target;
4938                                 code += sizeof (float);
4939                         }
4940
4941                         remove = TRUE;
4942                         break;
4943                 }
4944                 default:
4945                         break;
4946                 }
4947
4948                 if (remove) {
4949                         if (patch_info == cfg->patch_info)
4950                                 cfg->patch_info = patch_info->next;
4951                         else {
4952                                 MonoJumpInfo *tmp;
4953
4954                                 for (tmp = cfg->patch_info; tmp->next != patch_info; tmp = tmp->next)
4955                                         ;
4956                                 tmp->next = patch_info->next;
4957                         }
4958                 }
4959         }
4960
4961         cfg->code_len = code - cfg->native_code;
4962
4963         g_assert (cfg->code_len < cfg->code_size);
4964
4965 }
4966
4967 void*
4968 mono_arch_instrument_prolog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
4969 {
4970         guchar *code = p;
4971         CallInfo *cinfo = NULL;
4972         MonoMethodSignature *sig;
4973         MonoInst *inst;
4974         int i, n, stack_area = 0;
4975
4976         /* Keep this in sync with mono_arch_get_argument_info */
4977
4978         if (enable_arguments) {
4979                 /* Allocate a new area on the stack and save arguments there */
4980                 sig = mono_method_signature (cfg->method);
4981
4982                 cinfo = get_call_info (cfg, cfg->mempool, sig, FALSE);
4983
4984                 n = sig->param_count + sig->hasthis;
4985
4986                 stack_area = ALIGN_TO (n * 8, 16);
4987
4988                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, stack_area);
4989
4990                 for (i = 0; i < n; ++i) {
4991                         inst = cfg->args [i];
4992
4993                         if (inst->opcode == OP_REGVAR)
4994                                 amd64_mov_membase_reg (code, AMD64_RSP, (i * 8), inst->dreg, 8);
4995                         else {
4996                                 amd64_mov_reg_membase (code, AMD64_R11, inst->inst_basereg, inst->inst_offset, 8);
4997                                 amd64_mov_membase_reg (code, AMD64_RSP, (i * 8), AMD64_R11, 8);
4998                         }
4999                 }
5000         }
5001
5002         mono_add_patch_info (cfg, code-cfg->native_code, MONO_PATCH_INFO_METHODCONST, cfg->method);
5003         amd64_set_reg_template (code, AMD64_RDI);
5004         amd64_mov_reg_reg (code, AMD64_RSI, AMD64_RSP, 8);
5005         code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)func);
5006
5007         if (enable_arguments)
5008                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, stack_area);
5009
5010         return code;
5011 }
5012
5013 enum {
5014         SAVE_NONE,
5015         SAVE_STRUCT,
5016         SAVE_EAX,
5017         SAVE_EAX_EDX,
5018         SAVE_XMM
5019 };
5020
5021 void*
5022 mono_arch_instrument_epilog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
5023 {
5024         guchar *code = p;
5025         int save_mode = SAVE_NONE;
5026         MonoMethod *method = cfg->method;
5027         int rtype = mono_type_get_underlying_type (mono_method_signature (method)->ret)->type;
5028
5029         switch (rtype) {
5030         case MONO_TYPE_VOID:
5031                 /* special case string .ctor icall */
5032                 if (strcmp (".ctor", method->name) && method->klass == mono_defaults.string_class)
5033                         save_mode = SAVE_EAX;
5034                 else
5035                         save_mode = SAVE_NONE;
5036                 break;
5037         case MONO_TYPE_I8:
5038         case MONO_TYPE_U8:
5039                 save_mode = SAVE_EAX;
5040                 break;
5041         case MONO_TYPE_R4:
5042         case MONO_TYPE_R8:
5043                 save_mode = SAVE_XMM;
5044                 break;
5045         case MONO_TYPE_GENERICINST:
5046                 if (!mono_type_generic_inst_is_valuetype (mono_method_signature (method)->ret)) {
5047                         save_mode = SAVE_EAX;
5048                         break;
5049                 }
5050                 /* Fall through */
5051         case MONO_TYPE_VALUETYPE:
5052                 save_mode = SAVE_STRUCT;
5053                 break;
5054         default:
5055                 save_mode = SAVE_EAX;
5056                 break;
5057         }
5058
5059         /* Save the result and copy it into the proper argument register */
5060         switch (save_mode) {
5061         case SAVE_EAX:
5062                 amd64_push_reg (code, AMD64_RAX);
5063                 /* Align stack */
5064                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
5065                 if (enable_arguments)
5066                         amd64_mov_reg_reg (code, AMD64_RSI, AMD64_RAX, 8);
5067                 break;
5068         case SAVE_STRUCT:
5069                 /* FIXME: */
5070                 if (enable_arguments)
5071                         amd64_mov_reg_imm (code, AMD64_RSI, 0);
5072                 break;
5073         case SAVE_XMM:
5074                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
5075                 amd64_movsd_membase_reg (code, AMD64_RSP, 0, AMD64_XMM0);
5076                 /* Align stack */
5077                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
5078                 /*
5079                  * The result is already in the proper argument register so no copying
5080                  * needed.
5081                  */
5082                 break;
5083         case SAVE_NONE:
5084                 break;
5085         default:
5086                 g_assert_not_reached ();
5087         }
5088
5089         /* Set %al since this is a varargs call */
5090         if (save_mode == SAVE_XMM)
5091                 amd64_mov_reg_imm (code, AMD64_RAX, 1);
5092         else
5093                 amd64_mov_reg_imm (code, AMD64_RAX, 0);
5094
5095         mono_add_patch_info (cfg, code-cfg->native_code, MONO_PATCH_INFO_METHODCONST, method);
5096         amd64_set_reg_template (code, AMD64_RDI);
5097         code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)func);
5098
5099         /* Restore result */
5100         switch (save_mode) {
5101         case SAVE_EAX:
5102                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
5103                 amd64_pop_reg (code, AMD64_RAX);
5104                 break;
5105         case SAVE_STRUCT:
5106                 /* FIXME: */
5107                 break;
5108         case SAVE_XMM:
5109                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
5110                 amd64_movsd_reg_membase (code, AMD64_XMM0, AMD64_RSP, 0);
5111                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
5112                 break;
5113         case SAVE_NONE:
5114                 break;
5115         default:
5116                 g_assert_not_reached ();
5117         }
5118
5119         return code;
5120 }
5121
5122 void
5123 mono_arch_flush_icache (guint8 *code, gint size)
5124 {
5125         /* Not needed */
5126 }
5127
5128 void
5129 mono_arch_flush_register_windows (void)
5130 {
5131 }
5132
5133 gboolean
5134 mono_arch_is_inst_imm (gint64 imm)
5135 {
5136         return amd64_is_imm32 (imm);
5137 }
5138
5139 /*
5140  * Determine whenever the trap whose info is in SIGINFO is caused by
5141  * integer overflow.
5142  */
5143 gboolean
5144 mono_arch_is_int_overflow (void *sigctx, void *info)
5145 {
5146         MonoContext ctx;
5147         guint8* rip;
5148         int reg;
5149         gint64 value;
5150
5151         mono_arch_sigctx_to_monoctx (sigctx, &ctx);
5152
5153         rip = (guint8*)ctx.rip;
5154
5155         if (IS_REX (rip [0])) {
5156                 reg = amd64_rex_b (rip [0]);
5157                 rip ++;
5158         }
5159         else
5160                 reg = 0;
5161
5162         if ((rip [0] == 0xf7) && (x86_modrm_mod (rip [1]) == 0x3) && (x86_modrm_reg (rip [1]) == 0x7)) {
5163                 /* idiv REG */
5164                 reg += x86_modrm_rm (rip [1]);
5165
5166                 switch (reg) {
5167                 case AMD64_RAX:
5168                         value = ctx.rax;
5169                         break;
5170                 case AMD64_RBX:
5171                         value = ctx.rbx;
5172                         break;
5173                 case AMD64_RCX:
5174                         value = ctx.rcx;
5175                         break;
5176                 case AMD64_RDX:
5177                         value = ctx.rdx;
5178                         break;
5179                 case AMD64_RBP:
5180                         value = ctx.rbp;
5181                         break;
5182                 case AMD64_RSP:
5183                         value = ctx.rsp;
5184                         break;
5185                 case AMD64_RSI:
5186                         value = ctx.rsi;
5187                         break;
5188                 case AMD64_RDI:
5189                         value = ctx.rdi;
5190                         break;
5191                 case AMD64_R12:
5192                         value = ctx.r12;
5193                         break;
5194                 case AMD64_R13:
5195                         value = ctx.r13;
5196                         break;
5197                 case AMD64_R14:
5198                         value = ctx.r14;
5199                         break;
5200                 case AMD64_R15:
5201                         value = ctx.r15;
5202                         break;
5203                 default:
5204                         g_assert_not_reached ();
5205                         reg = -1;
5206                 }
5207
5208                 if (value == -1)
5209                         return TRUE;
5210         }
5211
5212         return FALSE;
5213 }
5214
5215 guint32
5216 mono_arch_get_patch_offset (guint8 *code)
5217 {
5218         return 3;
5219 }
5220
5221 gpointer*
5222 mono_arch_get_vcall_slot_addr (guint8* code, gpointer *regs)
5223 {
5224         guint32 reg;
5225         gint32 disp;
5226         guint8 rex = 0;
5227
5228         /* go to the start of the call instruction
5229          *
5230          * address_byte = (m << 6) | (o << 3) | reg
5231          * call opcode: 0xff address_byte displacement
5232          * 0xff m=1,o=2 imm8
5233          * 0xff m=2,o=2 imm32
5234          */
5235         code -= 7;
5236
5237         /*
5238          * A given byte sequence can match more than case here, so we have to be
5239          * really careful about the ordering of the cases. Longer sequences
5240          * come first.
5241          */
5242 #ifdef MONO_ARCH_HAVE_IMT
5243         if ((code [-2] == 0x41) && (code [-1] == 0xbb) && (code [4] == 0xff) && (x86_modrm_mod (code [5]) == 1) && (x86_modrm_reg (code [5]) == 2) && ((signed char)code [6] < 0)) {
5244                 /* IMT-based interface calls: with MONO_ARCH_IMT_REG == r11
5245                  * 41 bb 14 f8 28 08       mov    $0x828f814,%r11d
5246                  * ff 50 fc                call   *0xfffffffc(%rax)
5247                  */
5248                 reg = amd64_modrm_rm (code [5]);
5249                 disp = (signed char)code [6];
5250                 /* R10 is clobbered by the IMT thunk code */
5251                 g_assert (reg != AMD64_R10);
5252         }
5253 #else
5254         if (0) {
5255         }
5256 #endif
5257         else if ((code [-1] == 0x8b) && (amd64_modrm_mod (code [0]) == 0x2) && (code [5] == 0xff) && (amd64_modrm_reg (code [6]) == 0x2) && (amd64_modrm_mod (code [6]) == 0x0)) {
5258                         /*
5259                          * This is a interface call
5260                          * 48 8b 80 f0 e8 ff ff   mov    0xffffffffffffe8f0(%rax),%rax
5261                          * ff 10                  callq  *(%rax)
5262                          */
5263                 if (IS_REX (code [4]))
5264                         rex = code [4];
5265                 reg = amd64_modrm_rm (code [6]);
5266                 disp = 0;
5267                 /* R10 is clobbered by the IMT thunk code */
5268                 g_assert (reg != AMD64_R10);
5269         } else if ((code [0] == 0x41) && (code [1] == 0xff) && (code [2] == 0x15)) {
5270                 /* call OFFSET(%rip) */
5271                 disp = *(guint32*)(code + 3);
5272                 return (gpointer*)(code + disp + 7);
5273         }
5274         else if ((code [1] == 0xff) && (amd64_modrm_reg (code [2]) == 0x2) && (amd64_modrm_mod (code [2]) == 0x2)) {
5275                 /* call *[reg+disp32] */
5276                 if (IS_REX (code [0]))
5277                         rex = code [0];
5278                 reg = amd64_modrm_rm (code [2]);
5279                 disp = *(gint32*)(code + 3);
5280                 /* R10 is clobbered by the IMT thunk code */
5281                 g_assert (reg != AMD64_R10);
5282         }
5283         else if (code [2] == 0xe8) {
5284                 /* call <ADDR> */
5285                 return NULL;
5286         }
5287         else if (IS_REX (code [4]) && (code [5] == 0xff) && (amd64_modrm_reg (code [6]) == 0x2) && (amd64_modrm_mod (code [6]) == 0x3)) {
5288                 /* call *%reg */
5289                 return NULL;
5290         }
5291         else if ((code [4] == 0xff) && (amd64_modrm_reg (code [5]) == 0x2) && (amd64_modrm_mod (code [5]) == 0x1)) {
5292                 /* call *[reg+disp8] */
5293                 if (IS_REX (code [3]))
5294                         rex = code [3];
5295                 reg = amd64_modrm_rm (code [5]);
5296                 disp = *(gint8*)(code + 6);
5297                 //printf ("B: [%%r%d+0x%x]\n", reg, disp);
5298         }
5299         else if ((code [5] == 0xff) && (amd64_modrm_reg (code [6]) == 0x2) && (amd64_modrm_mod (code [6]) == 0x0)) {
5300                         /*
5301                          * This is a interface call: should check the above code can't catch it earlier
5302                          * 8b 40 30   mov    0x30(%eax),%eax
5303                          * ff 10      call   *(%eax)
5304                          */
5305                 if (IS_REX (code [4]))
5306                         rex = code [4];
5307                 reg = amd64_modrm_rm (code [6]);
5308                 disp = 0;
5309         }
5310         else
5311                 g_assert_not_reached ();
5312
5313         reg += amd64_rex_b (rex);
5314
5315         /* R11 is clobbered by the trampoline code */
5316         g_assert (reg != AMD64_R11);
5317
5318         return (gpointer)(((guint64)(regs [reg])) + disp);
5319 }
5320
5321 gpointer
5322 mono_arch_get_this_arg_from_call (MonoMethodSignature *sig, gssize *regs, guint8 *code)
5323 {
5324         if (MONO_TYPE_ISSTRUCT (sig->ret))
5325                 return (gpointer)regs [AMD64_RSI];
5326         else
5327                 return (gpointer)regs [AMD64_RDI];
5328 }
5329
5330 #define MAX_ARCH_DELEGATE_PARAMS 10
5331
5332 gpointer
5333 mono_arch_get_delegate_invoke_impl (MonoMethodSignature *sig, gboolean has_target)
5334 {
5335         guint8 *code, *start;
5336         int i;
5337
5338         if (sig->param_count > MAX_ARCH_DELEGATE_PARAMS)
5339                 return NULL;
5340
5341         /* FIXME: Support more cases */
5342         if (MONO_TYPE_ISSTRUCT (sig->ret))
5343                 return NULL;
5344
5345         if (has_target) {
5346                 static guint8* cached = NULL;
5347                 mono_mini_arch_lock ();
5348                 if (cached) {
5349                         mono_mini_arch_unlock ();
5350                         return cached;
5351                 }
5352
5353                 start = code = mono_global_codeman_reserve (64);
5354
5355                 /* Replace the this argument with the target */
5356                 amd64_mov_reg_reg (code, AMD64_RAX, AMD64_RDI, 8);
5357                 amd64_mov_reg_membase (code, AMD64_RDI, AMD64_RAX, G_STRUCT_OFFSET (MonoDelegate, target), 8);
5358                 amd64_jump_membase (code, AMD64_RAX, G_STRUCT_OFFSET (MonoDelegate, method_ptr));
5359
5360                 g_assert ((code - start) < 64);
5361
5362                 cached = start;
5363                 mono_mini_arch_unlock ();
5364         } else {
5365                 static guint8* cache [MAX_ARCH_DELEGATE_PARAMS + 1] = {NULL};
5366                 for (i = 0; i < sig->param_count; ++i)
5367                         if (!mono_is_regsize_var (sig->params [i]))
5368                                 return NULL;
5369                 if (sig->param_count > 4)
5370                         return NULL;
5371
5372                 mono_mini_arch_lock ();
5373                 code = cache [sig->param_count];
5374                 if (code) {
5375                         mono_mini_arch_unlock ();
5376                         return code;
5377                 }
5378
5379                 start = code = mono_global_codeman_reserve (64);
5380
5381                 if (sig->param_count == 0) {
5382                         amd64_jump_membase (code, AMD64_RDI, G_STRUCT_OFFSET (MonoDelegate, method_ptr));
5383                 } else {
5384                         /* We have to shift the arguments left */
5385                         amd64_mov_reg_reg (code, AMD64_RAX, AMD64_RDI, 8);
5386                         for (i = 0; i < sig->param_count; ++i)
5387                                 amd64_mov_reg_reg (code, param_regs [i], param_regs [i + 1], 8);
5388
5389                         amd64_jump_membase (code, AMD64_RAX, G_STRUCT_OFFSET (MonoDelegate, method_ptr));
5390                 }
5391                 g_assert ((code - start) < 64);
5392
5393                 cache [sig->param_count] = start;
5394
5395                 mono_mini_arch_unlock ();
5396         }
5397
5398         return start;
5399 }
5400
5401 /*
5402  * Support for fast access to the thread-local lmf structure using the GS
5403  * segment register on NPTL + kernel 2.6.x.
5404  */
5405
5406 static gboolean tls_offset_inited = FALSE;
5407
5408 void
5409 mono_arch_setup_jit_tls_data (MonoJitTlsData *tls)
5410 {
5411         if (!tls_offset_inited) {
5412                 tls_offset_inited = TRUE;
5413 #ifdef MONO_XEN_OPT
5414                 optimize_for_xen = access ("/proc/xen", F_OK) == 0;
5415 #endif
5416                 appdomain_tls_offset = mono_domain_get_tls_offset ();
5417                 lmf_tls_offset = mono_get_lmf_tls_offset ();
5418                 lmf_addr_tls_offset = mono_get_lmf_addr_tls_offset ();
5419                 thread_tls_offset = mono_thread_get_tls_offset ();
5420         }
5421 }
5422
5423 void
5424 mono_arch_free_jit_tls_data (MonoJitTlsData *tls)
5425 {
5426 }
5427
5428 void
5429 mono_arch_emit_this_vret_args (MonoCompile *cfg, MonoCallInst *inst, int this_reg, int this_type, int vt_reg)
5430 {
5431         MonoCallInst *call = (MonoCallInst*)inst;
5432         CallInfo * cinfo = get_call_info (cfg, cfg->mempool, inst->signature, FALSE);
5433
5434         if (vt_reg != -1) {
5435                 MonoInst *vtarg;
5436
5437                 if (cinfo->ret.storage == ArgValuetypeInReg) {
5438                         /*
5439                          * The valuetype is in RAX:RDX after the call, need to be copied to
5440                          * the stack. Push the address here, so the call instruction can
5441                          * access it.
5442                          */
5443                         MONO_INST_NEW (cfg, vtarg, OP_X86_PUSH);
5444                         vtarg->sreg1 = vt_reg;
5445                         mono_bblock_add_inst (cfg->cbb, vtarg);
5446
5447                         /* Align stack */
5448                         MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 8);
5449                 }
5450                 else {
5451                         MONO_INST_NEW (cfg, vtarg, OP_MOVE);
5452                         vtarg->sreg1 = vt_reg;
5453                         vtarg->dreg = mono_regstate_next_int (cfg->rs);
5454                         mono_bblock_add_inst (cfg->cbb, vtarg);
5455
5456                         mono_call_inst_add_outarg_reg (cfg, call, vtarg->dreg, cinfo->ret.reg, FALSE);
5457                 }
5458         }
5459
5460         /* add the this argument */
5461         if (this_reg != -1) {
5462                 MonoInst *this;
5463                 MONO_INST_NEW (cfg, this, OP_MOVE);
5464                 this->type = this_type;
5465                 this->sreg1 = this_reg;
5466                 this->dreg = mono_regstate_next_int (cfg->rs);
5467                 mono_bblock_add_inst (cfg->cbb, this);
5468
5469                 mono_call_inst_add_outarg_reg (cfg, call, this->dreg, cinfo->args [0].reg, FALSE);
5470         }
5471 }
5472
5473 #ifdef MONO_ARCH_HAVE_IMT
5474
5475 #define CMP_SIZE (6 + 1)
5476 #define CMP_REG_REG_SIZE (4 + 1)
5477 #define BR_SMALL_SIZE 2
5478 #define BR_LARGE_SIZE 6
5479 #define MOV_REG_IMM_SIZE 10
5480 #define MOV_REG_IMM_32BIT_SIZE 6
5481 #define JUMP_REG_SIZE (2 + 1)
5482
5483 static int
5484 imt_branch_distance (MonoIMTCheckItem **imt_entries, int start, int target)
5485 {
5486         int i, distance = 0;
5487         for (i = start; i < target; ++i)
5488                 distance += imt_entries [i]->chunk_size;
5489         return distance;
5490 }
5491
5492 /*
5493  * LOCKING: called with the domain lock held
5494  */
5495 gpointer
5496 mono_arch_build_imt_thunk (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count)
5497 {
5498         int i;
5499         int size = 0;
5500         guint8 *code, *start;
5501         gboolean vtable_is_32bit = ((long)(vtable) == (long)(int)(long)(vtable));
5502
5503         for (i = 0; i < count; ++i) {
5504                 MonoIMTCheckItem *item = imt_entries [i];
5505                 if (item->is_equals) {
5506                         if (item->check_target_idx) {
5507                                 if (!item->compare_done) {
5508                                         if (amd64_is_imm32 (item->method))
5509                                                 item->chunk_size += CMP_SIZE;
5510                                         else
5511                                                 item->chunk_size += MOV_REG_IMM_SIZE + CMP_REG_REG_SIZE;
5512                                 }
5513                                 if (vtable_is_32bit)
5514                                         item->chunk_size += MOV_REG_IMM_32BIT_SIZE;
5515                                 else
5516                                         item->chunk_size += MOV_REG_IMM_SIZE;
5517                                 item->chunk_size += BR_SMALL_SIZE + JUMP_REG_SIZE;
5518                         } else {
5519                                 if (vtable_is_32bit)
5520                                         item->chunk_size += MOV_REG_IMM_32BIT_SIZE;
5521                                 else
5522                                         item->chunk_size += MOV_REG_IMM_SIZE;
5523                                 item->chunk_size += JUMP_REG_SIZE;
5524                                 /* with assert below:
5525                                  * item->chunk_size += CMP_SIZE + BR_SMALL_SIZE + 1;
5526                                  */
5527                         }
5528                 } else {
5529                         if (amd64_is_imm32 (item->method))
5530                                 item->chunk_size += CMP_SIZE;
5531                         else
5532                                 item->chunk_size += MOV_REG_IMM_SIZE + CMP_REG_REG_SIZE;
5533                         item->chunk_size += BR_LARGE_SIZE;
5534                         imt_entries [item->check_target_idx]->compare_done = TRUE;
5535                 }
5536                 size += item->chunk_size;
5537         }
5538         code = mono_code_manager_reserve (domain->code_mp, size);
5539         start = code;
5540         for (i = 0; i < count; ++i) {
5541                 MonoIMTCheckItem *item = imt_entries [i];
5542                 item->code_target = code;
5543                 if (item->is_equals) {
5544                         if (item->check_target_idx) {
5545                                 if (!item->compare_done) {
5546                                         if (amd64_is_imm32 (item->method))
5547                                                 amd64_alu_reg_imm (code, X86_CMP, MONO_ARCH_IMT_REG, (guint32)(gssize)item->method);
5548                                         else {
5549                                                 amd64_mov_reg_imm (code, AMD64_R10, item->method);
5550                                                 amd64_alu_reg_reg (code, X86_CMP, MONO_ARCH_IMT_REG, AMD64_R10);
5551                                         }
5552                                 }
5553                                 item->jmp_code = code;
5554                                 amd64_branch8 (code, X86_CC_NE, 0, FALSE);
5555                                 amd64_mov_reg_imm (code, AMD64_R11, & (vtable->vtable [item->vtable_slot]));
5556                                 amd64_jump_membase (code, AMD64_R11, 0);
5557                         } else {
5558                                 /* enable the commented code to assert on wrong method */
5559 #if 0
5560                                 if (amd64_is_imm32 (item->method))
5561                                         amd64_alu_reg_imm (code, X86_CMP, MONO_ARCH_IMT_REG, (guint32)(gssize)item->method);
5562                                 else {
5563                                         amd64_mov_reg_imm (code, AMD64_R10, item->method);
5564                                         amd64_alu_reg_reg (code, X86_CMP, MONO_ARCH_IMT_REG, AMD64_R10);
5565                                 }
5566                                 item->jmp_code = code;
5567                                 amd64_branch8 (code, X86_CC_NE, 0, FALSE);
5568                                 amd64_mov_reg_imm (code, AMD64_R11, & (vtable->vtable [item->vtable_slot]));
5569                                 amd64_jump_membase (code, AMD64_R11, 0);
5570                                 amd64_patch (item->jmp_code, code);
5571                                 amd64_breakpoint (code);
5572                                 item->jmp_code = NULL;
5573 #else
5574                                 amd64_mov_reg_imm (code, AMD64_R11, & (vtable->vtable [item->vtable_slot]));
5575                                 amd64_jump_membase (code, AMD64_R11, 0);
5576 #endif
5577                         }
5578                 } else {
5579                         if (amd64_is_imm32 (item->method))
5580                                 amd64_alu_reg_imm (code, X86_CMP, MONO_ARCH_IMT_REG, (guint32)(gssize)item->method);
5581                         else {
5582                                 amd64_mov_reg_imm (code, AMD64_R10, item->method);
5583                                 amd64_alu_reg_reg (code, X86_CMP, MONO_ARCH_IMT_REG, AMD64_R10);
5584                         }
5585                         item->jmp_code = code;
5586                         if (x86_is_imm8 (imt_branch_distance (imt_entries, i, item->check_target_idx)))
5587                                 x86_branch8 (code, X86_CC_GE, 0, FALSE);
5588                         else
5589                                 x86_branch32 (code, X86_CC_GE, 0, FALSE);
5590                 }
5591                 g_assert (code - item->code_target <= item->chunk_size);
5592         }
5593         /* patch the branches to get to the target items */
5594         for (i = 0; i < count; ++i) {
5595                 MonoIMTCheckItem *item = imt_entries [i];
5596                 if (item->jmp_code) {
5597                         if (item->check_target_idx) {
5598                                 amd64_patch (item->jmp_code, imt_entries [item->check_target_idx]->code_target);
5599                         }
5600                 }
5601         }
5602
5603         mono_stats.imt_thunks_size += code - start;
5604         g_assert (code - start <= size);
5605
5606         return start;
5607 }
5608
5609 MonoMethod*
5610 mono_arch_find_imt_method (gpointer *regs, guint8 *code)
5611 {
5612         /*
5613          * R11 is clobbered by the trampoline code, so we have to retrieve the method
5614          * from the code.
5615          * 41 bb c0 f7 89 00     mov    $0x89f7c0,%r11d
5616          * ff 90 68 ff ff ff     callq  *0xffffffffffffff68(%rax)
5617          */
5618         /* Similar to get_vcall_slot_addr () */
5619
5620         /* Find the start of the call instruction */
5621         code -= 7;
5622         if ((code [-2] == 0x41) && (code [-1] == 0xbb) && (code [4] == 0xff) && (x86_modrm_mod (code [5]) == 1) && (x86_modrm_reg (code [5]) == 2) && ((signed char)code [6] < 0)) {
5623                 /* IMT-based interface calls
5624                  * 41 bb 14 f8 28 08       mov    $0x828f814,%r11
5625                  * ff 50 fc                call   *0xfffffffc(%rax)
5626                  */
5627                 code += 4;
5628         } else if ((code [1] == 0xff) && (amd64_modrm_reg (code [2]) == 0x2) && (amd64_modrm_mod (code [2]) == 0x2)) {
5629                 /* call *[reg+disp32] */
5630                 code += 1;
5631         } else if ((code [4] == 0xff) && (amd64_modrm_reg (code [5]) == 0x2) && (amd64_modrm_mod (code [5]) == 0x1)) {
5632                 /* call *[reg+disp8] */
5633                 code += 4;
5634         } else
5635                 g_assert_not_reached ();
5636
5637         /* Find the start of the mov instruction */
5638         code -= 10;
5639         if (code [0] == 0x49 && code [1] == 0xbb) {
5640                 return (MonoMethod*)*(gssize*)(code + 2);
5641         } else if (code [3] == 0x4d && code [4] == 0x8b && code [5] == 0x1d) {
5642                 /* mov    <OFFSET>(%rip),%r11 */
5643                 return (MonoMethod*)*(gssize*)(code + 10 + *(guint32*)(code + 6));
5644         } else if (code [4] == 0x41 && code [5] == 0xbb) {
5645                 return (MonoMethod*)(gssize)*(guint32*)(code + 6);
5646         } else {
5647                 int i;
5648
5649                 printf ("Unknown call sequence: ");
5650                 for (i = -10; i < 20; ++i)
5651                         printf ("%x ", code [i]);
5652                 g_assert_not_reached ();
5653                 return NULL;
5654         }
5655 }
5656
5657 MonoObject*
5658 mono_arch_find_this_argument (gpointer *regs, MonoMethod *method)
5659 {
5660         return mono_arch_get_this_arg_from_call (mono_method_signature (method), (gssize*)regs, NULL);
5661 }
5662 #endif
5663
5664 MonoInst*
5665 mono_arch_get_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args)
5666 {
5667         MonoInst *ins = NULL;
5668
5669         if (cmethod->klass == mono_defaults.math_class) {
5670                 if (strcmp (cmethod->name, "Sin") == 0) {
5671                         MONO_INST_NEW (cfg, ins, OP_SIN);
5672                         ins->inst_i0 = args [0];
5673                 } else if (strcmp (cmethod->name, "Cos") == 0) {
5674                         MONO_INST_NEW (cfg, ins, OP_COS);
5675                         ins->inst_i0 = args [0];
5676                 } else if (strcmp (cmethod->name, "Tan") == 0) {
5677                         if (use_sse2)
5678                                 return ins;
5679                         MONO_INST_NEW (cfg, ins, OP_TAN);
5680                         ins->inst_i0 = args [0];
5681                 } else if (strcmp (cmethod->name, "Atan") == 0) {
5682                         if (use_sse2)
5683                                 return ins;
5684                         MONO_INST_NEW (cfg, ins, OP_ATAN);
5685                         ins->inst_i0 = args [0];
5686                 } else if (strcmp (cmethod->name, "Sqrt") == 0) {
5687                         MONO_INST_NEW (cfg, ins, OP_SQRT);
5688                         ins->inst_i0 = args [0];
5689                 } else if (strcmp (cmethod->name, "Abs") == 0 && fsig->params [0]->type == MONO_TYPE_R8) {
5690                         MONO_INST_NEW (cfg, ins, OP_ABS);
5691                         ins->inst_i0 = args [0];
5692                 }
5693 #if 0
5694                 /* OP_FREM is not IEEE compatible */
5695                 else if (strcmp (cmethod->name, "IEEERemainder") == 0) {
5696                         MONO_INST_NEW (cfg, ins, OP_FREM);
5697                         ins->inst_i0 = args [0];
5698                         ins->inst_i1 = args [1];
5699                 }
5700 #endif
5701         } else if (cmethod->klass == mono_defaults.thread_class &&
5702                            strcmp (cmethod->name, "MemoryBarrier") == 0) {
5703                 MONO_INST_NEW (cfg, ins, OP_MEMORY_BARRIER);
5704         } else if(cmethod->klass->image == mono_defaults.corlib &&
5705                            (strcmp (cmethod->klass->name_space, "System.Threading") == 0) &&
5706                            (strcmp (cmethod->klass->name, "Interlocked") == 0)) {
5707
5708                 if (strcmp (cmethod->name, "Increment") == 0) {
5709                         MonoInst *ins_iconst;
5710                         guint32 opcode;
5711
5712                         if (fsig->params [0]->type == MONO_TYPE_I4)
5713                                 opcode = OP_ATOMIC_ADD_NEW_I4;
5714                         else if (fsig->params [0]->type == MONO_TYPE_I8)
5715                                 opcode = OP_ATOMIC_ADD_NEW_I8;
5716                         else
5717                                 g_assert_not_reached ();
5718                         MONO_INST_NEW (cfg, ins, opcode);
5719                         MONO_INST_NEW (cfg, ins_iconst, OP_ICONST);
5720                         ins_iconst->inst_c0 = 1;
5721
5722                         ins->inst_i0 = args [0];
5723                         ins->inst_i1 = ins_iconst;
5724                 } else if (strcmp (cmethod->name, "Decrement") == 0) {
5725                         MonoInst *ins_iconst;
5726                         guint32 opcode;
5727
5728                         if (fsig->params [0]->type == MONO_TYPE_I4)
5729                                 opcode = OP_ATOMIC_ADD_NEW_I4;
5730                         else if (fsig->params [0]->type == MONO_TYPE_I8)
5731                                 opcode = OP_ATOMIC_ADD_NEW_I8;
5732                         else
5733                                 g_assert_not_reached ();
5734                         MONO_INST_NEW (cfg, ins, opcode);
5735                         MONO_INST_NEW (cfg, ins_iconst, OP_ICONST);
5736                         ins_iconst->inst_c0 = -1;
5737
5738                         ins->inst_i0 = args [0];
5739                         ins->inst_i1 = ins_iconst;
5740                 } else if (strcmp (cmethod->name, "Add") == 0) {
5741                         guint32 opcode;
5742
5743                         if (fsig->params [0]->type == MONO_TYPE_I4)
5744                                 opcode = OP_ATOMIC_ADD_NEW_I4;
5745                         else if (fsig->params [0]->type == MONO_TYPE_I8)
5746                                 opcode = OP_ATOMIC_ADD_NEW_I8;
5747                         else
5748                                 g_assert_not_reached ();
5749
5750                         MONO_INST_NEW (cfg, ins, opcode);
5751
5752                         ins->inst_i0 = args [0];
5753                         ins->inst_i1 = args [1];
5754                 } else if (strcmp (cmethod->name, "Exchange") == 0) {
5755                         guint32 opcode;
5756
5757                         if (fsig->params [0]->type == MONO_TYPE_I4)
5758                                 opcode = OP_ATOMIC_EXCHANGE_I4;
5759                         else if ((fsig->params [0]->type == MONO_TYPE_I8) ||
5760                                          (fsig->params [0]->type == MONO_TYPE_I) ||
5761                                          (fsig->params [0]->type == MONO_TYPE_OBJECT))
5762                                 opcode = OP_ATOMIC_EXCHANGE_I8;
5763                         else
5764                                 return NULL;
5765
5766                         MONO_INST_NEW (cfg, ins, opcode);
5767
5768                         ins->inst_i0 = args [0];
5769                         ins->inst_i1 = args [1];
5770                 } else if (strcmp (cmethod->name, "Read") == 0 && (fsig->params [0]->type == MONO_TYPE_I8)) {
5771                         /* 64 bit reads are already atomic */
5772                         MONO_INST_NEW (cfg, ins, CEE_LDIND_I8);
5773                         ins->inst_i0 = args [0];
5774                 }
5775
5776                 /*
5777                  * Can't implement CompareExchange methods this way since they have
5778                  * three arguments.
5779                  */
5780         }
5781
5782         return ins;
5783 }
5784
5785 gboolean
5786 mono_arch_print_tree (MonoInst *tree, int arity)
5787 {
5788         return 0;
5789 }
5790
5791 MonoInst* mono_arch_get_domain_intrinsic (MonoCompile* cfg)
5792 {
5793         MonoInst* ins;
5794
5795         if (appdomain_tls_offset == -1)
5796                 return NULL;
5797
5798         MONO_INST_NEW (cfg, ins, OP_TLS_GET);
5799         ins->inst_offset = appdomain_tls_offset;
5800         return ins;
5801 }
5802
5803 MonoInst* mono_arch_get_thread_intrinsic (MonoCompile* cfg)
5804 {
5805         MonoInst* ins;
5806
5807         if (thread_tls_offset == -1)
5808                 return NULL;
5809
5810         MONO_INST_NEW (cfg, ins, OP_TLS_GET);
5811         ins->inst_offset = thread_tls_offset;
5812         return ins;
5813 }