mono/mini/mini-amd64.c

   1 /*
   2  * mini-amd64.c: AMD64 backend for the Mono code generator
   3  *
   4  * Based on mini-x86.c.
   5  *
   6  * Authors:
   7  *   Paolo Molaro (lupus@ximian.com)
   8  *   Dietmar Maurer (dietmar@ximian.com)
   9  *   Patrik Torstensson
  10  *
  11  * (C) 2003 Ximian, Inc.
  12  */
  13 #include "mini.h"
  14 #include <string.h>
  15 #include <math.h>
  16 #include <unistd.h>
  17 #include <sys/mman.h>
  18
  19 #include <mono/metadata/appdomain.h>
  20 #include <mono/metadata/debug-helpers.h>
  21 #include <mono/metadata/threads.h>
  22 #include <mono/metadata/profiler-private.h>
  23 #include <mono/utils/mono-math.h>
  24
  25 #include "trace.h"
  26 #include "mini-amd64.h"
  27 #include "inssel.h"
  28 #include "cpu-amd64.h"
  29
  30 static gint lmf_tls_offset = -1;
  31 static gint appdomain_tls_offset = -1;
  32 static gint thread_tls_offset = -1;
  33
  34 /* Use SSE2 instructions for fp arithmetic */
  35 static gboolean use_sse2 = TRUE;
  36
  37 /* xmm15 is reserved for use by some opcodes */
  38 #define AMD64_CALLEE_FREGS 0xef
  39
  40 #define FPSTACK_SIZE 6
  41
  42 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
  43
  44 #define IS_IMM32(val) ((((guint64)val) >> 32) == 0)
  45
  46 #ifdef PLATFORM_WIN32
  47 /* Under windows, the default pinvoke calling convention is stdcall */
  48 #define CALLCONV_IS_STDCALL(call_conv) (((call_conv) == MONO_CALL_STDCALL) || ((call_conv) == MONO_CALL_DEFAULT))
  49 #else
  50 #define CALLCONV_IS_STDCALL(call_conv) ((call_conv) == MONO_CALL_STDCALL)
  51 #endif
  52
  53 #define SIGNAL_STACK_SIZE (64 * 1024)
  54
  55 #define ARGS_OFFSET 16
  56 #define GP_SCRATCH_REG AMD64_R11
  57
  58 /*
  59  * AMD64 register usage:
  60  * - callee saved registers are used for global register allocation
  61  * - %r11 is used for materializing 64 bit constants in opcodes
  62  * - the rest is used for local allocation
  63  */
  64
  65 /*
  66  * Floating point comparison results:
  67  *                  ZF PF CF
  68  * A > B            0  0  0
  69  * A < B            0  0  1
  70  * A = B            1  0  0
  71  * A > B            0  0  0
  72  * UNORDERED        1  1  1
  73  */
  74
  75 #define NOT_IMPLEMENTED g_assert_not_reached ()
  76
  77 const char*
  78 mono_arch_regname (int reg) {
  79         switch (reg) {
  80         case AMD64_RAX: return "%rax";
  81         case AMD64_RBX: return "%rbx";
  82         case AMD64_RCX: return "%rcx";
  83         case AMD64_RDX: return "%rdx";
  84         case AMD64_RSP: return "%rsp";
  85         case AMD64_RBP: return "%rbp";
  86         case AMD64_RDI: return "%rdi";
  87         case AMD64_RSI: return "%rsi";
  88         case AMD64_R8: return "%r8";
  89         case AMD64_R9: return "%r9";
  90         case AMD64_R10: return "%r10";
  91         case AMD64_R11: return "%r11";
  92         case AMD64_R12: return "%r12";
  93         case AMD64_R13: return "%r13";
  94         case AMD64_R14: return "%r14";
  95         case AMD64_R15: return "%r15";
  96         }
  97         return "unknown";
  98 }
  99
 100 static const char * xmmregs [] = {
 101         "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8",
 102         "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"
 103 };
 104
 105 static const char*
 106 mono_arch_fregname (int reg)
 107 {
 108         if (reg < AMD64_XMM_NREG)
 109                 return xmmregs [reg];
 110         else
 111                 return "unknown";
 112 }
 113
 114 static const char*
 115 mono_amd64_regname (int reg, gboolean fp)
 116 {
 117         if (fp)
 118                 return mono_arch_fregname (reg);
 119         else
 120                 return mono_arch_regname (reg);
 121 }
 122
 123 static inline void
 124 amd64_patch (unsigned char* code, gpointer target)
 125 {
 126         /* Skip REX */
 127         if ((code [0] >= 0x40) && (code [0] <= 0x4f))
 128                 code += 1;
 129
 130         if ((code [0] & 0xf8) == 0xb8) {
 131                 /* amd64_set_reg_template */
 132                 *(guint64*)(code + 1) = (guint64)target;
 133         }
 134         else if (code [0] == 0x8b) {
 135                 /* mov 0(%rip), %dreg */
 136                 *(guint32*)(code + 2) = (guint32)(guint64)target - 7;
 137         }
 138         else if ((code [0] == 0xff) && (code [1] == 0x15)) {
 139                 /* call *<OFFSET>(%rip) */
 140                 *(guint32*)(code + 2) = ((guint32)(guint64)target) - 7;
 141         }
 142         else
 143                 x86_patch (code, (unsigned char*)target);
 144 }
 145
 146 typedef enum {
 147         ArgInIReg,
 148         ArgInFloatSSEReg,
 149         ArgInDoubleSSEReg,
 150         ArgOnStack,
 151         ArgValuetypeInReg,
 152         ArgNone /* only in pair_storage */
 153 } ArgStorage;
 154
 155 typedef struct {
 156         gint16 offset;
 157         gint8  reg;
 158         ArgStorage storage;
 159
 160         /* Only if storage == ArgValuetypeInReg */
 161         ArgStorage pair_storage [2];
 162         gint8 pair_regs [2];
 163 } ArgInfo;
 164
 165 typedef struct {
 166         int nargs;
 167         guint32 stack_usage;
 168         guint32 reg_usage;
 169         guint32 freg_usage;
 170         gboolean need_stack_align;
 171         ArgInfo ret;
 172         ArgInfo sig_cookie;
 173         ArgInfo args [1];
 174 } CallInfo;
 175
 176 #define DEBUG(a) if (cfg->verbose_level > 1) a
 177
 178 #define NEW_ICONST(cfg,dest,val) do {   \
 179                 (dest) = mono_mempool_alloc0 ((cfg)->mempool, sizeof (MonoInst));       \
 180                 (dest)->opcode = OP_ICONST;     \
 181                 (dest)->inst_c0 = (val);        \
 182                 (dest)->type = STACK_I4;        \
 183         } while (0)
 184
 185 #define PARAM_REGS 6
 186
 187 static AMD64_Reg_No param_regs [] = { AMD64_RDI, AMD64_RSI, AMD64_RDX, AMD64_RCX, AMD64_R8, AMD64_R9 };
 188
 189 static AMD64_Reg_No return_regs [] = { AMD64_RAX, AMD64_RDX };
 190
 191 static void inline
 192 add_general (guint32 *gr, guint32 *stack_size, ArgInfo *ainfo)
 193 {
 194     ainfo->offset = *stack_size;
 195
 196     if (*gr >= PARAM_REGS) {
 197                 ainfo->storage = ArgOnStack;
 198                 (*stack_size) += sizeof (gpointer);
 199     }
 200     else {
 201                 ainfo->storage = ArgInIReg;
 202                 ainfo->reg = param_regs [*gr];
 203                 (*gr) ++;
 204     }
 205 }
 206
 207 #define FLOAT_PARAM_REGS 8
 208
 209 static void inline
 210 add_float (guint32 *gr, guint32 *stack_size, ArgInfo *ainfo, gboolean is_double)
 211 {
 212     ainfo->offset = *stack_size;
 213
 214     if (*gr >= FLOAT_PARAM_REGS) {
 215                 ainfo->storage = ArgOnStack;
 216                 (*stack_size) += sizeof (gpointer);
 217     }
 218     else {
 219                 /* A double register */
 220                 if (is_double)
 221                         ainfo->storage = ArgInDoubleSSEReg;
 222                 else
 223                         ainfo->storage = ArgInFloatSSEReg;
 224                 ainfo->reg = *gr;
 225                 (*gr) += 1;
 226     }
 227 }
 228
 229 typedef enum ArgumentClass {
 230         ARG_CLASS_NO_CLASS,
 231         ARG_CLASS_MEMORY,
 232         ARG_CLASS_INTEGER,
 233         ARG_CLASS_SSE
 234 } ArgumentClass;
 235
 236 static ArgumentClass
 237 merge_argument_class_from_type (MonoType *type, ArgumentClass class1)
 238 {
 239         ArgumentClass class2;
 240         MonoType *ptype;
 241
 242         ptype = mono_type_get_underlying_type (type);
 243         switch (ptype->type) {
 244         case MONO_TYPE_BOOLEAN:
 245         case MONO_TYPE_CHAR:
 246         case MONO_TYPE_I1:
 247         case MONO_TYPE_U1:
 248         case MONO_TYPE_I2:
 249         case MONO_TYPE_U2:
 250         case MONO_TYPE_I4:
 251         case MONO_TYPE_U4:
 252         case MONO_TYPE_I:
 253         case MONO_TYPE_U:
 254         case MONO_TYPE_STRING:
 255         case MONO_TYPE_OBJECT:
 256         case MONO_TYPE_CLASS:
 257         case MONO_TYPE_SZARRAY:
 258         case MONO_TYPE_PTR:
 259         case MONO_TYPE_FNPTR:
 260         case MONO_TYPE_ARRAY:
 261         case MONO_TYPE_I8:
 262         case MONO_TYPE_U8:
 263                 class2 = ARG_CLASS_INTEGER;
 264                 break;
 265         case MONO_TYPE_R4:
 266         case MONO_TYPE_R8:
 267                 class2 = ARG_CLASS_SSE;
 268                 break;
 269
 270         case MONO_TYPE_TYPEDBYREF:
 271                 g_assert_not_reached ();
 272
 273         case MONO_TYPE_VALUETYPE: {
 274                 MonoMarshalType *info = mono_marshal_load_type_info (ptype->data.klass);
 275                 int i;
 276
 277                 for (i = 0; i < info->num_fields; ++i) {
 278                         class2 = class1;
 279                         class2 = merge_argument_class_from_type (info->fields [i].field->type, class2);
 280                 }
 281                 break;
 282         }
 283         default:
 284                 g_assert_not_reached ();
 285         }
 286
 287         /* Merge */
 288         if (class1 == class2)
 289                 ;
 290         else if (class1 == ARG_CLASS_NO_CLASS)
 291                 class1 = class2;
 292         else if ((class1 == ARG_CLASS_MEMORY) || (class2 == ARG_CLASS_MEMORY))
 293                 class1 = ARG_CLASS_MEMORY;
 294         else if ((class1 == ARG_CLASS_INTEGER) || (class2 == ARG_CLASS_INTEGER))
 295                 class1 = ARG_CLASS_INTEGER;
 296         else
 297                 class1 = ARG_CLASS_SSE;
 298
 299         return class1;
 300 }
 301
 302 static void
 303 add_valuetype (MonoMethodSignature *sig, ArgInfo *ainfo, MonoType *type,
 304                gboolean is_return,
 305                guint32 *gr, guint32 *fr, guint32 *stack_size)
 306 {
 307         guint32 size, quad, nquads, i;
 308         ArgumentClass args [2];
 309         MonoMarshalType *info;
 310         MonoClass *klass;
 311
 312         klass = mono_class_from_mono_type (type);
 313         if (sig->pinvoke)
 314                 size = mono_type_native_stack_size (&klass->byval_arg, NULL);
 315         else
 316                 size = mono_type_stack_size (&klass->byval_arg, NULL);
 317
 318         if (!sig->pinvoke || (size == 0) || (size > 16)) {
 319                 /* Allways pass in memory */
 320                 ainfo->offset = *stack_size;
 321                 *stack_size += ALIGN_TO (size, 8);
 322                 ainfo->storage = ArgOnStack;
 323
 324                 return;
 325         }
 326
 327         /* FIXME: Handle structs smaller than 8 bytes */
 328         //if ((size % 8) != 0)
 329         //      NOT_IMPLEMENTED;
 330
 331         if (size > 8)
 332                 nquads = 2;
 333         else
 334                 nquads = 1;
 335
 336         /*
 337          * Implement the algorithm from section 3.2.3 of the X86_64 ABI.
 338          * The X87 and SSEUP stuff is left out since there are no such types in
 339          * the CLR.
 340          */
 341         info = mono_marshal_load_type_info (klass);
 342         g_assert (info);
 343         if (info->native_size > 16) {
 344                 ainfo->offset = *stack_size;
 345                 *stack_size += ALIGN_TO (info->native_size, 8);
 346                 ainfo->storage = ArgOnStack;
 347
 348                 return;
 349         }
 350
 351         for (quad = 0; quad < nquads; ++quad) {
 352                 int size, align;
 353                 ArgumentClass class1;
 354
 355                 class1 = ARG_CLASS_NO_CLASS;
 356                 for (i = 0; i < info->num_fields; ++i) {
 357                         size = mono_marshal_type_size (info->fields [i].field->type,
 358                                                                                    info->fields [i].mspec,
 359                                                                                    &align, TRUE, klass->unicode);
 360                         if ((info->fields [i].offset < 8) && (info->fields [i].offset + size) > 8) {
 361                                 /* Unaligned field */
 362                                 NOT_IMPLEMENTED;
 363                         }
 364
 365                         /* Skip fields in other quad */
 366                         if ((quad == 0) && (info->fields [i].offset >= 8))
 367                                 continue;
 368                         if ((quad == 1) && (info->fields [i].offset < 8))
 369                                 continue;
 370
 371                         class1 = merge_argument_class_from_type (info->fields [i].field->type, class1);
 372                 }
 373                 g_assert (class1 != ARG_CLASS_NO_CLASS);
 374                 args [quad] = class1;
 375         }
 376
 377         /* Post merger cleanup */
 378         if ((args [0] == ARG_CLASS_MEMORY) || (args [1] == ARG_CLASS_MEMORY))
 379                 args [0] = args [1] = ARG_CLASS_MEMORY;
 380
 381         /* Allocate registers */
 382         {
 383                 int orig_gr = *gr;
 384                 int orig_fr = *fr;
 385
 386                 ainfo->storage = ArgValuetypeInReg;
 387                 ainfo->pair_storage [0] = ainfo->pair_storage [1] = ArgNone;
 388                 for (quad = 0; quad < nquads; ++quad) {
 389                         switch (args [quad]) {
 390                         case ARG_CLASS_INTEGER:
 391                                 if (*gr >= PARAM_REGS)
 392                                         args [quad] = ARG_CLASS_MEMORY;
 393                                 else {
 394                                         ainfo->pair_storage [quad] = ArgInIReg;
 395                                         if (is_return)
 396                                                 ainfo->pair_regs [quad] = return_regs [*gr];
 397                                         else
 398                                                 ainfo->pair_regs [quad] = param_regs [*gr];
 399                                         (*gr) ++;
 400                                 }
 401                                 break;
 402                         case ARG_CLASS_SSE:
 403                                 if (*fr >= FLOAT_PARAM_REGS)
 404                                         args [quad] = ARG_CLASS_MEMORY;
 405                                 else {
 406                                         ainfo->pair_storage [quad] = ArgInDoubleSSEReg;
 407                                         ainfo->pair_regs [quad] = *fr;
 408                                         (*fr) ++;
 409                                 }
 410                                 break;
 411                         case ARG_CLASS_MEMORY:
 412                                 break;
 413                         default:
 414                                 g_assert_not_reached ();
 415                         }
 416                 }
 417
 418                 if ((args [0] == ARG_CLASS_MEMORY) || (args [1] == ARG_CLASS_MEMORY)) {
 419                         /* Revert possible register assignments */
 420                         *gr = orig_gr;
 421                         *fr = orig_fr;
 422
 423                         ainfo->offset = *stack_size;
 424                         *stack_size += ALIGN_TO (info->native_size, 8);
 425                         ainfo->storage = ArgOnStack;
 426                 }
 427         }
 428 }
 429
 430 /*
 431  * get_call_info:
 432  *
 433  *  Obtain information about a call according to the calling convention.
 434  * For AMD64, see the "System V ABI, x86-64 Architecture Processor Supplement
 435  * Draft Version 0.23" document for more information.
 436  */
 437 static CallInfo*
 438 get_call_info (MonoMethodSignature *sig, gboolean is_pinvoke)
 439 {
 440         guint32 i, gr, fr;
 441         MonoType *ret_type;
 442         int n = sig->hasthis + sig->param_count;
 443         guint32 stack_size = 0;
 444         CallInfo *cinfo;
 445
 446         cinfo = g_malloc0 (sizeof (CallInfo) + (sizeof (ArgInfo) * n));
 447
 448         gr = 0;
 449         fr = 0;
 450
 451         /* return value */
 452         {
 453                 ret_type = mono_type_get_underlying_type (sig->ret);
 454                 switch (ret_type->type) {
 455                 case MONO_TYPE_BOOLEAN:
 456                 case MONO_TYPE_I1:
 457                 case MONO_TYPE_U1:
 458                 case MONO_TYPE_I2:
 459                 case MONO_TYPE_U2:
 460                 case MONO_TYPE_CHAR:
 461                 case MONO_TYPE_I4:
 462                 case MONO_TYPE_U4:
 463                 case MONO_TYPE_I:
 464                 case MONO_TYPE_U:
 465                 case MONO_TYPE_PTR:
 466                 case MONO_TYPE_FNPTR:
 467                 case MONO_TYPE_CLASS:
 468                 case MONO_TYPE_OBJECT:
 469                 case MONO_TYPE_SZARRAY:
 470                 case MONO_TYPE_ARRAY:
 471                 case MONO_TYPE_STRING:
 472                         cinfo->ret.storage = ArgInIReg;
 473                         cinfo->ret.reg = AMD64_RAX;
 474                         break;
 475                 case MONO_TYPE_U8:
 476                 case MONO_TYPE_I8:
 477                         cinfo->ret.storage = ArgInIReg;
 478                         cinfo->ret.reg = AMD64_RAX;
 479                         break;
 480                 case MONO_TYPE_R4:
 481                         cinfo->ret.storage = ArgInFloatSSEReg;
 482                         cinfo->ret.reg = AMD64_XMM0;
 483                         break;
 484                 case MONO_TYPE_R8:
 485                         cinfo->ret.storage = ArgInDoubleSSEReg;
 486                         cinfo->ret.reg = AMD64_XMM0;
 487                         break;
 488                 case MONO_TYPE_VALUETYPE: {
 489                         guint32 tmp_gr = 0, tmp_fr = 0, tmp_stacksize = 0;
 490
 491                         add_valuetype (sig, &cinfo->ret, sig->ret, TRUE, &tmp_gr, &tmp_fr, &tmp_stacksize);
 492                         if (cinfo->ret.storage == ArgOnStack)
 493                                 /* The caller passes the address where the value is stored */
 494                                 add_general (&gr, &stack_size, &cinfo->ret);
 495                         break;
 496                 }
 497                 case MONO_TYPE_TYPEDBYREF:
 498                         /* Same as a valuetype with size 24 */
 499                         add_general (&gr, &stack_size, &cinfo->ret);
 500                         ;
 501                         break;
 502                 case MONO_TYPE_VOID:
 503                         break;
 504                 default:
 505                         g_error ("Can't handle as return value 0x%x", sig->ret->type);
 506                 }
 507         }
 508
 509         /* this */
 510         if (sig->hasthis)
 511                 add_general (&gr, &stack_size, cinfo->args + 0);
 512
 513         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n == 0)) {
 514                 gr = PARAM_REGS;
 515                 fr = FLOAT_PARAM_REGS;
 516
 517                 /* Emit the signature cookie just before the implicit arguments */
 518                 add_general (&gr, &stack_size, &cinfo->sig_cookie);
 519         }
 520
 521         for (i = 0; i < sig->param_count; ++i) {
 522                 ArgInfo *ainfo = &cinfo->args [sig->hasthis + i];
 523                 MonoType *ptype;
 524
 525                 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
 526                         /* We allways pass the sig cookie on the stack for simplicity */
 527                         /*
 528                          * Prevent implicit arguments + the sig cookie from being passed
 529                          * in registers.
 530                          */
 531                         gr = PARAM_REGS;
 532                         fr = FLOAT_PARAM_REGS;
 533
 534                         /* Emit the signature cookie just before the implicit arguments */
 535                         add_general (&gr, &stack_size, &cinfo->sig_cookie);
 536                 }
 537
 538                 if (sig->params [i]->byref) {
 539                         add_general (&gr, &stack_size, ainfo);
 540                         continue;
 541                 }
 542                 ptype = mono_type_get_underlying_type (sig->params [i]);
 543                 switch (ptype->type) {
 544                 case MONO_TYPE_BOOLEAN:
 545                 case MONO_TYPE_I1:
 546                 case MONO_TYPE_U1:
 547                         add_general (&gr, &stack_size, ainfo);
 548                         break;
 549                 case MONO_TYPE_I2:
 550                 case MONO_TYPE_U2:
 551                 case MONO_TYPE_CHAR:
 552                         add_general (&gr, &stack_size, ainfo);
 553                         break;
 554                 case MONO_TYPE_I4:
 555                 case MONO_TYPE_U4:
 556                         add_general (&gr, &stack_size, ainfo);
 557                         break;
 558                 case MONO_TYPE_I:
 559                 case MONO_TYPE_U:
 560                 case MONO_TYPE_PTR:
 561                 case MONO_TYPE_FNPTR:
 562                 case MONO_TYPE_CLASS:
 563                 case MONO_TYPE_OBJECT:
 564                 case MONO_TYPE_STRING:
 565                 case MONO_TYPE_SZARRAY:
 566                 case MONO_TYPE_ARRAY:
 567                         add_general (&gr, &stack_size, ainfo);
 568                         break;
 569                 case MONO_TYPE_VALUETYPE:
 570                         add_valuetype (sig, ainfo, sig->params [i], FALSE, &gr, &fr, &stack_size);
 571                         break;
 572                 case MONO_TYPE_TYPEDBYREF:
 573                         stack_size += sizeof (MonoTypedRef);
 574                         ainfo->storage = ArgOnStack;
 575                         break;
 576                 case MONO_TYPE_U8:
 577                 case MONO_TYPE_I8:
 578                         add_general (&gr, &stack_size, ainfo);
 579                         break;
 580                 case MONO_TYPE_R4:
 581                         add_float (&fr, &stack_size, ainfo, FALSE);
 582                         break;
 583                 case MONO_TYPE_R8:
 584                         add_float (&fr, &stack_size, ainfo, TRUE);
 585                         break;
 586                 default:
 587                         g_assert_not_reached ();
 588                 }
 589         }
 590
 591         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (n > 0) && (sig->sentinelpos == sig->param_count)) {
 592                 gr = PARAM_REGS;
 593                 fr = FLOAT_PARAM_REGS;
 594
 595                 /* Emit the signature cookie just before the implicit arguments */
 596                 add_general (&gr, &stack_size, &cinfo->sig_cookie);
 597         }
 598
 599         if (stack_size & 0x8) {
 600                 /* The AMD64 ABI requires each stack frame to be 16 byte aligned */
 601                 cinfo->need_stack_align = TRUE;
 602                 stack_size += 8;
 603         }
 604
 605         cinfo->stack_usage = stack_size;
 606         cinfo->reg_usage = gr;
 607         cinfo->freg_usage = fr;
 608         return cinfo;
 609 }
 610
 611 /*
 612  * mono_arch_get_argument_info:
 613  * @csig:  a method signature
 614  * @param_count: the number of parameters to consider
 615  * @arg_info: an array to store the result infos
 616  *
 617  * Gathers information on parameters such as size, alignment and
 618  * padding. arg_info should be large enought to hold param_count + 1 entries.
 619  *
 620  * Returns the size of the argument area on the stack.
 621  */
 622 int
 623 mono_arch_get_argument_info (MonoMethodSignature *csig, int param_count, MonoJitArgumentInfo *arg_info)
 624 {
 625         int k;
 626         CallInfo *cinfo = get_call_info (csig, FALSE);
 627         guint32 args_size = cinfo->stack_usage;
 628
 629         /* The arguments are saved to a stack area in mono_arch_instrument_prolog */
 630         if (csig->hasthis) {
 631                 arg_info [0].offset = 0;
 632         }
 633
 634         for (k = 0; k < param_count; k++) {
 635                 arg_info [k + 1].offset = ((k + csig->hasthis) * 8);
 636                 /* FIXME: */
 637                 arg_info [k + 1].size = 0;
 638         }
 639
 640         g_free (cinfo);
 641
 642         return args_size;
 643 }
 644
 645 static int
 646 cpuid (int id, int* p_eax, int* p_ebx, int* p_ecx, int* p_edx)
 647 {
 648         return 0;
 649 }
 650
 651 /*
 652  * Initialize the cpu to execute managed code.
 653  */
 654 void
 655 mono_arch_cpu_init (void)
 656 {
 657         guint16 fpcw;
 658
 659         /* spec compliance requires running with double precision */
 660         __asm__  __volatile__ ("fnstcw %0\n": "=m" (fpcw));
 661         fpcw &= ~X86_FPCW_PRECC_MASK;
 662         fpcw |= X86_FPCW_PREC_DOUBLE;
 663         __asm__  __volatile__ ("fldcw %0\n": : "m" (fpcw));
 664         __asm__  __volatile__ ("fnstcw %0\n": "=m" (fpcw));
 665 }
 666
 667 /*
 668  * This function returns the optimizations supported on this cpu.
 669  */
 670 guint32
 671 mono_arch_cpu_optimizazions (guint32 *exclude_mask)
 672 {
 673         int eax, ebx, ecx, edx;
 674         guint32 opts = 0;
 675
 676         /* FIXME: AMD64 */
 677
 678         *exclude_mask = 0;
 679         /* Feature Flags function, flags returned in EDX. */
 680         if (cpuid (1, &eax, &ebx, &ecx, &edx)) {
 681                 if (edx & (1 << 15)) {
 682                         opts |= MONO_OPT_CMOV;
 683                         if (edx & 1)
 684                                 opts |= MONO_OPT_FCMOV;
 685                         else
 686                                 *exclude_mask |= MONO_OPT_FCMOV;
 687                 } else
 688                         *exclude_mask |= MONO_OPT_CMOV;
 689         }
 690         return opts;
 691 }
 692
 693 gboolean
 694 mono_amd64_is_sse2 (void)
 695 {
 696         return use_sse2;
 697 }
 698
 699 static gboolean
 700 is_regsize_var (MonoType *t) {
 701         if (t->byref)
 702                 return TRUE;
 703         t = mono_type_get_underlying_type (t);
 704         switch (t->type) {
 705         case MONO_TYPE_I4:
 706         case MONO_TYPE_U4:
 707         case MONO_TYPE_I:
 708         case MONO_TYPE_U:
 709         case MONO_TYPE_PTR:
 710         case MONO_TYPE_FNPTR:
 711                 return TRUE;
 712         case MONO_TYPE_OBJECT:
 713         case MONO_TYPE_STRING:
 714         case MONO_TYPE_CLASS:
 715         case MONO_TYPE_SZARRAY:
 716         case MONO_TYPE_ARRAY:
 717                 return TRUE;
 718         case MONO_TYPE_VALUETYPE:
 719                 return FALSE;
 720         }
 721         return FALSE;
 722 }
 723
 724 GList *
 725 mono_arch_get_allocatable_int_vars (MonoCompile *cfg)
 726 {
 727         GList *vars = NULL;
 728         int i;
 729
 730         for (i = 0; i < cfg->num_varinfo; i++) {
 731                 MonoInst *ins = cfg->varinfo [i];
 732                 MonoMethodVar *vmv = MONO_VARINFO (cfg, i);
 733
 734                 /* unused vars */
 735                 if (vmv->range.first_use.abs_pos >= vmv->range.last_use.abs_pos)
 736                         continue;
 737
 738                 if ((ins->flags & (MONO_INST_IS_DEAD|MONO_INST_VOLATILE|MONO_INST_INDIRECT)) ||
 739                     (ins->opcode != OP_LOCAL && ins->opcode != OP_ARG))
 740                         continue;
 741
 742                 /* we dont allocate I1 to registers because there is no simply way to sign extend
 743                  * 8bit quantities in caller saved registers on x86 */
 744                 if (is_regsize_var (ins->inst_vtype) || (ins->inst_vtype->type == MONO_TYPE_BOOLEAN) ||
 745                     (ins->inst_vtype->type == MONO_TYPE_U1) || (ins->inst_vtype->type == MONO_TYPE_U2)||
 746                     (ins->inst_vtype->type == MONO_TYPE_I2) || (ins->inst_vtype->type == MONO_TYPE_CHAR)) {
 747                         g_assert (MONO_VARINFO (cfg, i)->reg == -1);
 748                         g_assert (i == vmv->idx);
 749                         vars = g_list_prepend (vars, vmv);
 750                 }
 751         }
 752
 753         vars = mono_varlist_sort (cfg, vars, 0);
 754
 755         return vars;
 756 }
 757
 758 GList *
 759 mono_arch_get_global_int_regs (MonoCompile *cfg)
 760 {
 761         GList *regs = NULL;
 762
 763         /* We use the callee saved registers for global allocation */
 764         regs = g_list_prepend (regs, (gpointer)AMD64_RBX);
 765         regs = g_list_prepend (regs, (gpointer)AMD64_R12);
 766         regs = g_list_prepend (regs, (gpointer)AMD64_R13);
 767         regs = g_list_prepend (regs, (gpointer)AMD64_R14);
 768         regs = g_list_prepend (regs, (gpointer)AMD64_R15);
 769
 770         return regs;
 771 }
 772
 773 /*
 774  * mono_arch_regalloc_cost:
 775  *
 776  *  Return the cost, in number of memory references, of the action of
 777  * allocating the variable VMV into a register during global register
 778  * allocation.
 779  */
 780 guint32
 781 mono_arch_regalloc_cost (MonoCompile *cfg, MonoMethodVar *vmv)
 782 {
 783         MonoInst *ins = cfg->varinfo [vmv->idx];
 784
 785         if (cfg->method->save_lmf)
 786                 /* The register is already saved */
 787                 /* substract 1 for the invisible store in the prolog */
 788                 return (ins->opcode == OP_ARG) ? 0 : 1;
 789         else
 790                 /* push+pop */
 791                 return (ins->opcode == OP_ARG) ? 1 : 2;
 792 }
 793
 794 void
 795 mono_arch_allocate_vars (MonoCompile *m)
 796 {
 797         MonoMethodSignature *sig;
 798         MonoMethodHeader *header;
 799         MonoInst *inst;
 800         int i, offset;
 801         guint32 locals_stack_size, locals_stack_align;
 802         gint32 *offsets;
 803         CallInfo *cinfo;
 804
 805         header = mono_method_get_header (m->method);
 806
 807         sig = mono_method_signature (m->method);
 808
 809         cinfo = get_call_info (sig, FALSE);
 810
 811         /*
 812          * We use the ABI calling conventions for managed code as well.
 813          * Exception: valuetypes are never passed or returned in registers.
 814          */
 815
 816         /* Locals are allocated backwards from %fp */
 817         m->frame_reg = AMD64_RBP;
 818         offset = 0;
 819
 820         /* Reserve space for caller saved registers */
 821         for (i = 0; i < AMD64_NREG; ++i)
 822                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (m->used_int_regs & (1 << i))) {
 823                         offset += sizeof (gpointer);
 824                 }
 825
 826         if (m->method->save_lmf) {
 827                 /* Reserve stack space for saving LMF + argument regs */
 828                 offset += sizeof (MonoLMF);
 829                 if (lmf_tls_offset == -1)
 830                         /* Need to save argument regs too */
 831                         offset += (AMD64_NREG * 8) + (8 * 8);
 832                 m->arch.lmf_offset = offset;
 833         }
 834
 835         if (sig->ret->type != MONO_TYPE_VOID) {
 836                 switch (cinfo->ret.storage) {
 837                 case ArgInIReg:
 838                 case ArgInFloatSSEReg:
 839                 case ArgInDoubleSSEReg:
 840                         if ((MONO_TYPE_ISSTRUCT (sig->ret) && !mono_class_from_mono_type (sig->ret)->enumtype) || (sig->ret->type == MONO_TYPE_TYPEDBYREF)) {
 841                                 /* The register is volatile */
 842                                 m->ret->opcode = OP_REGOFFSET;
 843                                 m->ret->inst_basereg = AMD64_RBP;
 844                                 offset += 8;
 845                                 m->ret->inst_offset = - offset;
 846                         }
 847                         else {
 848                                 m->ret->opcode = OP_REGVAR;
 849                                 m->ret->inst_c0 = cinfo->ret.reg;
 850                         }
 851                         break;
 852                 case ArgValuetypeInReg:
 853                         /* Allocate a local to hold the result, the epilog will copy it to the correct place */
 854                         offset += 16;
 855                         m->ret->opcode = OP_REGOFFSET;
 856                         m->ret->inst_basereg = AMD64_RBP;
 857                         m->ret->inst_offset = - offset;
 858                         break;
 859                 default:
 860                         g_assert_not_reached ();
 861                 }
 862                 m->ret->dreg = m->ret->inst_c0;
 863         }
 864
 865         /* Allocate locals */
 866         offsets = mono_allocate_stack_slots (m, &locals_stack_size, &locals_stack_align);
 867         if (locals_stack_align) {
 868                 offset += (locals_stack_align - 1);
 869                 offset &= ~(locals_stack_align - 1);
 870         }
 871         for (i = m->locals_start; i < m->num_varinfo; i++) {
 872                 if (offsets [i] != -1) {
 873                         MonoInst *inst = m->varinfo [i];
 874                         inst->opcode = OP_REGOFFSET;
 875                         inst->inst_basereg = AMD64_RBP;
 876                         inst->inst_offset = - (offset + offsets [i]);
 877                         //printf ("allocated local %d to ", i); mono_print_tree_nl (inst);
 878                 }
 879         }
 880         g_free (offsets);
 881         offset += locals_stack_size;
 882
 883         if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG)) {
 884                 g_assert (cinfo->sig_cookie.storage == ArgOnStack);
 885                 m->sig_cookie = cinfo->sig_cookie.offset + ARGS_OFFSET;
 886         }
 887
 888         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
 889                 inst = m->varinfo [i];
 890                 if (inst->opcode != OP_REGVAR) {
 891                         ArgInfo *ainfo = &cinfo->args [i];
 892                         gboolean inreg = TRUE;
 893                         MonoType *arg_type;
 894
 895                         if (sig->hasthis && (i == 0))
 896                                 arg_type = &mono_defaults.object_class->byval_arg;
 897                         else
 898                                 arg_type = sig->params [i - sig->hasthis];
 899
 900                         /* FIXME: Allocate volatile arguments to registers */
 901                         if (inst->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT))
 902                                 inreg = FALSE;
 903
 904                         /*
 905                          * Under AMD64, all registers used to pass arguments to functions
 906                          * are volatile across calls.
 907                          * FIXME: Optimize this.
 908                          */
 909                         if ((ainfo->storage == ArgInIReg) || (ainfo->storage == ArgInFloatSSEReg) || (ainfo->storage == ArgInDoubleSSEReg) || (ainfo->storage == ArgValuetypeInReg))
 910                                 inreg = FALSE;
 911
 912                         inst->opcode = OP_REGOFFSET;
 913
 914                         switch (ainfo->storage) {
 915                         case ArgInIReg:
 916                         case ArgInFloatSSEReg:
 917                         case ArgInDoubleSSEReg:
 918                                 inst->opcode = OP_REGVAR;
 919                                 inst->dreg = ainfo->reg;
 920                                 break;
 921                         case ArgOnStack:
 922                                 inst->opcode = OP_REGOFFSET;
 923                                 inst->inst_basereg = AMD64_RBP;
 924                                 inst->inst_offset = ainfo->offset + ARGS_OFFSET;
 925                                 break;
 926                         case ArgValuetypeInReg:
 927                                 break;
 928                         default:
 929                                 NOT_IMPLEMENTED;
 930                         }
 931
 932                         if (!inreg && (ainfo->storage != ArgOnStack)) {
 933                                 inst->opcode = OP_REGOFFSET;
 934                                 inst->inst_basereg = AMD64_RBP;
 935                                 /* These arguments are saved to the stack in the prolog */
 936                                 if (ainfo->storage == ArgValuetypeInReg)
 937                                         offset += 2 * sizeof (gpointer);
 938                                 else
 939                                         offset += sizeof (gpointer);
 940                                 inst->inst_offset = - offset;
 941                         }
 942                 }
 943         }
 944
 945         m->stack_offset = offset;
 946
 947         g_free (cinfo);
 948 }
 949
 950 void
 951 mono_arch_create_vars (MonoCompile *cfg)
 952 {
 953         MonoMethodSignature *sig;
 954         CallInfo *cinfo;
 955
 956         sig = mono_method_signature (cfg->method);
 957
 958         cinfo = get_call_info (sig, FALSE);
 959
 960         if (cinfo->ret.storage == ArgValuetypeInReg)
 961                 cfg->ret_var_is_local = TRUE;
 962
 963         g_free (cinfo);
 964 }
 965
 966 static void
 967 add_outarg_reg (MonoCompile *cfg, MonoCallInst *call, MonoInst *arg, ArgStorage storage, int reg, MonoInst *tree)
 968 {
 969         switch (storage) {
 970         case ArgInIReg:
 971                 arg->opcode = OP_OUTARG_REG;
 972                 arg->inst_left = tree;
 973                 arg->inst_right = (MonoInst*)call;
 974                 arg->unused = reg;
 975                 call->used_iregs |= 1 << reg;
 976                 break;
 977         case ArgInFloatSSEReg:
 978                 arg->opcode = OP_AMD64_OUTARG_XMMREG_R4;
 979                 arg->inst_left = tree;
 980                 arg->inst_right = (MonoInst*)call;
 981                 arg->unused = reg;
 982                 call->used_fregs |= 1 << reg;
 983                 break;
 984         case ArgInDoubleSSEReg:
 985                 arg->opcode = OP_AMD64_OUTARG_XMMREG_R8;
 986                 arg->inst_left = tree;
 987                 arg->inst_right = (MonoInst*)call;
 988                 arg->unused = reg;
 989                 call->used_fregs |= 1 << reg;
 990                 break;
 991         default:
 992                 g_assert_not_reached ();
 993         }
 994 }
 995
 996 /* Fixme: we need an alignment solution for enter_method and mono_arch_call_opcode,
 997  * currently alignment in mono_arch_call_opcode is computed without arch_get_argument_info
 998  */
 999
1000 static int
1001 arg_storage_to_ldind (ArgStorage storage)
1002 {
1003         switch (storage) {
1004         case ArgInIReg:
1005                 return CEE_LDIND_I;
1006         case ArgInDoubleSSEReg:
1007                 return CEE_LDIND_R8;
1008         case ArgInFloatSSEReg:
1009                 return CEE_LDIND_R4;
1010         default:
1011                 g_assert_not_reached ();
1012         }
1013
1014         return -1;
1015 }
1016
1017 /*
1018  * take the arguments and generate the arch-specific
1019  * instructions to properly call the function in call.
1020  * This includes pushing, moving arguments to the right register
1021  * etc.
1022  * Issue: who does the spilling if needed, and when?
1023  */
1024 MonoCallInst*
1025 mono_arch_call_opcode (MonoCompile *cfg, MonoBasicBlock* bb, MonoCallInst *call, int is_virtual) {
1026         MonoInst *arg, *in;
1027         MonoMethodSignature *sig;
1028         int i, n, stack_size;
1029         CallInfo *cinfo;
1030         ArgInfo *ainfo;
1031
1032         stack_size = 0;
1033
1034         sig = call->signature;
1035         n = sig->param_count + sig->hasthis;
1036
1037         cinfo = get_call_info (sig, sig->pinvoke);
1038
1039         for (i = 0; i < n; ++i) {
1040                 ainfo = cinfo->args + i;
1041
1042                 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
1043                         MonoMethodSignature *tmp_sig;
1044
1045                         /* Emit the signature cookie just before the implicit arguments */
1046                         MonoInst *sig_arg;
1047                         /* FIXME: Add support for signature tokens to AOT */
1048                         cfg->disable_aot = TRUE;
1049
1050                         g_assert (cinfo->sig_cookie.storage == ArgOnStack);
1051
1052                         /*
1053                          * mono_ArgIterator_Setup assumes the signature cookie is
1054                          * passed first and all the arguments which were before it are
1055                          * passed on the stack after the signature. So compensate by
1056                          * passing a different signature.
1057                          */
1058                         tmp_sig = mono_metadata_signature_dup (call->signature);
1059                         tmp_sig->param_count -= call->signature->sentinelpos;
1060                         tmp_sig->sentinelpos = 0;
1061                         memcpy (tmp_sig->params, call->signature->params + call->signature->sentinelpos, tmp_sig->param_count * sizeof (MonoType*));
1062
1063                         MONO_INST_NEW (cfg, sig_arg, OP_ICONST);
1064                         sig_arg->inst_p0 = tmp_sig;
1065
1066                         MONO_INST_NEW (cfg, arg, OP_OUTARG);
1067                         arg->inst_left = sig_arg;
1068                         arg->type = STACK_PTR;
1069
1070                         /* prepend, so they get reversed */
1071                         arg->next = call->out_args;
1072                         call->out_args = arg;
1073                 }
1074
1075                 if (is_virtual && i == 0) {
1076                         /* the argument will be attached to the call instruction */
1077                         in = call->args [i];
1078                 } else {
1079                         MONO_INST_NEW (cfg, arg, OP_OUTARG);
1080                         in = call->args [i];
1081                         arg->cil_code = in->cil_code;
1082                         arg->inst_left = in;
1083                         arg->type = in->type;
1084                         /* prepend, so they get reversed */
1085                         arg->next = call->out_args;
1086                         call->out_args = arg;
1087
1088                         if ((i >= sig->hasthis) && (MONO_TYPE_ISSTRUCT(sig->params [i - sig->hasthis]))) {
1089                                 gint align;
1090                                 guint32 size;
1091
1092                                 if (sig->params [i - sig->hasthis]->type == MONO_TYPE_TYPEDBYREF) {
1093                                         size = sizeof (MonoTypedRef);
1094                                         align = sizeof (gpointer);
1095                                 }
1096                                 else
1097                                 if (sig->pinvoke)
1098                                         size = mono_type_native_stack_size (&in->klass->byval_arg, &align);
1099                                 else
1100                                         size = mono_type_stack_size (&in->klass->byval_arg, &align);
1101                                 if (ainfo->storage == ArgValuetypeInReg) {
1102                                         if (ainfo->pair_storage [1] == ArgNone) {
1103                                                 MonoInst *load;
1104
1105                                                 /* Simpler case */
1106
1107                                                 MONO_INST_NEW (cfg, load, arg_storage_to_ldind (ainfo->pair_storage [0]));
1108                                                 load->inst_left = in;
1109
1110                                                 add_outarg_reg (cfg, call, arg, ainfo->pair_storage [0], ainfo->pair_regs [0], load);
1111                                         }
1112                                         else {
1113                                                 /* Trees can't be shared so make a copy */
1114                                                 MonoInst *vtaddr = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1115                                                 MonoInst *load, *load2, *offset_ins;
1116
1117                                                 /* Reg1 */
1118                                                 MONO_INST_NEW (cfg, load, CEE_LDIND_I);
1119                                                 load->ssa_op = MONO_SSA_LOAD;
1120                                                 load->inst_i0 = (cfg)->varinfo [vtaddr->inst_c0];
1121
1122                                                 NEW_ICONST (cfg, offset_ins, 0);
1123                                                 MONO_INST_NEW (cfg, load2, CEE_ADD);
1124                                                 load2->inst_left = load;
1125                                                 load2->inst_right = offset_ins;
1126
1127                                                 MONO_INST_NEW (cfg, load, arg_storage_to_ldind (ainfo->pair_storage [0]));
1128                                                 load->inst_left = load2;
1129
1130                                                 add_outarg_reg (cfg, call, arg, ainfo->pair_storage [0], ainfo->pair_regs [0], load);
1131
1132                                                 /* Reg2 */
1133                                                 MONO_INST_NEW (cfg, load, CEE_LDIND_I);
1134                                                 load->ssa_op = MONO_SSA_LOAD;
1135                                                 load->inst_i0 = (cfg)->varinfo [vtaddr->inst_c0];
1136
1137                                                 NEW_ICONST (cfg, offset_ins, 8);
1138                                                 MONO_INST_NEW (cfg, load2, CEE_ADD);
1139                                                 load2->inst_left = load;
1140                                                 load2->inst_right = offset_ins;
1141
1142                                                 MONO_INST_NEW (cfg, load, arg_storage_to_ldind (ainfo->pair_storage [1]));
1143                                                 load->inst_left = load2;
1144
1145                                                 MONO_INST_NEW (cfg, arg, OP_OUTARG);
1146                                                 arg->cil_code = in->cil_code;
1147                                                 arg->type = in->type;
1148                                                 /* prepend, so they get reversed */
1149                                                 arg->next = call->out_args;
1150                                                 call->out_args = arg;
1151
1152                                                 add_outarg_reg (cfg, call, arg, ainfo->pair_storage [1], ainfo->pair_regs [1], load);
1153
1154                                                 /* Prepend a copy inst */
1155                                                 MONO_INST_NEW (cfg, arg, CEE_STIND_I);
1156                                                 arg->cil_code = in->cil_code;
1157                                                 arg->ssa_op = MONO_SSA_STORE;
1158                                                 arg->inst_left = vtaddr;
1159                                                 arg->inst_right = in;
1160                                                 arg->type = in->type;
1161
1162                                                 /* prepend, so they get reversed */
1163                                                 arg->next = call->out_args;
1164                                                 call->out_args = arg;
1165                                         }
1166                                 }
1167                                 else {
1168                                         arg->opcode = OP_OUTARG_VT;
1169                                         arg->klass = in->klass;
1170                                         arg->unused = sig->pinvoke;
1171                                         arg->inst_imm = size;
1172                                 }
1173                         }
1174                         else {
1175                                 switch (ainfo->storage) {
1176                                 case ArgInIReg:
1177                                         add_outarg_reg (cfg, call, arg, ainfo->storage, ainfo->reg, in);
1178                                         break;
1179                                 case ArgInFloatSSEReg:
1180                                 case ArgInDoubleSSEReg:
1181                                         add_outarg_reg (cfg, call, arg, ainfo->storage, ainfo->reg, in);
1182                                         break;
1183                                 case ArgOnStack:
1184                                         arg->opcode = OP_OUTARG;
1185                                         if (!sig->params [i - sig->hasthis]->byref) {
1186                                                 if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R4)
1187                                                         arg->opcode = OP_OUTARG_R4;
1188                                                 else
1189                                                         if (sig->params [i - sig->hasthis]->type == MONO_TYPE_R8)
1190                                                                 arg->opcode = OP_OUTARG_R8;
1191                                         }
1192                                         break;
1193                                 default:
1194                                         g_assert_not_reached ();
1195                                 }
1196                         }
1197                 }
1198         }
1199
1200         if (cinfo->need_stack_align) {
1201                 MONO_INST_NEW (cfg, arg, OP_AMD64_OUTARG_ALIGN_STACK);
1202                 /* prepend, so they get reversed */
1203                 arg->next = call->out_args;
1204                 call->out_args = arg;
1205         }
1206
1207         call->stack_usage = cinfo->stack_usage;
1208         cfg->param_area = MAX (cfg->param_area, call->stack_usage);
1209         cfg->flags |= MONO_CFG_HAS_CALLS;
1210
1211         g_free (cinfo);
1212
1213         return call;
1214 }
1215
1216 #define EMIT_COND_BRANCH(ins,cond,sign) \
1217 if (ins->flags & MONO_INST_BRLABEL) { \
1218         if (ins->inst_i0->inst_c0) { \
1219                 x86_branch (code, cond, cfg->native_code + ins->inst_i0->inst_c0, sign); \
1220         } else { \
1221                 mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_LABEL, ins->inst_i0); \
1222                 if ((cfg->opt & MONO_OPT_BRANCH) && \
1223                     x86_is_imm8 (ins->inst_i0->inst_c1 - cpos)) \
1224                         x86_branch8 (code, cond, 0, sign); \
1225                 else \
1226                         x86_branch32 (code, cond, 0, sign); \
1227         } \
1228 } else { \
1229         if (ins->inst_true_bb->native_offset) { \
1230                 x86_branch (code, cond, cfg->native_code + ins->inst_true_bb->native_offset, sign); \
1231         } else { \
1232                 mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_BB, ins->inst_true_bb); \
1233                 if ((cfg->opt & MONO_OPT_BRANCH) && \
1234                     x86_is_imm8 (ins->inst_true_bb->max_offset - cpos)) \
1235                         x86_branch8 (code, cond, 0, sign); \
1236                 else \
1237                         x86_branch32 (code, cond, 0, sign); \
1238         } \
1239 }
1240
1241 /* emit an exception if condition is fail */
1242 #define EMIT_COND_SYSTEM_EXCEPTION(cond,signed,exc_name)            \
1243         do {                                                        \
1244                 mono_add_patch_info (cfg, code - cfg->native_code,   \
1245                                     MONO_PATCH_INFO_EXC, exc_name);  \
1246                 x86_branch32 (code, cond, 0, signed);               \
1247         } while (0);
1248
1249 #define EMIT_FPCOMPARE(code) do { \
1250         amd64_fcompp (code); \
1251         amd64_fnstsw (code); \
1252 } while (0);
1253
1254 #define EMIT_SSE2_FPFUNC(code, op, dreg, sreg1) do { \
1255     amd64_movsd_membase_reg (code, AMD64_RSP, -8, (sreg1)); \
1256         amd64_fld_membase (code, AMD64_RSP, -8, TRUE); \
1257         amd64_ ##op (code); \
1258         amd64_fst_membase (code, AMD64_RSP, -8, TRUE, TRUE); \
1259         amd64_movsd_reg_membase (code, (dreg), AMD64_RSP, -8); \
1260 } while (0);
1261
1262 static guint8*
1263 emit_call (MonoCompile *cfg, guint8 *code, guint32 patch_type, gconstpointer data)
1264 {
1265         mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
1266
1267         if (mono_compile_aot) {
1268                 amd64_call_membase (code, AMD64_RIP, 0);
1269         }
1270         else {
1271                 gboolean near_call = FALSE;
1272
1273                 /*
1274                  * Indirect calls are expensive so try to make a near call if possible.
1275                  * The caller memory is allocated by the code manager so it is
1276                  * guaranteed to be at a 32 bit offset.
1277                  */
1278
1279                 if (patch_type != MONO_PATCH_INFO_ABS) {
1280                         /* The target is in memory allocated using the code manager */
1281                         near_call = TRUE;
1282
1283                         if ((patch_type == MONO_PATCH_INFO_METHOD) || (patch_type == MONO_PATCH_INFO_METHOD_JUMP)) {
1284                                 if (((MonoMethod*)data)->klass->image->assembly->aot_module)
1285                                         /* The callee might be an AOT method */
1286                                         near_call = FALSE;
1287                         }
1288
1289                         if (patch_type == MONO_PATCH_INFO_INTERNAL_METHOD) {
1290                                 /*
1291                                  * The call might go directly to a native function without
1292                                  * the wrapper.
1293                                  */
1294                                 MonoJitICallInfo *mi = mono_find_jit_icall_by_name (data);
1295                                 if (mi) {
1296                                         gconstpointer target = mono_icall_get_wrapper (mi);
1297                                         if ((((guint64)target) >> 32) != 0)
1298                                                 near_call = FALSE;
1299                                 }
1300                         }
1301                 }
1302                 else {
1303                         if (mono_find_class_init_trampoline_by_addr (data))
1304                                 near_call = TRUE;
1305                         else {
1306                                 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (data);
1307                                 if (info) {
1308                                         if ((cfg->method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) &&
1309                                                 strstr (cfg->method->name, info->name)) {
1310                                                 /* A call to the wrapped function */
1311                                                 if ((((guint64)data) >> 32) == 0)
1312                                                         near_call = TRUE;
1313                                         }
1314                                         else if (info->func == info->wrapper) {
1315                                                 /* No wrapper */
1316                                                 if ((((guint64)info->func) >> 32) == 0)
1317                                                         near_call = TRUE;
1318                                         }
1319                                         else
1320                                                 near_call = TRUE;
1321                                 }
1322                                 else if ((((guint64)data) >> 32) == 0)
1323                                         near_call = TRUE;
1324                         }
1325                 }
1326
1327                 if (near_call) {
1328                         amd64_call_code (code, 0);
1329                 }
1330                 else {
1331                         amd64_set_reg_template (code, GP_SCRATCH_REG);
1332                         amd64_call_reg (code, GP_SCRATCH_REG);
1333                 }
1334         }
1335
1336         return code;
1337 }
1338
1339 /* FIXME: Add more instructions */
1340 #define INST_IGNORES_CFLAGS(ins) (((ins)->opcode == CEE_BR) || ((ins)->opcode == OP_STORE_MEMBASE_IMM) || ((ins)->opcode == OP_STOREI8_MEMBASE_REG) || ((ins)->opcode == OP_MOVE) || ((ins)->opcode == OP_SETREG) || ((ins)->opcode == OP_ICONST) || ((ins)->opcode == OP_I8CONST) || ((ins)->opcode == OP_LOAD_MEMBASE))
1341
1342 static void
1343 peephole_pass (MonoCompile *cfg, MonoBasicBlock *bb)
1344 {
1345         MonoInst *ins, *last_ins = NULL;
1346         ins = bb->code;
1347
1348         while (ins) {
1349
1350                 switch (ins->opcode) {
1351                 case OP_ICONST:
1352                 case OP_I8CONST:
1353                         /* reg = 0 -> XOR (reg, reg) */
1354                         /* XOR sets cflags on x86, so we cant do it always */
1355                         if (ins->inst_c0 == 0 && (ins->next && INST_IGNORES_CFLAGS (ins->next))) {
1356                                 ins->opcode = CEE_XOR;
1357                                 ins->sreg1 = ins->dreg;
1358                                 ins->sreg2 = ins->dreg;
1359                         }
1360                         break;
1361                 case OP_MUL_IMM:
1362                         /* remove unnecessary multiplication with 1 */
1363                         if (ins->inst_imm == 1) {
1364                                 if (ins->dreg != ins->sreg1) {
1365                                         ins->opcode = OP_MOVE;
1366                                 } else {
1367                                         last_ins->next = ins->next;
1368                                         ins = ins->next;
1369                                         continue;
1370                                 }
1371                         }
1372                         break;
1373                 case OP_COMPARE_IMM:
1374                         /* OP_COMPARE_IMM (reg, 0)
1375                          * -->
1376                          * OP_AMD64_TEST_NULL (reg)
1377                          */
1378                         if (!ins->inst_imm)
1379                                 ins->opcode = OP_AMD64_TEST_NULL;
1380                         break;
1381                 case OP_ICOMPARE_IMM:
1382                         if (!ins->inst_imm)
1383                                 ins->opcode = OP_X86_TEST_NULL;
1384                         break;
1385                 case OP_AMD64_ICOMPARE_MEMBASE_IMM:
1386                         /*
1387                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
1388                          * OP_X86_COMPARE_MEMBASE_IMM offset(basereg), imm
1389                          * -->
1390                          * OP_STORE_MEMBASE_REG reg, offset(basereg)
1391                          * OP_COMPARE_IMM reg, imm
1392                          *
1393                          * Note: if imm = 0 then OP_COMPARE_IMM replaced with OP_X86_TEST_NULL
1394                          */
1395                         if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_REG) &&
1396                             ins->inst_basereg == last_ins->inst_destbasereg &&
1397                             ins->inst_offset == last_ins->inst_offset) {
1398                                         ins->opcode = OP_ICOMPARE_IMM;
1399                                         ins->sreg1 = last_ins->sreg1;
1400
1401                                         /* check if we can remove cmp reg,0 with test null */
1402                                         if (!ins->inst_imm)
1403                                                 ins->opcode = OP_X86_TEST_NULL;
1404                                 }
1405
1406                         break;
1407                 case OP_LOAD_MEMBASE:
1408                 case OP_LOADI4_MEMBASE:
1409                         /*
1410                          * Note: if reg1 = reg2 the load op is removed
1411                          *
1412                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1413                          * OP_LOAD_MEMBASE offset(basereg), reg2
1414                          * -->
1415                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1416                          * OP_MOVE reg1, reg2
1417                          */
1418                         if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_REG
1419                                          || last_ins->opcode == OP_STORE_MEMBASE_REG) &&
1420                             ins->inst_basereg == last_ins->inst_destbasereg &&
1421                             ins->inst_offset == last_ins->inst_offset) {
1422                                 if (ins->dreg == last_ins->sreg1) {
1423                                         last_ins->next = ins->next;
1424                                         ins = ins->next;
1425                                         continue;
1426                                 } else {
1427                                         //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1428                                         ins->opcode = OP_MOVE;
1429                                         ins->sreg1 = last_ins->sreg1;
1430                                 }
1431
1432                         /*
1433                          * Note: reg1 must be different from the basereg in the second load
1434                          * Note: if reg1 = reg2 is equal then second load is removed
1435                          *
1436                          * OP_LOAD_MEMBASE offset(basereg), reg1
1437                          * OP_LOAD_MEMBASE offset(basereg), reg2
1438                          * -->
1439                          * OP_LOAD_MEMBASE offset(basereg), reg1
1440                          * OP_MOVE reg1, reg2
1441                          */
1442                         } if (last_ins && (last_ins->opcode == OP_LOADI4_MEMBASE
1443                                            || last_ins->opcode == OP_LOAD_MEMBASE) &&
1444                               ins->inst_basereg != last_ins->dreg &&
1445                               ins->inst_basereg == last_ins->inst_basereg &&
1446                               ins->inst_offset == last_ins->inst_offset) {
1447
1448                                 if (ins->dreg == last_ins->dreg) {
1449                                         last_ins->next = ins->next;
1450                                         ins = ins->next;
1451                                         continue;
1452                                 } else {
1453                                         ins->opcode = OP_MOVE;
1454                                         ins->sreg1 = last_ins->dreg;
1455                                 }
1456
1457                                 //g_assert_not_reached ();
1458
1459 #if 0
1460                         /*
1461                          * OP_STORE_MEMBASE_IMM imm, offset(basereg)
1462                          * OP_LOAD_MEMBASE offset(basereg), reg
1463                          * -->
1464                          * OP_STORE_MEMBASE_IMM imm, offset(basereg)
1465                          * OP_ICONST reg, imm
1466                          */
1467                         } else if (last_ins && (last_ins->opcode == OP_STOREI4_MEMBASE_IMM
1468                                                 || last_ins->opcode == OP_STORE_MEMBASE_IMM) &&
1469                                    ins->inst_basereg == last_ins->inst_destbasereg &&
1470                                    ins->inst_offset == last_ins->inst_offset) {
1471                                 //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1472                                 ins->opcode = OP_ICONST;
1473                                 ins->inst_c0 = last_ins->inst_imm;
1474                                 g_assert_not_reached (); // check this rule
1475 #endif
1476                         }
1477                         break;
1478                 case OP_LOADU1_MEMBASE:
1479                 case OP_LOADI1_MEMBASE:
1480                         /*
1481                          * Note: if reg1 = reg2 the load op is removed
1482                          *
1483                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1484                          * OP_LOAD_MEMBASE offset(basereg), reg2
1485                          * -->
1486                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1487                          * OP_MOVE reg1, reg2
1488                          */
1489                         if (last_ins && (last_ins->opcode == OP_STOREI1_MEMBASE_REG) &&
1490                                         ins->inst_basereg == last_ins->inst_destbasereg &&
1491                                         ins->inst_offset == last_ins->inst_offset) {
1492                                 if (ins->dreg == last_ins->sreg1) {
1493                                         last_ins->next = ins->next;
1494                                         ins = ins->next;
1495                                         continue;
1496                                 } else {
1497                                         //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1498                                         ins->opcode = OP_MOVE;
1499                                         ins->sreg1 = last_ins->sreg1;
1500                                 }
1501                         }
1502                         break;
1503                 case OP_LOADU2_MEMBASE:
1504                 case OP_LOADI2_MEMBASE:
1505                         /*
1506                          * Note: if reg1 = reg2 the load op is removed
1507                          *
1508                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1509                          * OP_LOAD_MEMBASE offset(basereg), reg2
1510                          * -->
1511                          * OP_STORE_MEMBASE_REG reg1, offset(basereg)
1512                          * OP_MOVE reg1, reg2
1513                          */
1514                         if (last_ins && (last_ins->opcode == OP_STOREI2_MEMBASE_REG) &&
1515                                         ins->inst_basereg == last_ins->inst_destbasereg &&
1516                                         ins->inst_offset == last_ins->inst_offset) {
1517                                 if (ins->dreg == last_ins->sreg1) {
1518                                         last_ins->next = ins->next;
1519                                         ins = ins->next;
1520                                         continue;
1521                                 } else {
1522                                         //static int c = 0; printf ("MATCHX %s %d\n", cfg->method->name,c++);
1523                                         ins->opcode = OP_MOVE;
1524                                         ins->sreg1 = last_ins->sreg1;
1525                                 }
1526                         }
1527                         break;
1528                 case CEE_CONV_I4:
1529                 case CEE_CONV_U4:
1530                 case OP_MOVE:
1531                         /*
1532                          * Removes:
1533                          *
1534                          * OP_MOVE reg, reg
1535                          */
1536                         if (ins->dreg == ins->sreg1) {
1537                                 if (last_ins)
1538                                         last_ins->next = ins->next;
1539                                 ins = ins->next;
1540                                 continue;
1541                         }
1542                         /*
1543                          * Removes:
1544                          *
1545                          * OP_MOVE sreg, dreg
1546                          * OP_MOVE dreg, sreg
1547                          */
1548                         if (last_ins && last_ins->opcode == OP_MOVE &&
1549                             ins->sreg1 == last_ins->dreg &&
1550                             ins->dreg == last_ins->sreg1) {
1551                                 last_ins->next = ins->next;
1552                                 ins = ins->next;
1553                                 continue;
1554                         }
1555                         break;
1556                 }
1557                 last_ins = ins;
1558                 ins = ins->next;
1559         }
1560         bb->last_ins = last_ins;
1561 }
1562
1563 static const int
1564 branch_cc_table [] = {
1565         X86_CC_EQ, X86_CC_GE, X86_CC_GT, X86_CC_LE, X86_CC_LT,
1566         X86_CC_NE, X86_CC_GE, X86_CC_GT, X86_CC_LE, X86_CC_LT,
1567         X86_CC_O, X86_CC_NO, X86_CC_C, X86_CC_NC
1568 };
1569
1570 static int
1571 opcode_to_x86_cond (int opcode)
1572 {
1573         switch (opcode) {
1574         case OP_IBEQ:
1575                 return X86_CC_EQ;
1576         case OP_IBNE_UN:
1577                 return X86_CC_NE;
1578         case OP_IBLT:
1579                 return X86_CC_LT;
1580         case OP_IBLT_UN:
1581                 return X86_CC_LT;
1582         case OP_IBGT:
1583                 return X86_CC_GT;
1584         case OP_IBGT_UN:
1585                 return X86_CC_GT;
1586         case OP_IBGE:
1587                 return X86_CC_GE;
1588         case OP_IBGE_UN:
1589                 return X86_CC_GE;
1590         case OP_IBLE:
1591                 return X86_CC_LE;
1592         case OP_IBLE_UN:
1593                 return X86_CC_LE;
1594         case OP_COND_EXC_IOV:
1595                 return X86_CC_O;
1596         case OP_COND_EXC_IC:
1597                 return X86_CC_C;
1598         default:
1599                 g_assert_not_reached ();
1600         }
1601
1602         return -1;
1603 }
1604
1605 /*
1606  * returns the offset used by spillvar. It allocates a new
1607  * spill variable if necessary.
1608  */
1609 static int
1610 mono_spillvar_offset (MonoCompile *cfg, int spillvar)
1611 {
1612         MonoSpillInfo **si, *info;
1613         int i = 0;
1614
1615         si = &cfg->spill_info;
1616
1617         while (i <= spillvar) {
1618
1619                 if (!*si) {
1620                         *si = info = mono_mempool_alloc (cfg->mempool, sizeof (MonoSpillInfo));
1621                         info->next = NULL;
1622                         cfg->stack_offset += sizeof (gpointer);
1623                         info->offset = - cfg->stack_offset;
1624                 }
1625
1626                 if (i == spillvar)
1627                         return (*si)->offset;
1628
1629                 i++;
1630                 si = &(*si)->next;
1631         }
1632
1633         g_assert_not_reached ();
1634         return 0;
1635 }
1636
1637 /*
1638  * returns the offset used by spillvar. It allocates a new
1639  * spill float variable if necessary.
1640  * (same as mono_spillvar_offset but for float)
1641  */
1642 static int
1643 mono_spillvar_offset_float (MonoCompile *cfg, int spillvar)
1644 {
1645         MonoSpillInfo **si, *info;
1646         int i = 0;
1647
1648         si = &cfg->spill_info_float;
1649
1650         while (i <= spillvar) {
1651
1652                 if (!*si) {
1653                         *si = info = mono_mempool_alloc (cfg->mempool, sizeof (MonoSpillInfo));
1654                         info->next = NULL;
1655                         cfg->stack_offset += sizeof (double);
1656                         info->offset = - cfg->stack_offset;
1657                 }
1658
1659                 if (i == spillvar)
1660                         return (*si)->offset;
1661
1662                 i++;
1663                 si = &(*si)->next;
1664         }
1665
1666         g_assert_not_reached ();
1667         return 0;
1668 }
1669
1670 /*
1671  * Creates a store for spilled floating point items
1672  */
1673 static MonoInst*
1674 create_spilled_store_float (MonoCompile *cfg, int spill, int reg, MonoInst *ins)
1675 {
1676         MonoInst *store;
1677         MONO_INST_NEW (cfg, store, OP_STORER8_MEMBASE_REG);
1678         store->sreg1 = reg;
1679         store->inst_destbasereg = AMD64_RBP;
1680         store->inst_offset = mono_spillvar_offset_float (cfg, spill);
1681
1682         DEBUG (g_print ("SPILLED FLOAT STORE (%d at 0x%08lx(%%sp)) (from %d)\n", spill, (long)store->inst_offset, reg));
1683         return store;
1684 }
1685
1686 /*
1687  * Creates a load for spilled floating point items
1688  */
1689 static MonoInst*
1690 create_spilled_load_float (MonoCompile *cfg, int spill, int reg, MonoInst *ins)
1691 {
1692         MonoInst *load;
1693         MONO_INST_NEW (cfg, load, OP_LOADR8_SPILL_MEMBASE);
1694         load->dreg = reg;
1695         load->inst_basereg = AMD64_RBP;
1696         load->inst_offset = mono_spillvar_offset_float (cfg, spill);
1697
1698         DEBUG (g_print ("SPILLED FLOAT LOAD (%d at 0x%08lx(%%sp)) (from %d)\n", spill, (long)load->inst_offset, reg));
1699         return load;
1700 }
1701
1702 #define is_global_ireg(r) ((r) >= 0 && (r) <= 15 && AMD64_IS_CALLEE_SAVED_REG ((r)))
1703 #define ireg_is_freeable(r) ((r) >= 0 && (r) <= 15 && AMD64_IS_CALLEE_REG ((r)))
1704 #define freg_is_freeable(r) ((r) >= 0 && (r) <= AMD64_XMM_NREG)
1705
1706 #define reg_is_freeable(r,fp) ((fp) ? freg_is_freeable ((r)) : ireg_is_freeable ((r)))
1707 #define reg_is_hard(r,fp) ((fp) ? ((r) < MONO_MAX_FREGS) : ((r) < MONO_MAX_IREGS))
1708 #define reg_is_soft(r,fp) (!reg_is_hard((r),(fp)))
1709 #define rassign(cfg,reg,fp) ((fp) ? (cfg)->rs->fassign [(reg)] : (cfg)->rs->iassign [(reg)])
1710 #define sreg1_is_fp(ins) (ins_spec [(ins)->opcode] [MONO_INST_SRC1] == 'f')
1711 #define sreg2_is_fp(ins) (ins_spec [(ins)->opcode] [MONO_INST_SRC2] == 'f')
1712 #define dreg_is_fp(ins)  (ins_spec [(ins)->opcode] [MONO_INST_DEST] == 'f')
1713
1714 typedef struct {
1715         int born_in;
1716         int killed_in;
1717         int last_use;
1718         int prev_use;
1719         int flags;              /* used to track fp spill/load */
1720 } RegTrack;
1721
1722 static const char*const * ins_spec = amd64_desc;
1723
1724 static void
1725 print_ins (int i, MonoInst *ins)
1726 {
1727         const char *spec = ins_spec [ins->opcode];
1728         g_print ("\t%-2d %s", i, mono_inst_name (ins->opcode));
1729         if (!spec)
1730                 g_error ("Unknown opcode: %s\n", mono_inst_name (ins->opcode));
1731         if (spec [MONO_INST_DEST]) {
1732                 gboolean fp = (spec [MONO_INST_DEST] == 'f');
1733                 if (reg_is_soft (ins->dreg, fp))
1734                         g_print (" R%d <-", ins->dreg);
1735                 else
1736                         g_print (" %s <-", mono_amd64_regname (ins->dreg, fp));
1737         }
1738         if (spec [MONO_INST_SRC1]) {
1739                 gboolean fp = (spec [MONO_INST_SRC1] == 'f');
1740                 if (reg_is_soft (ins->sreg1, fp))
1741                         g_print (" R%d", ins->sreg1);
1742                 else
1743                         g_print (" %s", mono_amd64_regname (ins->sreg1, fp));
1744         }
1745         if (spec [MONO_INST_SRC2]) {
1746                 gboolean fp = (spec [MONO_INST_SRC2] == 'f');
1747                 if (reg_is_soft (ins->sreg2, fp))
1748                         g_print (" R%d", ins->sreg2);
1749                 else
1750                         g_print (" %s", mono_amd64_regname (ins->sreg2, fp));
1751         }
1752         if (spec [MONO_INST_CLOB])
1753                 g_print (" clobbers: %c", spec [MONO_INST_CLOB]);
1754         g_print ("\n");
1755 }
1756
1757 static void
1758 print_regtrack (RegTrack *t, int num)
1759 {
1760         int i;
1761         char buf [32];
1762         const char *r;
1763
1764         for (i = 0; i < num; ++i) {
1765                 if (!t [i].born_in)
1766                         continue;
1767                 if (i >= MONO_MAX_IREGS) {
1768                         g_snprintf (buf, sizeof(buf), "R%d", i);
1769                         r = buf;
1770                 } else
1771                         r = mono_arch_regname (i);
1772                 g_print ("liveness: %s [%d - %d]\n", r, t [i].born_in, t[i].last_use);
1773         }
1774 }
1775
1776 typedef struct InstList InstList;
1777
1778 struct InstList {
1779         InstList *prev;
1780         InstList *next;
1781         MonoInst *data;
1782 };
1783
1784 static inline InstList*
1785 inst_list_prepend (MonoMemPool *pool, InstList *list, MonoInst *data)
1786 {
1787         InstList *item = mono_mempool_alloc (pool, sizeof (InstList));
1788         item->data = data;
1789         item->prev = NULL;
1790         item->next = list;
1791         if (list)
1792                 list->prev = item;
1793         return item;
1794 }
1795
1796 /*
1797  * Force the spilling of the variable in the symbolic register 'reg'.
1798  */
1799 static int
1800 get_register_force_spilling (MonoCompile *cfg, InstList *item, MonoInst *ins, int reg, gboolean fp)
1801 {
1802         MonoInst *load;
1803         int i, sel, spill;
1804         int *assign, *symbolic;
1805
1806         if (fp) {
1807                 assign = cfg->rs->fassign;
1808                 symbolic = cfg->rs->fsymbolic;
1809         }
1810         else {
1811                 assign = cfg->rs->iassign;
1812                 symbolic = cfg->rs->isymbolic;
1813         }
1814
1815         sel = assign [reg];
1816         /*i = cfg->rs->isymbolic [sel];
1817         g_assert (i == reg);*/
1818         i = reg;
1819         spill = ++cfg->spill_count;
1820         assign [i] = -spill - 1;
1821         if (fp)
1822                 mono_regstate_free_float (cfg->rs, sel);
1823         else
1824                 mono_regstate_free_int (cfg->rs, sel);
1825         /* we need to create a spill var and insert a load to sel after the current instruction */
1826         if (fp)
1827                 MONO_INST_NEW (cfg, load, OP_LOADR8_MEMBASE);
1828         else
1829                 MONO_INST_NEW (cfg, load, OP_LOAD_MEMBASE);
1830         load->dreg = sel;
1831         load->inst_basereg = AMD64_RBP;
1832         load->inst_offset = mono_spillvar_offset (cfg, spill);
1833         if (item->prev) {
1834                 while (ins->next != item->prev->data)
1835                         ins = ins->next;
1836         }
1837         load->next = ins->next;
1838         ins->next = load;
1839         DEBUG (g_print ("SPILLED LOAD (%d at 0x%08lx(%%ebp)) R%d (freed %s)\n", spill, (long)load->inst_offset, i, mono_amd64_regname (sel, fp)));
1840         if (fp)
1841                 i = mono_regstate_alloc_float (cfg->rs, 1 << sel);
1842         else
1843                 i = mono_regstate_alloc_int (cfg->rs, 1 << sel);
1844         g_assert (i == sel);
1845
1846         return sel;
1847 }
1848
1849 static int
1850 get_register_spilling (MonoCompile *cfg, InstList *item, MonoInst *ins, guint32 regmask, int reg, gboolean fp)
1851 {
1852         MonoInst *load;
1853         int i, sel, spill;
1854         int *assign, *symbolic;
1855
1856         if (fp) {
1857                 assign = cfg->rs->fassign;
1858                 symbolic = cfg->rs->fsymbolic;
1859         }
1860         else {
1861                 assign = cfg->rs->iassign;
1862                 symbolic = cfg->rs->isymbolic;
1863         }
1864
1865         DEBUG (g_print ("\tstart regmask to assign R%d: 0x%08x (R%d <- R%d R%d)\n", reg, regmask, ins->dreg, ins->sreg1, ins->sreg2));
1866         /* exclude the registers in the current instruction */
1867         if ((sreg1_is_fp (ins) == fp) && (reg != ins->sreg1) && (reg_is_freeable (ins->sreg1, fp) || (reg_is_soft (ins->sreg1, fp) && rassign (cfg, ins->sreg1, fp) >= 0))) {
1868                 if (reg_is_soft (ins->sreg1, fp))
1869                         regmask &= ~ (1 << rassign (cfg, ins->sreg1, fp));
1870                 else
1871                         regmask &= ~ (1 << ins->sreg1);
1872                 DEBUG (g_print ("\t\texcluding sreg1 %s\n", mono_amd64_regname (ins->sreg1, fp)));
1873         }
1874         if ((sreg2_is_fp (ins) == fp) && (reg != ins->sreg2) && (reg_is_freeable (ins->sreg2, fp) || (reg_is_soft (ins->sreg2, fp) && rassign (cfg, ins->sreg2, fp) >= 0))) {
1875                 if (reg_is_soft (ins->sreg2, fp))
1876                         regmask &= ~ (1 << rassign (cfg, ins->sreg2, fp));
1877                 else
1878                         regmask &= ~ (1 << ins->sreg2);
1879                 DEBUG (g_print ("\t\texcluding sreg2 %s %d\n", mono_amd64_regname (ins->sreg2, fp), ins->sreg2));
1880         }
1881         if ((dreg_is_fp (ins) == fp) && (reg != ins->dreg) && reg_is_freeable (ins->dreg, fp)) {
1882                 regmask &= ~ (1 << ins->dreg);
1883                 DEBUG (g_print ("\t\texcluding dreg %s\n", mono_amd64_regname (ins->dreg, fp)));
1884         }
1885
1886         DEBUG (g_print ("\t\tavailable regmask: 0x%08x\n", regmask));
1887         g_assert (regmask); /* need at least a register we can free */
1888         sel = -1;
1889         /* we should track prev_use and spill the register that's farther */
1890         if (fp) {
1891                 for (i = 0; i < MONO_MAX_FREGS; ++i) {
1892                         if (regmask & (1 << i)) {
1893                                 sel = i;
1894                                 DEBUG (g_print ("\t\tselected register %s has assignment %d\n", mono_arch_fregname (sel), cfg->rs->fassign [sel]));
1895                                 break;
1896                         }
1897                 }
1898
1899                 i = cfg->rs->fsymbolic [sel];
1900                 spill = ++cfg->spill_count;
1901                 cfg->rs->fassign [i] = -spill - 1;
1902                 mono_regstate_free_float (cfg->rs, sel);
1903         }
1904         else {
1905                 for (i = 0; i < MONO_MAX_IREGS; ++i) {
1906                         if (regmask & (1 << i)) {
1907                                 sel = i;
1908                                 DEBUG (g_print ("\t\tselected register %s has assignment %d\n", mono_arch_regname (sel), cfg->rs->iassign [sel]));
1909                                 break;
1910                         }
1911                 }
1912
1913                 i = cfg->rs->isymbolic [sel];
1914                 spill = ++cfg->spill_count;
1915                 cfg->rs->iassign [i] = -spill - 1;
1916                 mono_regstate_free_int (cfg->rs, sel);
1917         }
1918
1919         /* we need to create a spill var and insert a load to sel after the current instruction */
1920         MONO_INST_NEW (cfg, load, fp ? OP_LOADR8_MEMBASE : OP_LOAD_MEMBASE);
1921         load->dreg = sel;
1922         load->inst_basereg = AMD64_RBP;
1923         load->inst_offset = mono_spillvar_offset (cfg, spill);
1924         if (item->prev) {
1925                 while (ins->next != item->prev->data)
1926                         ins = ins->next;
1927         }
1928         load->next = ins->next;
1929         ins->next = load;
1930         DEBUG (g_print ("\tSPILLED LOAD (%d at 0x%08lx(%%ebp)) R%d (freed %s)\n", spill, (long)load->inst_offset, i, mono_amd64_regname (sel, fp)));
1931         if (fp)
1932                 i = mono_regstate_alloc_float (cfg->rs, 1 << sel);
1933         else
1934                 i = mono_regstate_alloc_int (cfg->rs, 1 << sel);
1935         g_assert (i == sel);
1936
1937         return sel;
1938 }
1939
1940 static MonoInst*
1941 create_copy_ins (MonoCompile *cfg, int dest, int src, MonoInst *ins, gboolean fp)
1942 {
1943         MonoInst *copy;
1944
1945         if (fp)
1946                 MONO_INST_NEW (cfg, copy, OP_FMOVE);
1947         else
1948                 MONO_INST_NEW (cfg, copy, OP_MOVE);
1949
1950         copy->dreg = dest;
1951         copy->sreg1 = src;
1952         if (ins) {
1953                 copy->next = ins->next;
1954                 ins->next = copy;
1955         }
1956         DEBUG (g_print ("\tforced copy from %s to %s\n", mono_arch_regname (src), mono_arch_regname (dest)));
1957         return copy;
1958 }
1959
1960 static MonoInst*
1961 create_spilled_store (MonoCompile *cfg, int spill, int reg, int prev_reg, MonoInst *ins, gboolean fp)
1962 {
1963         MonoInst *store;
1964         MONO_INST_NEW (cfg, store, fp ? OP_STORER8_MEMBASE_REG : OP_STORE_MEMBASE_REG);
1965         store->sreg1 = reg;
1966         store->inst_destbasereg = AMD64_RBP;
1967         store->inst_offset = mono_spillvar_offset (cfg, spill);
1968         if (ins) {
1969                 store->next = ins->next;
1970                 ins->next = store;
1971         }
1972         DEBUG (g_print ("\tSPILLED STORE (%d at 0x%08lx(%%ebp)) R%d (from %s)\n", spill, (long)store->inst_offset, prev_reg, mono_amd64_regname (reg, fp)));
1973         return store;
1974 }
1975
1976 static void
1977 insert_before_ins (MonoInst *ins, InstList *item, MonoInst* to_insert)
1978 {
1979         MonoInst *prev;
1980         if (item->next) {
1981                 prev = item->next->data;
1982
1983                 while (prev->next != ins)
1984                         prev = prev->next;
1985                 to_insert->next = ins;
1986                 prev->next = to_insert;
1987         } else {
1988                 to_insert->next = ins;
1989         }
1990         /*
1991          * needed otherwise in the next instruction we can add an ins to the
1992          * end and that would get past this instruction.
1993          */
1994         item->data = to_insert;
1995 }
1996
1997 /* flags used in reginfo->flags */
1998 enum {
1999         MONO_X86_FP_NEEDS_LOAD_SPILL    = 1 << 0,
2000         MONO_X86_FP_NEEDS_SPILL                 = 1 << 1,
2001         MONO_X86_FP_NEEDS_LOAD                  = 1 << 2,
2002         MONO_X86_REG_NOT_ECX                    = 1 << 3,
2003         MONO_X86_REG_EAX                                = 1 << 4,
2004         MONO_X86_REG_EDX                                = 1 << 5,
2005         MONO_X86_REG_ECX                                = 1 << 6
2006 };
2007
2008 static int
2009 mono_amd64_alloc_int_reg (MonoCompile *cfg, InstList *tmp, MonoInst *ins, guint32 dest_mask, int sym_reg, int flags)
2010 {
2011         int val;
2012         int test_mask = dest_mask;
2013
2014         if (flags & MONO_X86_REG_EAX)
2015                 test_mask &= (1 << AMD64_RAX);
2016         else if (flags & MONO_X86_REG_EDX)
2017                 test_mask &= (1 << AMD64_RDX);
2018         else if (flags & MONO_X86_REG_ECX)
2019                 test_mask &= (1 << AMD64_RCX);
2020         else if (flags & MONO_X86_REG_NOT_ECX)
2021                 test_mask &= ~ (1 << AMD64_RCX);
2022
2023         val = mono_regstate_alloc_int (cfg->rs, test_mask);
2024         if (val >= 0 && test_mask != dest_mask)
2025                 DEBUG(g_print ("\tUsed flag to allocate reg %s for R%u\n", mono_arch_regname (val), sym_reg));
2026
2027         if (val < 0 && (flags & MONO_X86_REG_NOT_ECX)) {
2028                 DEBUG(g_print ("\tFailed to allocate flag suggested mask (%u) but exluding ECX\n", test_mask));
2029                 val = mono_regstate_alloc_int (cfg->rs, (dest_mask & (~1 << AMD64_RCX)));
2030         }
2031
2032         if (val < 0) {
2033                 val = mono_regstate_alloc_int (cfg->rs, dest_mask);
2034                 if (val < 0)
2035                         val = get_register_spilling (cfg, tmp, ins, dest_mask, sym_reg, FALSE);
2036         }
2037
2038         return val;
2039 }
2040
2041 static int
2042 mono_amd64_alloc_float_reg (MonoCompile *cfg, InstList *tmp, MonoInst *ins, guint32 dest_mask, int sym_reg)
2043 {
2044         int val;
2045
2046         val = mono_regstate_alloc_float (cfg->rs, dest_mask);
2047
2048         if (val < 0) {
2049                 val = get_register_spilling (cfg, tmp, ins, dest_mask, sym_reg, TRUE);
2050         }
2051
2052         return val;
2053 }
2054
2055 static inline void
2056 assign_ireg (MonoRegState *rs, int reg, int hreg)
2057 {
2058         g_assert (reg >= MONO_MAX_IREGS);
2059         g_assert (hreg < MONO_MAX_IREGS);
2060         g_assert (! is_global_ireg (hreg));
2061
2062         rs->iassign [reg] = hreg;
2063         rs->isymbolic [hreg] = reg;
2064         rs->ifree_mask &= ~ (1 << hreg);
2065 }
2066
2067 /*#include "cprop.c"*/
2068
2069 /*
2070  * Local register allocation.
2071  * We first scan the list of instructions and we save the liveness info of
2072  * each register (when the register is first used, when it's value is set etc.).
2073  * We also reverse the list of instructions (in the InstList list) because assigning
2074  * registers backwards allows for more tricks to be used.
2075  */
2076 void
2077 mono_arch_local_regalloc (MonoCompile *cfg, MonoBasicBlock *bb)
2078 {
2079         MonoInst *ins;
2080         MonoRegState *rs = cfg->rs;
2081         int i, val, fpcount;
2082         RegTrack *reginfo, *reginfof;
2083         RegTrack *reginfo1, *reginfo2, *reginfod;
2084         InstList *tmp, *reversed = NULL;
2085         const char *spec;
2086         guint32 src1_mask, src2_mask, dest_mask;
2087         GList *fspill_list = NULL;
2088         int fspill = 0;
2089
2090         if (!bb->code)
2091                 return;
2092         rs->next_vireg = bb->max_ireg;
2093         rs->next_vfreg = bb->max_freg;
2094         mono_regstate_assign (rs);
2095         reginfo = g_malloc0 (sizeof (RegTrack) * rs->next_vireg);
2096         reginfof = g_malloc0 (sizeof (RegTrack) * rs->next_vfreg);
2097         rs->ifree_mask = AMD64_CALLEE_REGS;
2098         rs->ffree_mask = AMD64_CALLEE_FREGS;
2099
2100         if (!use_sse2)
2101                 /* The fp stack is 6 entries deep */
2102                 rs->ffree_mask = 0x3f;
2103
2104         ins = bb->code;
2105
2106         /*if (cfg->opt & MONO_OPT_COPYPROP)
2107                 local_copy_prop (cfg, ins);*/
2108
2109         i = 1;
2110         fpcount = 0;
2111         DEBUG (g_print ("LOCAL regalloc: basic block: %d\n", bb->block_num));
2112         /* forward pass on the instructions to collect register liveness info */
2113         while (ins) {
2114                 spec = ins_spec [ins->opcode];
2115
2116                 DEBUG (print_ins (i, ins));
2117
2118                 if (spec [MONO_INST_SRC1]) {
2119                         if (spec [MONO_INST_SRC1] == 'f') {
2120                                 reginfo1 = reginfof;
2121
2122                                 if (!use_sse2) {
2123                                         GList *spill;
2124
2125                                         spill = g_list_first (fspill_list);
2126                                         if (spill && fpcount < FPSTACK_SIZE) {
2127                                                 reginfo1 [ins->sreg1].flags |= MONO_X86_FP_NEEDS_LOAD;
2128                                                 fspill_list = g_list_remove (fspill_list, spill->data);
2129                                         } else
2130                                                 fpcount--;
2131                                 }
2132                         }
2133                         else
2134                                 reginfo1 = reginfo;
2135                         reginfo1 [ins->sreg1].prev_use = reginfo1 [ins->sreg1].last_use;
2136                         reginfo1 [ins->sreg1].last_use = i;
2137                         if (spec [MONO_INST_SRC1] == 'L') {
2138                                 /* The virtual register is allocated sequentially */
2139                                 reginfo1 [ins->sreg1 + 1].prev_use = reginfo1 [ins->sreg1 + 1].last_use;
2140                                 reginfo1 [ins->sreg1 + 1].last_use = i;
2141                                 if (reginfo1 [ins->sreg1 + 1].born_in == 0 || reginfo1 [ins->sreg1 + 1].born_in > i)
2142                                         reginfo1 [ins->sreg1 + 1].born_in = i;
2143
2144                                 reginfo1 [ins->sreg1].flags |= MONO_X86_REG_EAX;
2145                                 reginfo1 [ins->sreg1 + 1].flags |= MONO_X86_REG_EDX;
2146                         }
2147                 } else {
2148                         ins->sreg1 = -1;
2149                 }
2150                 if (spec [MONO_INST_SRC2]) {
2151                         if (spec [MONO_INST_SRC2] == 'f') {
2152                                 reginfo2 = reginfof;
2153
2154                                 if (!use_sse2) {
2155                                         GList *spill;
2156
2157                                         spill = g_list_first (fspill_list);
2158                                         if (spill) {
2159                                                 reginfo2 [ins->sreg2].flags |= MONO_X86_FP_NEEDS_LOAD;
2160                                                 fspill_list = g_list_remove (fspill_list, spill->data);
2161                                                 if (fpcount >= FPSTACK_SIZE) {
2162                                                         fspill++;
2163                                                         fspill_list = g_list_prepend (fspill_list, GINT_TO_POINTER(fspill));
2164                                                         reginfo2 [ins->sreg2].flags |= MONO_X86_FP_NEEDS_LOAD_SPILL;
2165                                                 }
2166                                         } else
2167                                                 fpcount--;
2168                                 }
2169                         }
2170                         else
2171                                 reginfo2 = reginfo;
2172                         reginfo2 [ins->sreg2].prev_use = reginfo2 [ins->sreg2].last_use;
2173                         reginfo2 [ins->sreg2].last_use = i;
2174                         if (spec [MONO_INST_SRC2] == 'L') {
2175                                 /* The virtual register is allocated sequentially */
2176                                 reginfo2 [ins->sreg2 + 1].prev_use = reginfo2 [ins->sreg2 + 1].last_use;
2177                                 reginfo2 [ins->sreg2 + 1].last_use = i;
2178                                 if (reginfo2 [ins->sreg2 + 1].born_in == 0 || reginfo2 [ins->sreg2 + 1].born_in > i)
2179                                         reginfo2 [ins->sreg2 + 1].born_in = i;
2180                         }
2181                         if (spec [MONO_INST_CLOB] == 's') {
2182                                 reginfo2 [ins->sreg1].flags |= MONO_X86_REG_NOT_ECX;
2183                                 reginfo2 [ins->sreg2].flags |= MONO_X86_REG_ECX;
2184                         }
2185                 } else {
2186                         ins->sreg2 = -1;
2187                 }
2188                 if (spec [MONO_INST_DEST]) {
2189                         if (spec [MONO_INST_DEST] == 'f') {
2190                                 reginfod = reginfof;
2191                                 if (!use_sse2 && (spec [MONO_INST_CLOB] != 'm')) {
2192                                         if (fpcount >= FPSTACK_SIZE) {
2193                                                 reginfod [ins->dreg].flags |= MONO_X86_FP_NEEDS_SPILL;
2194                                                 fspill++;
2195                                                 fspill_list = g_list_prepend (fspill_list, GINT_TO_POINTER(fspill));
2196                                                 fpcount--;
2197                                         }
2198                                         fpcount++;
2199                                 }
2200                         }
2201                         else
2202                                 reginfod = reginfo;
2203                         if (spec [MONO_INST_DEST] != 'b') /* it's not just a base register */
2204                                 reginfod [ins->dreg].killed_in = i;
2205                         reginfod [ins->dreg].prev_use = reginfod [ins->dreg].last_use;
2206                         reginfod [ins->dreg].last_use = i;
2207                         if (reginfod [ins->dreg].born_in == 0 || reginfod [ins->dreg].born_in > i)
2208                                 reginfod [ins->dreg].born_in = i;
2209                         if (spec [MONO_INST_DEST] == 'l' || spec [MONO_INST_DEST] == 'L') {
2210                                 /* The virtual register is allocated sequentially */
2211                                 reginfod [ins->dreg + 1].prev_use = reginfod [ins->dreg + 1].last_use;
2212                                 reginfod [ins->dreg + 1].last_use = i;
2213                                 if (reginfod [ins->dreg + 1].born_in == 0 || reginfod [ins->dreg + 1].born_in > i)
2214                                         reginfod [ins->dreg + 1].born_in = i;
2215
2216                                 reginfod [ins->dreg].flags |= MONO_X86_REG_EAX;
2217                                 reginfod [ins->dreg + 1].flags |= MONO_X86_REG_EDX;
2218                         }
2219                 } else {
2220                         ins->dreg = -1;
2221                 }
2222
2223                 if (spec [MONO_INST_CLOB] == 'c') {
2224                         /* A call instruction implicitly uses all registers in call->out_ireg_args */
2225
2226                         MonoCallInst *call = (MonoCallInst*)ins;
2227                         GSList *list;
2228
2229                         list = call->out_ireg_args;
2230                         if (list) {
2231                                 while (list) {
2232                                         guint64 regpair;
2233                                         int reg, hreg;
2234
2235                                         regpair = (guint64) (list->data);
2236                                         hreg = regpair >> 32;
2237                                         reg = regpair & 0xffffffff;
2238
2239                                         reginfo [reg].prev_use = reginfo [reg].last_use;
2240                                         reginfo [reg].last_use = i;
2241
2242                                         list = g_slist_next (list);
2243                                 }
2244                         }
2245
2246                         list = call->out_freg_args;
2247                         if (use_sse2 && list) {
2248                                 while (list) {
2249                                         guint64 regpair;
2250                                         int reg, hreg;
2251
2252                                         regpair = (guint64) (list->data);
2253                                         hreg = regpair >> 32;
2254                                         reg = regpair & 0xffffffff;
2255
2256                                         reginfof [reg].prev_use = reginfof [reg].last_use;
2257                                         reginfof [reg].last_use = i;
2258
2259                                         list = g_slist_next (list);
2260                                 }
2261                         }
2262                 }
2263
2264                 reversed = inst_list_prepend (cfg->mempool, reversed, ins);
2265                 ++i;
2266                 ins = ins->next;
2267         }
2268
2269         // todo: check if we have anything left on fp stack, in verify mode?
2270         fspill = 0;
2271
2272         DEBUG (print_regtrack (reginfo, rs->next_vireg));
2273         DEBUG (print_regtrack (reginfof, rs->next_vfreg));
2274         tmp = reversed;
2275         while (tmp) {
2276                 int prev_dreg, prev_sreg1, prev_sreg2, clob_dreg;
2277                 dest_mask = src1_mask = src2_mask = AMD64_CALLEE_REGS;
2278                 --i;
2279                 ins = tmp->data;
2280                 spec = ins_spec [ins->opcode];
2281                 prev_dreg = -1;
2282                 clob_dreg = -1;
2283                 DEBUG (g_print ("processing:"));
2284                 DEBUG (print_ins (i, ins));
2285                 if (spec [MONO_INST_CLOB] == 's') {
2286                         /*
2287                          * Shift opcodes, SREG2 must be RCX
2288                          */
2289                         if (rs->ifree_mask & (1 << AMD64_RCX)) {
2290                                 if (ins->sreg2 < MONO_MAX_IREGS) {
2291                                         /* Argument already in hard reg, need to copy */
2292                                         MonoInst *copy = create_copy_ins (cfg, AMD64_RCX, ins->sreg2, NULL, FALSE);
2293                                         insert_before_ins (ins, tmp, copy);
2294                                 }
2295                                 else {
2296                                         DEBUG (g_print ("\tshortcut assignment of R%d to ECX\n", ins->sreg2));
2297                                         assign_ireg (rs, ins->sreg2, AMD64_RCX);
2298                                 }
2299                         } else {
2300                                 int need_ecx_spill = TRUE;
2301                                 /*
2302                                  * we first check if src1/dreg is already assigned a register
2303                                  * and then we force a spill of the var assigned to ECX.
2304                                  */
2305                                 /* the destination register can't be ECX */
2306                                 dest_mask &= ~ (1 << AMD64_RCX);
2307                                 src1_mask &= ~ (1 << AMD64_RCX);
2308                                 val = rs->iassign [ins->dreg];
2309                                 /*
2310                                  * the destination register is already assigned to ECX:
2311                                  * we need to allocate another register for it and then
2312                                  * copy from this to ECX.
2313                                  */
2314                                 if (val == AMD64_RCX && ins->dreg != ins->sreg2) {
2315                                         int new_dest;
2316                                         new_dest = mono_amd64_alloc_int_reg (cfg, tmp, ins, dest_mask, ins->dreg, reginfo [ins->dreg].flags);
2317                                         g_assert (new_dest >= 0);
2318                                         DEBUG (g_print ("\tclob:s changing dreg R%d to %s from ECX\n", ins->dreg, mono_arch_regname (new_dest)));
2319
2320                                         rs->isymbolic [new_dest] = ins->dreg;
2321                                         rs->iassign [ins->dreg] = new_dest;
2322                                         clob_dreg = ins->dreg;
2323                                         ins->dreg = new_dest;
2324                                         create_copy_ins (cfg, AMD64_RCX, new_dest, ins, FALSE);
2325                                         need_ecx_spill = FALSE;
2326                                         /*DEBUG (g_print ("\tforced spill of R%d\n", ins->dreg));
2327                                         val = get_register_force_spilling (cfg, tmp, ins, ins->dreg);
2328                                         rs->iassign [ins->dreg] = val;
2329                                         rs->isymbolic [val] = prev_dreg;
2330                                         ins->dreg = val;*/
2331                                 }
2332                                 if (is_global_ireg (ins->sreg2)) {
2333                                         MonoInst *copy = create_copy_ins (cfg, AMD64_RCX, ins->sreg2, NULL, FALSE);
2334                                         insert_before_ins (ins, tmp, copy);
2335                                 }
2336                                 else {
2337                                         val = rs->iassign [ins->sreg2];
2338                                         if (val >= 0 && val != AMD64_RCX) {
2339                                                 MonoInst *move = create_copy_ins (cfg, AMD64_RCX, val, NULL, FALSE);
2340                                                 DEBUG (g_print ("\tmoved arg from R%d (%d) to ECX\n", val, ins->sreg2));
2341                                                 move->next = ins;
2342                                                 g_assert_not_reached ();
2343                                                 /* FIXME: where is move connected to the instruction list? */
2344                                                 //tmp->prev->data->next = move;
2345                                         }
2346                                         else {
2347                                                 if (val == AMD64_RCX)
2348                                                 need_ecx_spill = FALSE;
2349                                         }
2350                                 }
2351                                 if (need_ecx_spill && !(rs->ifree_mask & (1 << AMD64_RCX))) {
2352                                         DEBUG (g_print ("\tforced spill of R%d\n", rs->isymbolic [AMD64_RCX]));
2353                                         get_register_force_spilling (cfg, tmp, ins, rs->isymbolic [AMD64_RCX], FALSE);
2354                                         mono_regstate_free_int (rs, AMD64_RCX);
2355                                 }
2356                                 if (!is_global_ireg (ins->sreg2))
2357                                         /* force-set sreg2 */
2358                                         assign_ireg (rs, ins->sreg2, AMD64_RCX);
2359                         }
2360                         ins->sreg2 = AMD64_RCX;
2361                 } else if (spec [MONO_INST_CLOB] == 'd') {
2362                         /*
2363                          * DIVISION/REMAINER
2364                          */
2365                         int dest_reg = AMD64_RAX;
2366                         int clob_reg = AMD64_RDX;
2367                         if (spec [MONO_INST_DEST] == 'd') {
2368                                 dest_reg = AMD64_RDX; /* reminder */
2369                                 clob_reg = AMD64_RAX;
2370                         }
2371                         if (is_global_ireg (ins->dreg))
2372                                 val = ins->dreg;
2373                         else
2374                                 val = rs->iassign [ins->dreg];
2375                         if (0 && val >= 0 && val != dest_reg && !(rs->ifree_mask & (1 << dest_reg))) {
2376                                 DEBUG (g_print ("\tforced spill of R%d\n", rs->isymbolic [dest_reg]));
2377                                 get_register_force_spilling (cfg, tmp, ins, rs->isymbolic [dest_reg], FALSE);
2378                                 mono_regstate_free_int (rs, dest_reg);
2379                         }
2380                         if (val < 0) {
2381                                 if (val < -1) {
2382                                         /* the register gets spilled after this inst */
2383                                         int spill = -val -1;
2384                                         dest_mask = 1 << clob_reg;
2385                                         prev_dreg = ins->dreg;
2386                                         val = mono_regstate_alloc_int (rs, dest_mask);
2387                                         if (val < 0)
2388                                                 val = get_register_spilling (cfg, tmp, ins, dest_mask, ins->dreg, FALSE);
2389                                         rs->iassign [ins->dreg] = val;
2390                                         if (spill)
2391                                                 create_spilled_store (cfg, spill, val, prev_dreg, ins, FALSE);
2392                                         DEBUG (g_print ("\tassigned dreg %s to dest R%d\n", mono_arch_regname (val), ins->dreg));
2393                                         rs->isymbolic [val] = prev_dreg;
2394                                         ins->dreg = val;
2395                                 } else {
2396                                         DEBUG (g_print ("\tshortcut assignment of R%d to %s\n", ins->dreg, mono_arch_regname (dest_reg)));
2397                                         prev_dreg = ins->dreg;
2398                                         assign_ireg (rs, ins->dreg, dest_reg);
2399                                         ins->dreg = dest_reg;
2400                                         val = dest_reg;
2401                                 }
2402                         }
2403
2404                         //DEBUG (g_print ("dest reg in div assigned: %s\n", mono_arch_regname (val)));
2405                         if (val != dest_reg) { /* force a copy */
2406                                 create_copy_ins (cfg, val, dest_reg, ins, FALSE);
2407                                 if (!(rs->ifree_mask & (1 << dest_reg)) && rs->isymbolic [dest_reg] >= MONO_MAX_IREGS) {
2408                                         DEBUG (g_print ("\tforced spill of R%d\n", rs->isymbolic [dest_reg]));
2409                                         get_register_force_spilling (cfg, tmp, ins, rs->isymbolic [dest_reg], FALSE);
2410                                         mono_regstate_free_int (rs, dest_reg);
2411                                 }
2412                         }
2413                         if (!(rs->ifree_mask & (1 << clob_reg)) && (clob_reg != val) && (rs->isymbolic [clob_reg] >= MONO_MAX_IREGS)) {
2414                                 DEBUG (g_print ("\tforced spill of clobbered reg R%d\n", rs->isymbolic [clob_reg]));
2415                                 get_register_force_spilling (cfg, tmp, ins, rs->isymbolic [clob_reg], FALSE);
2416                                 mono_regstate_free_int (rs, clob_reg);
2417                         }
2418                         src1_mask = 1 << AMD64_RAX;
2419                         src2_mask = 1 << AMD64_RCX;
2420                 }
2421                 if (spec [MONO_INST_DEST] == 'l') {
2422                         int hreg;
2423                         val = rs->iassign [ins->dreg];
2424                         /* check special case when dreg have been moved from ecx (clob shift) */
2425                         if (spec [MONO_INST_CLOB] == 's' && clob_dreg != -1)
2426                                 hreg = clob_dreg + 1;
2427                         else
2428                                 hreg = ins->dreg + 1;
2429
2430                         /* base prev_dreg on fixed hreg, handle clob case */
2431                         val = hreg - 1;
2432
2433                         if (val != rs->isymbolic [AMD64_RAX] && !(rs->ifree_mask & (1 << AMD64_RAX))) {
2434                                 DEBUG (g_print ("\t(long-low) forced spill of R%d\n", rs->isymbolic [AMD64_RAX]));
2435                                 get_register_force_spilling (cfg, tmp, ins, rs->isymbolic [AMD64_RAX], FALSE);
2436                                 mono_regstate_free_int (rs, AMD64_RAX);
2437                         }
2438                         if (hreg != rs->isymbolic [AMD64_RDX] && !(rs->ifree_mask & (1 << AMD64_RDX))) {
2439                                 DEBUG (g_print ("\t(long-high) forced spill of R%d\n", rs->isymbolic [AMD64_RDX]));
2440                                 get_register_force_spilling (cfg, tmp, ins, rs->isymbolic [AMD64_RDX], FALSE);
2441                                 mono_regstate_free_int (rs, AMD64_RDX);
2442                         }
2443                 }
2444
2445                 /*
2446                  * TRACK DREG
2447                  */
2448                 if (spec [MONO_INST_DEST] == 'f') {
2449                         if (use_sse2) {
2450                                 /* Allocate an XMM reg the same way as an int reg */
2451                                 if (reg_is_soft (ins->dreg, TRUE)) {
2452                                         val = rs->fassign [ins->dreg];
2453                                         prev_dreg = ins->dreg;
2454
2455                                         if (val < 0) {
2456                                                 int spill = 0;
2457                                                 if (val < -1) {
2458                                                         /* the register gets spilled after this inst */
2459                                                         spill = -val -1;
2460                                                 }
2461                                                 val = mono_amd64_alloc_float_reg (cfg, tmp, ins, AMD64_CALLEE_FREGS, ins->dreg);
2462                                                 rs->fassign [ins->dreg] = val;
2463                                                 if (spill)
2464                                                         create_spilled_store (cfg, spill, val, prev_dreg, ins, TRUE);
2465                                         }
2466                                         DEBUG (g_print ("\tassigned dreg %s to dest R%d\n", mono_amd64_regname (val, TRUE), ins->dreg));
2467                                         rs->fsymbolic [val] = prev_dreg;
2468                                         ins->dreg = val;
2469                                 }
2470                         }
2471                         else if (spec [MONO_INST_CLOB] != 'm') {
2472                                 if (reginfof [ins->dreg].flags & MONO_X86_FP_NEEDS_SPILL) {
2473                                         GList *spill_node;
2474                                         MonoInst *store;
2475                                         spill_node = g_list_first (fspill_list);
2476                                         g_assert (spill_node);
2477
2478                                         store = create_spilled_store_float (cfg, GPOINTER_TO_INT (spill_node->data), ins->dreg, ins);
2479                                         insert_before_ins (ins, tmp, store);
2480                                         fspill_list = g_list_remove (fspill_list, spill_node->data);
2481                                         fspill--;
2482                                 }
2483                         }
2484                 } else if (spec [MONO_INST_DEST] == 'L') {
2485                         int hreg;
2486                         val = rs->iassign [ins->dreg];
2487                         /* check special case when dreg have been moved from ecx (clob shift) */
2488                         if (spec [MONO_INST_CLOB] == 's' && clob_dreg != -1)
2489                                 hreg = clob_dreg + 1;
2490                         else
2491                                 hreg = ins->dreg + 1;
2492
2493                         /* base prev_dreg on fixed hreg, handle clob case */
2494                         prev_dreg = hreg - 1;
2495
2496                         if (val < 0) {
2497                                 int spill = 0;
2498                                 if (val < -1) {
2499                                         /* the register gets spilled after this inst */
2500                                         spill = -val -1;
2501                                 }
2502                                 val = mono_amd64_alloc_int_reg (cfg, tmp, ins, dest_mask, ins->dreg, reginfo [ins->dreg].flags);
2503                                 rs->iassign [ins->dreg] = val;
2504                                 if (spill)
2505                                         create_spilled_store (cfg, spill, val, prev_dreg, ins, FALSE);
2506                         }
2507
2508                         DEBUG (g_print ("\tassigned dreg (long) %s to dest R%d\n", mono_arch_regname (val), hreg - 1));
2509
2510                         rs->isymbolic [val] = hreg - 1;
2511                         ins->dreg = val;
2512
2513                         val = rs->iassign [hreg];
2514                         if (val < 0) {
2515                                 int spill = 0;
2516                                 if (val < -1) {
2517                                         /* the register gets spilled after this inst */
2518                                         spill = -val -1;
2519                                 }
2520                                 val = mono_amd64_alloc_int_reg (cfg, tmp, ins, dest_mask, hreg, reginfo [hreg].flags);
2521                                 rs->iassign [hreg] = val;
2522                                 if (spill)
2523                                         create_spilled_store (cfg, spill, val, hreg, ins, FALSE);
2524                         }
2525
2526                         DEBUG (g_print ("\tassigned hreg (long-high) %s to dest R%d\n", mono_arch_regname (val), hreg));
2527                         rs->isymbolic [val] = hreg;
2528                         /* save reg allocating into unused */
2529                         ins->unused = val;
2530
2531                         /* check if we can free our long reg */
2532                         if (reg_is_freeable (val, FALSE) && hreg >= 0 && reginfo [hreg].born_in >= i) {
2533                                 DEBUG (g_print ("\tfreeable %s (R%d) (born in %d)\n", mono_arch_regname (val), hreg, reginfo [hreg].born_in));
2534                                 mono_regstate_free_int (rs, val);
2535                         }
2536                 }
2537                 else if (ins->dreg >= MONO_MAX_IREGS) {
2538                         int hreg;
2539                         val = rs->iassign [ins->dreg];
2540                         if (spec [MONO_INST_DEST] == 'l') {
2541                                 /* check special case when dreg have been moved from ecx (clob shift) */
2542                                 if (spec [MONO_INST_CLOB] == 's' && clob_dreg != -1)
2543                                         hreg = clob_dreg + 1;
2544                                 else
2545                                         hreg = ins->dreg + 1;
2546
2547                                 /* base prev_dreg on fixed hreg, handle clob case */
2548                                 prev_dreg = hreg - 1;
2549                         } else
2550                                 prev_dreg = ins->dreg;
2551
2552                         if (val < 0) {
2553                                 int spill = 0;
2554                                 if (val < -1) {
2555                                         /* the register gets spilled after this inst */
2556                                         spill = -val -1;
2557                                 }
2558                                 val = mono_amd64_alloc_int_reg (cfg, tmp, ins, dest_mask, ins->dreg, reginfo [ins->dreg].flags);
2559                                 rs->iassign [ins->dreg] = val;
2560                                 if (spill)
2561                                         create_spilled_store (cfg, spill, val, prev_dreg, ins, FALSE);
2562                         }
2563                         DEBUG (g_print ("\tassigned dreg %s to dest R%d\n", mono_arch_regname (val), ins->dreg));
2564                         rs->isymbolic [val] = prev_dreg;
2565                         ins->dreg = val;
2566                         /* handle cases where lreg needs to be eax:edx */
2567                         if (spec [MONO_INST_DEST] == 'l') {
2568                                 /* check special case when dreg have been moved from ecx (clob shift) */
2569                                 int hreg = prev_dreg + 1;
2570                                 val = rs->iassign [hreg];
2571                                 if (val < 0) {
2572                                         int spill = 0;
2573                                         if (val < -1) {
2574                                                 /* the register gets spilled after this inst */
2575                                                 spill = -val -1;
2576                                         }
2577                                         val = mono_amd64_alloc_int_reg (cfg, tmp, ins, dest_mask, hreg, reginfo [hreg].flags);
2578                                         rs->iassign [hreg] = val;
2579                                         if (spill)
2580                                                 create_spilled_store (cfg, spill, val, hreg, ins, FALSE);
2581                                 }
2582                                 DEBUG (g_print ("\tassigned hreg %s to dest R%d\n", mono_arch_regname (val), hreg));
2583                                 rs->isymbolic [val] = hreg;
2584                                 if (ins->dreg == AMD64_RAX) {
2585                                         if (val != AMD64_RDX)
2586                                                 create_copy_ins (cfg, val, AMD64_RDX, ins, FALSE);
2587                                 } else if (ins->dreg == AMD64_RDX) {
2588                                         if (val == AMD64_RAX) {
2589                                                 /* swap */
2590                                                 g_assert_not_reached ();
2591                                         } else {
2592                                                 /* two forced copies */
2593                                                 create_copy_ins (cfg, val, AMD64_RDX, ins, FALSE);
2594                                                 create_copy_ins (cfg, ins->dreg, AMD64_RAX, ins, FALSE);
2595                                         }
2596                                 } else {
2597                                         if (val == AMD64_RDX) {
2598                                                 create_copy_ins (cfg, ins->dreg, AMD64_RAX, ins, FALSE);
2599                                         } else {
2600                                                 /* two forced copies */
2601                                                 create_copy_ins (cfg, val, AMD64_RDX, ins, FALSE);
2602                                                 create_copy_ins (cfg, ins->dreg, AMD64_RAX, ins, FALSE);
2603                                         }
2604                                 }
2605                                 if (reg_is_freeable (val, FALSE) && hreg >= 0 && reginfo [hreg].born_in >= i) {
2606                                         DEBUG (g_print ("\tfreeable %s (R%d)\n", mono_arch_regname (val), hreg));
2607                                         mono_regstate_free_int (rs, val);
2608                                 }
2609                         } else if (spec [MONO_INST_DEST] == 'a' && ins->dreg != AMD64_RAX && spec [MONO_INST_CLOB] != 'd') {
2610                                 /* this instruction only outputs to EAX, need to copy */
2611                                 create_copy_ins (cfg, ins->dreg, AMD64_RAX, ins, FALSE);
2612                         } else if (spec [MONO_INST_DEST] == 'd' && ins->dreg != AMD64_RDX && spec [MONO_INST_CLOB] != 'd') {
2613                                 create_copy_ins (cfg, ins->dreg, AMD64_RDX, ins, FALSE);
2614                         }
2615                 }
2616
2617                 if (use_sse2 && spec [MONO_INST_DEST] == 'f' && reg_is_freeable (ins->dreg, TRUE) && prev_dreg >= 0 && reginfof [prev_dreg].born_in >= i) {
2618                         DEBUG (g_print ("\tfreeable %s (R%d) (born in %d)\n", mono_arch_fregname (ins->dreg), prev_dreg, reginfof [prev_dreg].born_in));
2619                         mono_regstate_free_float (rs, ins->dreg);
2620                 }
2621                 if (spec [MONO_INST_DEST] != 'f' && reg_is_freeable (ins->dreg, FALSE) && prev_dreg >= 0 && reginfo [prev_dreg].born_in >= i) {
2622                         DEBUG (g_print ("\tfreeable %s (R%d) (born in %d)\n", mono_arch_regname (ins->dreg), prev_dreg, reginfo [prev_dreg].born_in));
2623                         mono_regstate_free_int (rs, ins->dreg);
2624                 }
2625
2626                 /* put src1 in EAX if it needs to be */
2627                 if (spec [MONO_INST_SRC1] == 'a') {
2628                         if (!(rs->ifree_mask & (1 << AMD64_RAX))) {
2629                                 DEBUG (g_print ("\tforced spill of R%d\n", rs->isymbolic [AMD64_RAX]));
2630                                 get_register_force_spilling (cfg, tmp, ins, rs->isymbolic [AMD64_RAX], FALSE);
2631                                 mono_regstate_free_int (rs, AMD64_RAX);
2632                         }
2633                         if (ins->sreg1 < MONO_MAX_IREGS) {
2634                                 /* The argument is already in a hard reg, need to copy */
2635                                 MonoInst *copy = create_copy_ins (cfg, AMD64_RAX, ins->sreg1, NULL, FALSE);
2636                                 insert_before_ins (ins, tmp, copy);
2637                         }
2638                         else
2639                                 /* force-set sreg1 */
2640                                 assign_ireg (rs, ins->sreg1, AMD64_RAX);
2641                         ins->sreg1 = AMD64_RAX;
2642                 }
2643
2644                 /*
2645                  * TRACK SREG1
2646                  */
2647                 if (spec [MONO_INST_SRC1] == 'f') {
2648                         if (use_sse2) {
2649                                 if (reg_is_soft (ins->sreg1, TRUE)) {
2650                                         val = rs->fassign [ins->sreg1];
2651                                         prev_sreg1 = ins->sreg1;
2652                                         if (val < 0) {
2653                                                 int spill = 0;
2654                                                 if (val < -1) {
2655                                                         /* the register gets spilled after this inst */
2656                                                         spill = -val -1;
2657                                                 }
2658                                                 val = mono_amd64_alloc_float_reg (cfg, tmp, ins, AMD64_CALLEE_FREGS, ins->sreg1);
2659                                                 rs->fassign [ins->sreg1] = val;
2660                                                 DEBUG (g_print ("\tassigned sreg1 %s to R%d\n", mono_arch_fregname (val), ins->sreg1));
2661                                                 if (spill) {
2662                                                         MonoInst *store = create_spilled_store (cfg, spill, val, prev_sreg1, NULL, TRUE);
2663                                                         insert_before_ins (ins, tmp, store);
2664                                                 }
2665                                         }
2666                                         rs->fsymbolic [val] = prev_sreg1;
2667                                         ins->sreg1 = val;
2668                                 } else {
2669                                         prev_sreg1 = -1;
2670                                 }
2671                         }
2672                         else
2673                                 if (reginfof [ins->sreg1].flags & MONO_X86_FP_NEEDS_LOAD) {
2674                                 MonoInst *load;
2675                                 MonoInst *store = NULL;
2676
2677                                 if (reginfof [ins->sreg1].flags & MONO_X86_FP_NEEDS_LOAD_SPILL) {
2678                                         GList *spill_node;
2679                                         spill_node = g_list_first (fspill_list);
2680                                         g_assert (spill_node);
2681
2682                                         store = create_spilled_store_float (cfg, GPOINTER_TO_INT (spill_node->data), ins->sreg1, ins);
2683                                         fspill_list = g_list_remove (fspill_list, spill_node->data);
2684                                 }
2685
2686                                 fspill++;
2687                                 fspill_list = g_list_prepend (fspill_list, GINT_TO_POINTER(fspill));
2688                                 load = create_spilled_load_float (cfg, fspill, ins->sreg1, ins);
2689                                 insert_before_ins (ins, tmp, load);
2690                                 if (store)
2691                                         insert_before_ins (load, tmp, store);
2692                         }
2693                 } else if ((spec [MONO_INST_DEST] == 'L') && (spec [MONO_INST_SRC1] == 'L')) {
2694                         /* force source to be same as dest */
2695                         rs->iassign [ins->sreg1] = ins->dreg;
2696                         rs->iassign [ins->sreg1 + 1] = ins->unused;
2697
2698                         DEBUG (g_print ("\tassigned sreg1 (long) %s to sreg1 R%d\n", mono_arch_regname (ins->dreg), ins->sreg1));
2699                         DEBUG (g_print ("\tassigned sreg1 (long-high) %s to sreg1 R%d\n", mono_arch_regname (ins->unused), ins->sreg1 + 1));
2700
2701                         ins->sreg1 = ins->dreg;
2702                         /*
2703                          * No need for saving the reg, we know that src1=dest in this cases
2704                          * ins->inst_c0 = ins->unused;
2705                          */
2706
2707                         /* make sure that we remove them from free mask */
2708                         rs->ifree_mask &= ~ (1 << ins->dreg);
2709                         rs->ifree_mask &= ~ (1 << ins->unused);
2710                 }
2711                 else if (ins->sreg1 >= MONO_MAX_IREGS) {
2712                         val = rs->iassign [ins->sreg1];
2713                         prev_sreg1 = ins->sreg1;
2714                         if (val < 0) {
2715                                 int spill = 0;
2716                                 if (val < -1) {
2717                                         /* the register gets spilled after this inst */
2718                                         spill = -val -1;
2719                                 }
2720                                 if (0 && (ins->opcode == OP_MOVE)) {
2721                                         /*
2722                                          * small optimization: the dest register is already allocated
2723                                          * but the src one is not: we can simply assign the same register
2724                                          * here and peephole will get rid of the instruction later.
2725                                          * This optimization may interfere with the clobbering handling:
2726                                          * it removes a mov operation that will be added again to handle clobbering.
2727                                          * There are also some other issues that should with make testjit.
2728                                          */
2729                                         mono_regstate_alloc_int (rs, 1 << ins->dreg);
2730                                         val = rs->iassign [ins->sreg1] = ins->dreg;
2731                                         //g_assert (val >= 0);
2732                                         DEBUG (g_print ("\tfast assigned sreg1 %s to R%d\n", mono_arch_regname (val), ins->sreg1));
2733                                 } else {
2734                                         //g_assert (val == -1); /* source cannot be spilled */
2735                                         val = mono_amd64_alloc_int_reg (cfg, tmp, ins, src1_mask, ins->sreg1, reginfo [ins->sreg1].flags);
2736                                         rs->iassign [ins->sreg1] = val;
2737                                         DEBUG (g_print ("\tassigned sreg1 %s to R%d\n", mono_arch_regname (val), ins->sreg1));
2738                                 }
2739                                 if (spill) {
2740                                         MonoInst *store = create_spilled_store (cfg, spill, val, prev_sreg1, NULL, FALSE);
2741                                         insert_before_ins (ins, tmp, store);
2742                                 }
2743                         }
2744                         rs->isymbolic [val] = prev_sreg1;
2745                         ins->sreg1 = val;
2746                 } else {
2747                         prev_sreg1 = -1;
2748                 }
2749
2750                 /* handle clobbering of sreg1 */
2751                 if (((spec [MONO_INST_DEST] == 'f' && spec [MONO_INST_SRC1] == 'f' && use_sse2) || spec [MONO_INST_CLOB] == '1' || spec [MONO_INST_CLOB] == 's') && ins->dreg != ins->sreg1) {
2752                         MonoInst *sreg2_copy = NULL;
2753                         MonoInst *copy;
2754                         gboolean fp = (spec [MONO_INST_SRC1] == 'f');
2755
2756                         if (ins->dreg == ins->sreg2) {
2757                                 /*
2758                                  * copying sreg1 to dreg could clobber sreg2, so allocate a new
2759                                  * register for it.
2760                                  */
2761                                 int reg2 = 0;
2762
2763                                 if (fp)
2764                                         reg2 = mono_amd64_alloc_float_reg (cfg, tmp, ins, AMD64_CALLEE_FREGS, ins->sreg2);
2765                                 else
2766                                         reg2 = mono_amd64_alloc_int_reg (cfg, tmp, ins, dest_mask, ins->sreg2, 0);
2767
2768                                 DEBUG (g_print ("\tneed to copy sreg2 %s to reg %s\n", mono_amd64_regname (ins->sreg2, fp), mono_amd64_regname (reg2, fp)));
2769                                 sreg2_copy = create_copy_ins (cfg, reg2, ins->sreg2, NULL, fp);
2770                                 prev_sreg2 = ins->sreg2 = reg2;
2771
2772                                 if (fp)
2773                                         mono_regstate_free_float (rs, reg2);
2774                                 else
2775                                         mono_regstate_free_int (rs, reg2);
2776                         }
2777
2778                         copy = create_copy_ins (cfg, ins->dreg, ins->sreg1, NULL, fp);
2779                         DEBUG (g_print ("\tneed to copy sreg1 %s to dreg %s\n", mono_amd64_regname (ins->sreg1, fp), mono_amd64_regname (ins->dreg, fp)));
2780                         insert_before_ins (ins, tmp, copy);
2781
2782                         if (sreg2_copy)
2783                                 insert_before_ins (copy, tmp, sreg2_copy);
2784
2785                         /*
2786                          * Need to prevent sreg2 to be allocated to sreg1, since that
2787                          * would screw up the previous copy.
2788                          */
2789                         src2_mask &= ~ (1 << ins->sreg1);
2790                         /* we set sreg1 to dest as well */
2791                         prev_sreg1 = ins->sreg1 = ins->dreg;
2792                         src2_mask &= ~ (1 << ins->dreg);
2793                 }
2794
2795                 /*
2796                  * TRACK SREG2
2797                  */
2798                 if (spec [MONO_INST_SRC2] == 'f') {
2799                         if (use_sse2) {
2800                                 if (reg_is_soft (ins->sreg2, TRUE)) {
2801                                         val = rs->fassign [ins->sreg2];
2802                                         prev_sreg2 = ins->sreg2;
2803                                         if (val < 0) {
2804                                                 int spill = 0;
2805                                                 if (val < -1) {
2806                                                         /* the register gets spilled after this inst */
2807                                                         spill = -val -1;
2808                                                 }
2809                                                 val = mono_amd64_alloc_float_reg (cfg, tmp, ins, AMD64_CALLEE_FREGS, ins->sreg2);
2810                                                 rs->fassign [ins->sreg2] = val;
2811                                                 DEBUG (g_print ("\tassigned sreg2 %s to R%d\n", mono_arch_fregname (val), ins->sreg2));
2812                                                 if (spill)
2813                                                         create_spilled_store (cfg, spill, val, prev_sreg2, ins, TRUE);
2814                                         }
2815                                         rs->fsymbolic [val] = prev_sreg2;
2816                                         ins->sreg2 = val;
2817                                 } else {
2818                                         prev_sreg2 = -1;
2819                                 }
2820                         }
2821                         else
2822                         if (reginfof [ins->sreg2].flags & MONO_X86_FP_NEEDS_LOAD) {
2823                                 MonoInst *load;
2824                                 MonoInst *store = NULL;
2825
2826                                 if (reginfof [ins->sreg2].flags & MONO_X86_FP_NEEDS_LOAD_SPILL) {
2827                                         GList *spill_node;
2828
2829                                         spill_node = g_list_first (fspill_list);
2830                                         g_assert (spill_node);
2831                                         if (spec [MONO_INST_SRC1] == 'f' && (reginfof [ins->sreg1].flags & MONO_X86_FP_NEEDS_LOAD_SPILL))
2832                                                 spill_node = g_list_next (spill_node);
2833
2834                                         store = create_spilled_store_float (cfg, GPOINTER_TO_INT (spill_node->data), ins->sreg2, ins);
2835                                         fspill_list = g_list_remove (fspill_list, spill_node->data);
2836                                 }
2837
2838                                 fspill++;
2839                                 fspill_list = g_list_prepend (fspill_list, GINT_TO_POINTER(fspill));
2840                                 load = create_spilled_load_float (cfg, fspill, ins->sreg2, ins);
2841                                 insert_before_ins (ins, tmp, load);
2842                                 if (store)
2843                                         insert_before_ins (load, tmp, store);
2844                         }
2845                 }
2846                 else if (ins->sreg2 >= MONO_MAX_IREGS) {
2847                         val = rs->iassign [ins->sreg2];
2848                         prev_sreg2 = ins->sreg2;
2849                         if (val < 0) {
2850                                 int spill = 0;
2851                                 if (val < -1) {
2852                                         /* the register gets spilled after this inst */
2853                                         spill = -val -1;
2854                                 }
2855                                 val = mono_amd64_alloc_int_reg (cfg, tmp, ins, src2_mask, ins->sreg2, reginfo [ins->sreg2].flags);
2856                                 rs->iassign [ins->sreg2] = val;
2857                                 DEBUG (g_print ("\tassigned sreg2 %s to R%d\n", mono_arch_regname (val), ins->sreg2));
2858                                 if (spill)
2859                                         create_spilled_store (cfg, spill, val, prev_sreg2, ins, FALSE);
2860                         }
2861                         rs->isymbolic [val] = prev_sreg2;
2862                         ins->sreg2 = val;
2863                         if (spec [MONO_INST_CLOB] == 's' && ins->sreg2 != AMD64_RCX) {
2864                                 DEBUG (g_print ("\tassigned sreg2 %s to R%d, but ECX is needed (R%d)\n", mono_arch_regname (val), ins->sreg2, rs->iassign [AMD64_RCX]));
2865                         }
2866                 } else {
2867                         prev_sreg2 = -1;
2868                 }
2869
2870                 if (spec [MONO_INST_CLOB] == 'c') {
2871                         int j, s;
2872                         MonoCallInst *call = (MonoCallInst*)ins;
2873                         GSList *list;
2874                         guint32 clob_mask = AMD64_CALLEE_REGS;
2875
2876                         for (j = 0; j < MONO_MAX_IREGS; ++j) {
2877                                 s = 1 << j;
2878                                 if ((clob_mask & s) && !(rs->ifree_mask & s) && j != ins->sreg1) {
2879                                         get_register_force_spilling (cfg, tmp, ins, rs->isymbolic [j], FALSE);
2880                                         mono_regstate_free_int (rs, j);
2881                                         //g_warning ("register %s busy at call site\n", mono_arch_regname (j));
2882                                 }
2883                         }
2884
2885                         if (use_sse2) {
2886                                 clob_mask = AMD64_CALLEE_FREGS;
2887
2888                                 for (j = 0; j < MONO_MAX_FREGS; ++j) {
2889                                         s = 1 << j;
2890                                         if ((clob_mask & s) && !(rs->ffree_mask & s) && j != ins->sreg1) {
2891                                                 get_register_force_spilling (cfg, tmp, ins, rs->fsymbolic [j], TRUE);
2892                                                 mono_regstate_free_float (rs, j);
2893                                                 //g_warning ("register %s busy at call site\n", mono_arch_regname (j));
2894                                         }
2895                                 }
2896                         }
2897
2898                         /*
2899                          * Assign all registers in call->out_reg_args to the proper
2900                          * argument registers.
2901                          */
2902
2903                         list = call->out_ireg_args;
2904                         if (list) {
2905                                 while (list) {
2906                                         guint64 regpair;
2907                                         int reg, hreg;
2908
2909                                         regpair = (guint64) (list->data);
2910                                         hreg = regpair >> 32;
2911                                         reg = regpair & 0xffffffff;
2912
2913                                         assign_ireg (rs, reg, hreg);
2914
2915                                         DEBUG (g_print ("\tassigned arg reg %s to R%d\n", mono_arch_regname (hreg), reg));
2916
2917                                         list = g_slist_next (list);
2918                                 }
2919                                 g_slist_free (call->out_ireg_args);
2920                         }
2921
2922                         list = call->out_freg_args;
2923                         if (list && use_sse2) {
2924                                 while (list) {
2925                                         guint64 regpair;
2926                                         int reg, hreg;
2927
2928                                         regpair = (guint64) (list->data);
2929                                         hreg = regpair >> 32;
2930                                         reg = regpair & 0xffffffff;
2931
2932                                         rs->fassign [reg] = hreg;
2933                                         rs->fsymbolic [hreg] = reg;
2934                                         rs->ffree_mask &= ~ (1 << hreg);
2935
2936                                         list = g_slist_next (list);
2937                                 }
2938                         }
2939                         if (call->out_freg_args)
2940                                 g_slist_free (call->out_freg_args);
2941                 }
2942
2943                 /*if (reg_is_freeable (ins->sreg1) && prev_sreg1 >= 0 && reginfo [prev_sreg1].born_in >= i) {
2944                         DEBUG (g_print ("freeable %s\n", mono_arch_regname (ins->sreg1)));
2945                         mono_regstate_free_int (rs, ins->sreg1);
2946                 }
2947                 if (reg_is_freeable (ins->sreg2) && prev_sreg2 >= 0 && reginfo [prev_sreg2].born_in >= i) {
2948                         DEBUG (g_print ("freeable %s\n", mono_arch_regname (ins->sreg2)));
2949                         mono_regstate_free_int (rs, ins->sreg2);
2950                 }*/
2951
2952                 DEBUG (print_ins (i, ins));
2953                 /* this may result from a insert_before call */
2954                 if (!tmp->next)
2955                         bb->code = tmp->data;
2956                 tmp = tmp->next;
2957         }
2958
2959         g_free (reginfo);
2960         g_free (reginfof);
2961         g_list_free (fspill_list);
2962 }
2963
2964 static unsigned char*
2965 emit_float_to_int (MonoCompile *cfg, guchar *code, int dreg, int sreg, int size, gboolean is_signed)
2966 {
2967         if (use_sse2) {
2968                 amd64_sse_cvttsd2si_reg_reg (code, dreg, sreg);
2969         }
2970         else {
2971                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 16);
2972                 x86_fnstcw_membase(code, AMD64_RSP, 0);
2973                 amd64_mov_reg_membase (code, dreg, AMD64_RSP, 0, 2);
2974                 amd64_alu_reg_imm (code, X86_OR, dreg, 0xc00);
2975                 amd64_mov_membase_reg (code, AMD64_RSP, 2, dreg, 2);
2976                 amd64_fldcw_membase (code, AMD64_RSP, 2);
2977                 amd64_push_reg (code, AMD64_RAX); // SP = SP - 8
2978                 amd64_fist_pop_membase (code, AMD64_RSP, 0, size == 8);
2979                 amd64_pop_reg (code, dreg);
2980                 amd64_fldcw_membase (code, AMD64_RSP, 0);
2981                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 16);
2982         }
2983
2984         if (size == 1)
2985                 amd64_widen_reg (code, dreg, dreg, is_signed, FALSE);
2986         else if (size == 2)
2987                 amd64_widen_reg (code, dreg, dreg, is_signed, TRUE);
2988         return code;
2989 }
2990
2991 static unsigned char*
2992 mono_emit_stack_alloc (guchar *code, MonoInst* tree)
2993 {
2994         int sreg = tree->sreg1;
2995         int need_touch = FALSE;
2996
2997 #if defined(PLATFORM_WIN32) || defined(MONO_ARCH_SIGSEGV_ON_ALTSTACK)
2998         if (!tree->flags & MONO_INST_INIT)
2999                 need_touch = TRUE;
3000 #endif
3001
3002         if (need_touch) {
3003                 guint8* br[5];
3004
3005                 /*
3006                  * Under Windows:
3007                  * If requested stack size is larger than one page,
3008                  * perform stack-touch operation
3009                  */
3010                 /*
3011                  * Generate stack probe code.
3012                  * Under Windows, it is necessary to allocate one page at a time,
3013                  * "touching" stack after each successful sub-allocation. This is
3014                  * because of the way stack growth is implemented - there is a
3015                  * guard page before the lowest stack page that is currently commited.
3016                  * Stack normally grows sequentially so OS traps access to the
3017                  * guard page and commits more pages when needed.
3018                  */
3019                 amd64_test_reg_imm (code, sreg, ~0xFFF);
3020                 br[0] = code; x86_branch8 (code, X86_CC_Z, 0, FALSE);
3021
3022                 br[2] = code; /* loop */
3023                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 0x1000);
3024                 amd64_test_membase_reg (code, AMD64_RSP, 0, AMD64_RSP);
3025                 amd64_alu_reg_imm (code, X86_SUB, sreg, 0x1000);
3026                 amd64_alu_reg_imm (code, X86_CMP, sreg, 0x1000);
3027                 br[3] = code; x86_branch8 (code, X86_CC_AE, 0, FALSE);
3028                 amd64_patch (br[3], br[2]);
3029                 amd64_test_reg_reg (code, sreg, sreg);
3030                 br[4] = code; x86_branch8 (code, X86_CC_Z, 0, FALSE);
3031                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, sreg);
3032
3033                 br[1] = code; x86_jump8 (code, 0);
3034
3035                 amd64_patch (br[0], code);
3036                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, sreg);
3037                 amd64_patch (br[1], code);
3038                 amd64_patch (br[4], code);
3039         }
3040         else
3041                 amd64_alu_reg_reg (code, X86_SUB, AMD64_RSP, tree->sreg1);
3042
3043         if (tree->flags & MONO_INST_INIT) {
3044                 int offset = 0;
3045                 if (tree->dreg != AMD64_RAX && sreg != AMD64_RAX) {
3046                         amd64_push_reg (code, AMD64_RAX);
3047                         offset += 8;
3048                 }
3049                 if (tree->dreg != AMD64_RCX && sreg != AMD64_RCX) {
3050                         amd64_push_reg (code, AMD64_RCX);
3051                         offset += 8;
3052                 }
3053                 if (tree->dreg != AMD64_RDI && sreg != AMD64_RDI) {
3054                         amd64_push_reg (code, AMD64_RDI);
3055                         offset += 8;
3056                 }
3057
3058                 amd64_shift_reg_imm (code, X86_SHR, sreg, 4);
3059                 if (sreg != AMD64_RCX)
3060                         amd64_mov_reg_reg (code, AMD64_RCX, sreg, 8);
3061                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
3062
3063                 amd64_lea_membase (code, AMD64_RDI, AMD64_RSP, offset);
3064                 amd64_cld (code);
3065                 amd64_prefix (code, X86_REP_PREFIX);
3066                 amd64_stosl (code);
3067
3068                 if (tree->dreg != AMD64_RDI && sreg != AMD64_RDI)
3069                         amd64_pop_reg (code, AMD64_RDI);
3070                 if (tree->dreg != AMD64_RCX && sreg != AMD64_RCX)
3071                         amd64_pop_reg (code, AMD64_RCX);
3072                 if (tree->dreg != AMD64_RAX && sreg != AMD64_RAX)
3073                         amd64_pop_reg (code, AMD64_RAX);
3074         }
3075         return code;
3076 }
3077
3078 static guint8*
3079 emit_move_return_value (MonoCompile *cfg, MonoInst *ins, guint8 *code)
3080 {
3081         CallInfo *cinfo;
3082         guint32 offset, quad;
3083
3084         /* Move return value to the target register */
3085         /* FIXME: do this in the local reg allocator */
3086         switch (ins->opcode) {
3087         case CEE_CALL:
3088         case OP_CALL_REG:
3089         case OP_CALL_MEMBASE:
3090         case OP_LCALL:
3091         case OP_LCALL_REG:
3092         case OP_LCALL_MEMBASE:
3093                 if (ins->dreg != AMD64_RAX)
3094                         amd64_mov_reg_reg (code, ins->dreg, AMD64_RAX, 8);
3095                 break;
3096         case OP_FCALL:
3097         case OP_FCALL_REG:
3098         case OP_FCALL_MEMBASE:
3099                 /* FIXME: optimize this */
3100                 offset = mono_spillvar_offset_float (cfg, 0);
3101                 if (((MonoCallInst*)ins)->signature->ret->type == MONO_TYPE_R4) {
3102                         if (use_sse2)
3103                                 amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, AMD64_XMM0);
3104                         else {
3105                                 amd64_movss_membase_reg (code, AMD64_RBP, offset, AMD64_XMM0);
3106                                 amd64_fld_membase (code, AMD64_RBP, offset, FALSE);
3107                         }
3108                 }
3109                 else {
3110                         if (use_sse2) {
3111                                 if (ins->dreg != AMD64_XMM0)
3112                                         amd64_sse_movsd_reg_reg (code, ins->dreg, AMD64_XMM0);
3113                         }
3114                         else {
3115                                 amd64_movsd_membase_reg (code, AMD64_RBP, offset, AMD64_XMM0);
3116                                 amd64_fld_membase (code, AMD64_RBP, offset, TRUE);
3117                         }
3118                 }
3119                 break;
3120         case OP_VCALL:
3121         case OP_VCALL_REG:
3122         case OP_VCALL_MEMBASE:
3123                 cinfo = get_call_info (((MonoCallInst*)ins)->signature, FALSE);
3124                 if (cinfo->ret.storage == ArgValuetypeInReg) {
3125                         /* Pop the destination address from the stack */
3126                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
3127                         amd64_pop_reg (code, AMD64_RCX);
3128
3129                         for (quad = 0; quad < 2; quad ++) {
3130                                 switch (cinfo->ret.pair_storage [quad]) {
3131                                 case ArgInIReg:
3132                                         amd64_mov_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad], 8);
3133                                         break;
3134                                 case ArgInFloatSSEReg:
3135                                         amd64_movss_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad]);
3136                                         break;
3137                                 case ArgInDoubleSSEReg:
3138                                         amd64_movsd_membase_reg (code, AMD64_RCX, (quad * 8), cinfo->ret.pair_regs [quad]);
3139                                         break;
3140                                 case ArgNone:
3141                                         break;
3142                                 default:
3143                                         NOT_IMPLEMENTED;
3144                                 }
3145                         }
3146                 }
3147                 g_free (cinfo);
3148                 break;
3149         }
3150
3151         return code;
3152 }
3153
3154 /*
3155  * emit_load_volatile_arguments:
3156  *
3157  *  Load volatile arguments from the stack to the original input registers.
3158  * Required before a tail call.
3159  */
3160 static guint8*
3161 emit_load_volatile_arguments (MonoCompile *cfg, guint8 *code)
3162 {
3163         MonoMethod *method = cfg->method;
3164         MonoMethodSignature *sig;
3165         MonoInst *inst;
3166         CallInfo *cinfo;
3167         guint32 i;
3168
3169         /* FIXME: Generate intermediate code instead */
3170
3171         sig = mono_method_signature (method);
3172
3173         cinfo = get_call_info (sig, FALSE);
3174
3175         /* This is the opposite of the code in emit_prolog */
3176
3177         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
3178                 ArgInfo *ainfo = cinfo->args + i;
3179                 MonoType *arg_type;
3180                 inst = cfg->varinfo [i];
3181
3182                 if (sig->hasthis && (i == 0))
3183                         arg_type = &mono_defaults.object_class->byval_arg;
3184                 else
3185                         arg_type = sig->params [i - sig->hasthis];
3186
3187                 if (inst->opcode != OP_REGVAR) {
3188                         switch (ainfo->storage) {
3189                         case ArgInIReg: {
3190                                 guint32 size = 8;
3191
3192                                 /* FIXME: I1 etc */
3193                                 amd64_mov_reg_membase (code, ainfo->reg, inst->inst_basereg, inst->inst_offset, size);
3194                                 break;
3195                         }
3196                         case ArgInFloatSSEReg:
3197                                 amd64_movss_reg_membase (code, ainfo->reg, inst->inst_basereg, inst->inst_offset);
3198                                 break;
3199                         case ArgInDoubleSSEReg:
3200                                 amd64_movsd_reg_membase (code, ainfo->reg, inst->inst_basereg, inst->inst_offset);
3201                                 break;
3202                         default:
3203                                 break;
3204                         }
3205                 }
3206         }
3207
3208         g_free (cinfo);
3209
3210         return code;
3211 }
3212
3213 #define REAL_PRINT_REG(text,reg) \
3214 mono_assert (reg >= 0); \
3215 amd64_push_reg (code, AMD64_RAX); \
3216 amd64_push_reg (code, AMD64_RDX); \
3217 amd64_push_reg (code, AMD64_RCX); \
3218 amd64_push_reg (code, reg); \
3219 amd64_push_imm (code, reg); \
3220 amd64_push_imm (code, text " %d %p\n"); \
3221 amd64_mov_reg_imm (code, AMD64_RAX, printf); \
3222 amd64_call_reg (code, AMD64_RAX); \
3223 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 3*4); \
3224 amd64_pop_reg (code, AMD64_RCX); \
3225 amd64_pop_reg (code, AMD64_RDX); \
3226 amd64_pop_reg (code, AMD64_RAX);
3227
3228 /* benchmark and set based on cpu */
3229 #define LOOP_ALIGNMENT 8
3230 #define bb_is_loop_start(bb) ((bb)->loop_body_start && (bb)->nesting)
3231
3232 void
3233 mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
3234 {
3235         MonoInst *ins;
3236         MonoCallInst *call;
3237         guint offset;
3238         guint8 *code = cfg->native_code + cfg->code_len;
3239         MonoInst *last_ins = NULL;
3240         guint last_offset = 0;
3241         int max_len, cpos;
3242
3243         if (cfg->opt & MONO_OPT_PEEPHOLE)
3244                 peephole_pass (cfg, bb);
3245
3246         if (cfg->opt & MONO_OPT_LOOP) {
3247                 int pad, align = LOOP_ALIGNMENT;
3248                 /* set alignment depending on cpu */
3249                 if (bb_is_loop_start (bb) && (pad = (cfg->code_len & (align - 1)))) {
3250                         pad = align - pad;
3251                         /*g_print ("adding %d pad at %x to loop in %s\n", pad, cfg->code_len, cfg->method->name);*/
3252                         amd64_padding (code, pad);
3253                         cfg->code_len += pad;
3254                         bb->native_offset = cfg->code_len;
3255                 }
3256         }
3257
3258         if (cfg->verbose_level > 2)
3259                 g_print ("Basic block %d starting at offset 0x%x\n", bb->block_num, bb->native_offset);
3260
3261         cpos = bb->max_offset;
3262
3263         if (cfg->prof_options & MONO_PROFILE_COVERAGE) {
3264                 MonoProfileCoverageInfo *cov = cfg->coverage_info;
3265                 g_assert (!mono_compile_aot);
3266                 cpos += 6;
3267
3268                 cov->data [bb->dfn].cil_code = bb->cil_code;
3269                 /* this is not thread save, but good enough */
3270                 amd64_inc_mem (code, (guint64)&cov->data [bb->dfn].count);
3271         }
3272
3273         offset = code - cfg->native_code;
3274
3275         ins = bb->code;
3276         while (ins) {
3277                 offset = code - cfg->native_code;
3278
3279                 max_len = ((guint8 *)ins_spec [ins->opcode])[MONO_INST_LEN];
3280
3281                 if (offset > (cfg->code_size - max_len - 16)) {
3282                         cfg->code_size *= 2;
3283                         cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
3284                         code = cfg->native_code + offset;
3285                         mono_jit_stats.code_reallocs++;
3286                 }
3287
3288                 mono_debug_record_line_number (cfg, ins, offset);
3289
3290                 switch (ins->opcode) {
3291                 case OP_BIGMUL:
3292                         amd64_mul_reg (code, ins->sreg2, TRUE);
3293                         break;
3294                 case OP_BIGMUL_UN:
3295                         amd64_mul_reg (code, ins->sreg2, FALSE);
3296                         break;
3297                 case OP_X86_SETEQ_MEMBASE:
3298                         amd64_set_membase (code, X86_CC_EQ, ins->inst_basereg, ins->inst_offset, TRUE);
3299                         break;
3300                 case OP_STOREI1_MEMBASE_IMM:
3301                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 1);
3302                         break;
3303                 case OP_STOREI2_MEMBASE_IMM:
3304                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 2);
3305                         break;
3306                 case OP_STOREI4_MEMBASE_IMM:
3307                         amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 4);
3308                         break;
3309                 case OP_STOREI1_MEMBASE_REG:
3310                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 1);
3311                         break;
3312                 case OP_STOREI2_MEMBASE_REG:
3313                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 2);
3314                         break;
3315                 case OP_STORE_MEMBASE_REG:
3316                 case OP_STOREI8_MEMBASE_REG:
3317                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 8);
3318                         break;
3319                 case OP_STOREI4_MEMBASE_REG:
3320                         amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1, 4);
3321                         break;
3322                 case OP_STORE_MEMBASE_IMM:
3323                 case OP_STOREI8_MEMBASE_IMM:
3324                         if (amd64_is_imm32 (ins->inst_imm))
3325                                 amd64_mov_membase_imm (code, ins->inst_destbasereg, ins->inst_offset, ins->inst_imm, 8);
3326                         else {
3327                                 amd64_mov_reg_imm (code, GP_SCRATCH_REG, ins->inst_imm);
3328                                 amd64_mov_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, GP_SCRATCH_REG, 8);
3329                         }
3330                         break;
3331                 case CEE_LDIND_I:
3332                         amd64_mov_reg_mem (code, ins->dreg, (gssize)ins->inst_p0, sizeof (gpointer));
3333                         break;
3334                 case CEE_LDIND_I4:
3335                         amd64_mov_reg_mem (code, ins->dreg, (gssize)ins->inst_p0, 4);
3336                         break;
3337                 case CEE_LDIND_U4:
3338                         amd64_mov_reg_mem (code, ins->dreg, (gssize)ins->inst_p0, 4);
3339                         break;
3340                 case OP_LOADU4_MEM:
3341                         amd64_mov_reg_imm (code, ins->dreg, ins->inst_p0);
3342                         amd64_mov_reg_membase (code, ins->dreg, ins->dreg, 0, 4);
3343                         break;
3344                 case OP_LOAD_MEMBASE:
3345                 case OP_LOADI8_MEMBASE:
3346                         if (amd64_is_imm32 (ins->inst_offset)) {
3347                                 amd64_mov_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, sizeof (gpointer));
3348                         }
3349                         else {
3350                                 amd64_mov_reg_imm_size (code, GP_SCRATCH_REG, ins->inst_offset, 8);
3351                                 amd64_mov_reg_memindex_size (code, ins->dreg, ins->inst_basereg, 0, GP_SCRATCH_REG, 0, 8);
3352                         }
3353                         break;
3354                 case OP_LOADI4_MEMBASE:
3355                         amd64_movsxd_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
3356                         break;
3357                 case OP_LOADU4_MEMBASE:
3358                         amd64_mov_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, 4);
3359                         break;
3360                 case OP_LOADU1_MEMBASE:
3361                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, FALSE, FALSE);
3362                         break;
3363                 case OP_LOADI1_MEMBASE:
3364                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, TRUE, FALSE);
3365                         break;
3366                 case OP_LOADU2_MEMBASE:
3367                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, FALSE, TRUE);
3368                         break;
3369                 case OP_LOADI2_MEMBASE:
3370                         amd64_widen_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset, TRUE, TRUE);
3371                         break;
3372                 case CEE_CONV_I1:
3373                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, FALSE);
3374                         break;
3375                 case CEE_CONV_I2:
3376                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, TRUE);
3377                         break;
3378                 case CEE_CONV_U1:
3379                         amd64_widen_reg (code, ins->dreg, ins->sreg1, FALSE, FALSE);
3380                         break;
3381                 case CEE_CONV_U2:
3382                         amd64_widen_reg (code, ins->dreg, ins->sreg1, FALSE, TRUE);
3383                         break;
3384                 case CEE_CONV_U8:
3385                 case CEE_CONV_U:
3386                         /* Clean out the upper word */
3387                         amd64_mov_reg_reg_size (code, ins->dreg, ins->sreg1, 4);
3388                         break;
3389                 case CEE_CONV_I8:
3390                 case CEE_CONV_I:
3391                         amd64_movsxd_reg_reg (code, ins->dreg, ins->sreg1);
3392                         break;
3393                 case OP_COMPARE:
3394                 case OP_LCOMPARE:
3395                         amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, ins->sreg2);
3396                         break;
3397                 case OP_COMPARE_IMM:
3398                         if (!amd64_is_imm32 (ins->inst_imm)) {
3399                                 amd64_mov_reg_imm (code, AMD64_R11, ins->inst_imm);
3400                                 amd64_alu_reg_reg (code, X86_CMP, ins->sreg1, AMD64_R11);
3401                         } else {
3402                                 amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, ins->inst_imm);
3403                         }
3404                         break;
3405                 case OP_X86_COMPARE_REG_MEMBASE:
3406                         amd64_alu_reg_membase (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset);
3407                         break;
3408                 case OP_X86_TEST_NULL:
3409                         amd64_test_reg_reg_size (code, ins->sreg1, ins->sreg1, 4);
3410                         break;
3411                 case OP_AMD64_TEST_NULL:
3412                         amd64_test_reg_reg (code, ins->sreg1, ins->sreg1);
3413                         break;
3414                 case OP_X86_ADD_MEMBASE_IMM:
3415                         /* FIXME: Make a 64 version too */
3416                         amd64_alu_membase_imm_size (code, X86_ADD, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3417                         break;
3418                 case OP_X86_ADD_MEMBASE:
3419                         amd64_alu_reg_membase_size (code, X86_ADD, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3420                         break;
3421                 case OP_X86_SUB_MEMBASE_IMM:
3422                         g_assert (amd64_is_imm32 (ins->inst_imm));
3423                         amd64_alu_membase_imm_size (code, X86_SUB, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3424                         break;
3425                 case OP_X86_SUB_MEMBASE:
3426                         amd64_alu_reg_membase_size (code, X86_SUB, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3427                         break;
3428                 case OP_X86_INC_MEMBASE:
3429                         amd64_inc_membase_size (code, ins->inst_basereg, ins->inst_offset, 4);
3430                         break;
3431                 case OP_X86_INC_REG:
3432                         amd64_inc_reg_size (code, ins->dreg, 4);
3433                         break;
3434                 case OP_X86_DEC_MEMBASE:
3435                         amd64_dec_membase_size (code, ins->inst_basereg, ins->inst_offset, 4);
3436                         break;
3437                 case OP_X86_DEC_REG:
3438                         amd64_dec_reg_size (code, ins->dreg, 4);
3439                         break;
3440                 case OP_X86_MUL_MEMBASE:
3441                         amd64_imul_reg_membase_size (code, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3442                         break;
3443                 case OP_AMD64_ICOMPARE_MEMBASE_REG:
3444                         amd64_alu_membase_reg_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->sreg2, 4);
3445                         break;
3446                 case OP_AMD64_ICOMPARE_MEMBASE_IMM:
3447                         amd64_alu_membase_imm_size (code, X86_CMP, ins->inst_basereg, ins->inst_offset, ins->inst_imm, 4);
3448                         break;
3449                 case OP_AMD64_ICOMPARE_REG_MEMBASE:
3450                         amd64_alu_reg_membase_size (code, X86_CMP, ins->sreg1, ins->sreg2, ins->inst_offset, 4);
3451                         break;
3452                 case CEE_BREAK:
3453                         amd64_breakpoint (code);
3454                         break;
3455                 case OP_ADDCC:
3456                 case CEE_ADD:
3457                         amd64_alu_reg_reg (code, X86_ADD, ins->sreg1, ins->sreg2);
3458                         break;
3459                 case OP_ADC:
3460                         amd64_alu_reg_reg (code, X86_ADC, ins->sreg1, ins->sreg2);
3461                         break;
3462                 case OP_ADD_IMM:
3463                         g_assert (amd64_is_imm32 (ins->inst_imm));
3464                         amd64_alu_reg_imm (code, X86_ADD, ins->dreg, ins->inst_imm);
3465                         break;
3466                 case OP_ADC_IMM:
3467                         g_assert (amd64_is_imm32 (ins->inst_imm));
3468                         amd64_alu_reg_imm (code, X86_ADC, ins->dreg, ins->inst_imm);
3469                         break;
3470                 case OP_SUBCC:
3471                 case CEE_SUB:
3472                         amd64_alu_reg_reg (code, X86_SUB, ins->sreg1, ins->sreg2);
3473                         break;
3474                 case OP_SBB:
3475                         amd64_alu_reg_reg (code, X86_SBB, ins->sreg1, ins->sreg2);
3476                         break;
3477                 case OP_SUB_IMM:
3478                         g_assert (amd64_is_imm32 (ins->inst_imm));
3479                         amd64_alu_reg_imm (code, X86_SUB, ins->dreg, ins->inst_imm);
3480                         break;
3481                 case OP_SBB_IMM:
3482                         g_assert (amd64_is_imm32 (ins->inst_imm));
3483                         amd64_alu_reg_imm (code, X86_SBB, ins->dreg, ins->inst_imm);
3484                         break;
3485                 case CEE_AND:
3486                         amd64_alu_reg_reg (code, X86_AND, ins->sreg1, ins->sreg2);
3487                         break;
3488                 case OP_AND_IMM:
3489                         g_assert (amd64_is_imm32 (ins->inst_imm));
3490                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ins->inst_imm);
3491                         break;
3492                 case CEE_MUL:
3493                 case OP_LMUL:
3494                         amd64_imul_reg_reg (code, ins->sreg1, ins->sreg2);
3495                         break;
3496                 case OP_MUL_IMM:
3497                 case OP_LMUL_IMM:
3498                         amd64_imul_reg_reg_imm (code, ins->dreg, ins->sreg1, ins->inst_imm);
3499                         break;
3500                 case CEE_DIV:
3501                 case OP_LDIV:
3502                         amd64_cdq (code);
3503                         amd64_div_reg (code, ins->sreg2, TRUE);
3504                         break;
3505                 case CEE_DIV_UN:
3506                 case OP_LDIV_UN:
3507                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
3508                         amd64_div_reg (code, ins->sreg2, FALSE);
3509                         break;
3510                 case OP_DIV_IMM:
3511                         g_assert (amd64_is_imm32 (ins->inst_imm));
3512                         amd64_mov_reg_imm (code, ins->sreg2, ins->inst_imm);
3513                         amd64_cdq (code);
3514                         amd64_div_reg (code, ins->sreg2, TRUE);
3515                         break;
3516                 case CEE_REM:
3517                 case OP_LREM:
3518                         amd64_cdq (code);
3519                         amd64_div_reg (code, ins->sreg2, TRUE);
3520                         break;
3521                 case CEE_REM_UN:
3522                 case OP_LREM_UN:
3523                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
3524                         amd64_div_reg (code, ins->sreg2, FALSE);
3525                         break;
3526                 case OP_REM_IMM:
3527                         g_assert (amd64_is_imm32 (ins->inst_imm));
3528                         amd64_mov_reg_imm (code, ins->sreg2, ins->inst_imm);
3529                         amd64_cdq (code);
3530                         amd64_div_reg (code, ins->sreg2, TRUE);
3531                         break;
3532                 case OP_LMUL_OVF:
3533                         amd64_imul_reg_reg (code, ins->sreg1, ins->sreg2);
3534                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
3535                         break;
3536                 case CEE_OR:
3537                         amd64_alu_reg_reg (code, X86_OR, ins->sreg1, ins->sreg2);
3538                         break;
3539                 case OP_OR_IMM
3540 :                       g_assert (amd64_is_imm32 (ins->inst_imm));
3541                         amd64_alu_reg_imm (code, X86_OR, ins->sreg1, ins->inst_imm);
3542                         break;
3543                 case CEE_XOR:
3544                         amd64_alu_reg_reg (code, X86_XOR, ins->sreg1, ins->sreg2);
3545                         break;
3546                 case OP_XOR_IMM:
3547                         g_assert (amd64_is_imm32 (ins->inst_imm));
3548                         amd64_alu_reg_imm (code, X86_XOR, ins->sreg1, ins->inst_imm);
3549                         break;
3550                 case CEE_SHL:
3551                 case OP_LSHL:
3552                         g_assert (ins->sreg2 == AMD64_RCX);
3553                         amd64_shift_reg (code, X86_SHL, ins->dreg);
3554                         break;
3555                 case CEE_SHR:
3556                 case OP_LSHR:
3557                         g_assert (ins->sreg2 == AMD64_RCX);
3558                         amd64_shift_reg (code, X86_SAR, ins->dreg);
3559                         break;
3560                 case OP_SHR_IMM:
3561                         g_assert (amd64_is_imm32 (ins->inst_imm));
3562                         amd64_shift_reg_imm_size (code, X86_SAR, ins->dreg, ins->inst_imm, 4);
3563                         break;
3564                 case OP_LSHR_IMM:
3565                         g_assert (amd64_is_imm32 (ins->inst_imm));
3566                         amd64_shift_reg_imm (code, X86_SAR, ins->dreg, ins->inst_imm);
3567                         break;
3568                 case OP_SHR_UN_IMM:
3569                         g_assert (amd64_is_imm32 (ins->inst_imm));
3570                         amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, ins->inst_imm, 4);
3571                         break;
3572                 case OP_LSHR_UN_IMM:
3573                         g_assert (amd64_is_imm32 (ins->inst_imm));
3574                         amd64_shift_reg_imm (code, X86_SHR, ins->dreg, ins->inst_imm);
3575                         break;
3576                 case CEE_SHR_UN:
3577                         g_assert (ins->sreg2 == AMD64_RCX);
3578                         amd64_shift_reg_size (code, X86_SHR, ins->dreg, 4);
3579                         break;
3580                 case OP_LSHR_UN:
3581                         g_assert (ins->sreg2 == AMD64_RCX);
3582                         amd64_shift_reg (code, X86_SHR, ins->dreg);
3583                         break;
3584                 case OP_SHL_IMM:
3585                         g_assert (amd64_is_imm32 (ins->inst_imm));
3586                         amd64_shift_reg_imm_size (code, X86_SHL, ins->dreg, ins->inst_imm, 4);
3587                         break;
3588                 case OP_LSHL_IMM:
3589                         g_assert (amd64_is_imm32 (ins->inst_imm));
3590                         amd64_shift_reg_imm (code, X86_SHL, ins->dreg, ins->inst_imm);
3591                         break;
3592
3593                 case OP_IADDCC:
3594                 case OP_IADD:
3595                         amd64_alu_reg_reg_size (code, X86_ADD, ins->sreg1, ins->sreg2, 4);
3596                         break;
3597                 case OP_IADC:
3598                         amd64_alu_reg_reg_size (code, X86_ADC, ins->sreg1, ins->sreg2, 4);
3599                         break;
3600                 case OP_IADD_IMM:
3601                         amd64_alu_reg_imm_size (code, X86_ADD, ins->dreg, ins->inst_imm, 4);
3602                         break;
3603                 case OP_IADC_IMM:
3604                         amd64_alu_reg_imm_size (code, X86_ADC, ins->dreg, ins->inst_imm, 4);
3605                         break;
3606                 case OP_ISUBCC:
3607                 case OP_ISUB:
3608                         amd64_alu_reg_reg_size (code, X86_SUB, ins->sreg1, ins->sreg2, 4);
3609                         break;
3610                 case OP_ISBB:
3611                         amd64_alu_reg_reg_size (code, X86_SBB, ins->sreg1, ins->sreg2, 4);
3612                         break;
3613                 case OP_ISUB_IMM:
3614                         amd64_alu_reg_imm_size (code, X86_SUB, ins->dreg, ins->inst_imm, 4);
3615                         break;
3616                 case OP_ISBB_IMM:
3617                         amd64_alu_reg_imm_size (code, X86_SBB, ins->dreg, ins->inst_imm, 4);
3618                         break;
3619                 case OP_IAND:
3620                         amd64_alu_reg_reg_size (code, X86_AND, ins->sreg1, ins->sreg2, 4);
3621                         break;
3622                 case OP_IAND_IMM:
3623                         amd64_alu_reg_imm_size (code, X86_AND, ins->sreg1, ins->inst_imm, 4);
3624                         break;
3625                 case OP_IOR:
3626                         amd64_alu_reg_reg_size (code, X86_OR, ins->sreg1, ins->sreg2, 4);
3627                         break;
3628                 case OP_IOR_IMM:
3629                         amd64_alu_reg_imm_size (code, X86_OR, ins->sreg1, ins->inst_imm, 4);
3630                         break;
3631                 case OP_IXOR:
3632                         amd64_alu_reg_reg_size (code, X86_XOR, ins->sreg1, ins->sreg2, 4);
3633                         break;
3634                 case OP_IXOR_IMM:
3635                         amd64_alu_reg_imm_size (code, X86_XOR, ins->sreg1, ins->inst_imm, 4);
3636                         break;
3637                 case OP_INEG:
3638                         amd64_neg_reg_size (code, ins->sreg1, 4);
3639                         break;
3640                 case OP_INOT:
3641                         amd64_not_reg_size (code, ins->sreg1, 4);
3642                         break;
3643                 case OP_ISHL:
3644                         g_assert (ins->sreg2 == AMD64_RCX);
3645                         amd64_shift_reg_size (code, X86_SHL, ins->dreg, 4);
3646                         break;
3647                 case OP_ISHR:
3648                         g_assert (ins->sreg2 == AMD64_RCX);
3649                         amd64_shift_reg_size (code, X86_SAR, ins->dreg, 4);
3650                         break;
3651                 case OP_ISHR_IMM:
3652                         amd64_shift_reg_imm_size (code, X86_SAR, ins->dreg, ins->inst_imm, 4);
3653                         break;
3654                 case OP_ISHR_UN_IMM:
3655                         amd64_shift_reg_imm_size (code, X86_SHR, ins->dreg, ins->inst_imm, 4);
3656                         break;
3657                 case OP_ISHR_UN:
3658                         g_assert (ins->sreg2 == AMD64_RCX);
3659                         amd64_shift_reg_size (code, X86_SHR, ins->dreg, 4);
3660                         break;
3661                 case OP_ISHL_IMM:
3662                         amd64_shift_reg_imm_size (code, X86_SHL, ins->dreg, ins->inst_imm, 4);
3663                         break;
3664                 case OP_IMUL:
3665                         amd64_imul_reg_reg_size (code, ins->sreg1, ins->sreg2, 4);
3666                         break;
3667                 case OP_IMUL_IMM:
3668                         amd64_imul_reg_reg_imm_size (code, ins->dreg, ins->sreg1, ins->inst_imm, 4);
3669                         break;
3670                 case OP_IMUL_OVF:
3671                         amd64_imul_reg_reg_size (code, ins->sreg1, ins->sreg2, 4);
3672                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
3673                         break;
3674                 case OP_IMUL_OVF_UN:
3675                 case OP_LMUL_OVF_UN: {
3676                         /* the mul operation and the exception check should most likely be split */
3677                         int non_eax_reg, saved_eax = FALSE, saved_edx = FALSE;
3678                         int size = (ins->opcode == OP_IMUL_OVF_UN) ? 4 : 8;
3679                         /*g_assert (ins->sreg2 == X86_EAX);
3680                         g_assert (ins->dreg == X86_EAX);*/
3681                         if (ins->sreg2 == X86_EAX) {
3682                                 non_eax_reg = ins->sreg1;
3683                         } else if (ins->sreg1 == X86_EAX) {
3684                                 non_eax_reg = ins->sreg2;
3685                         } else {
3686                                 /* no need to save since we're going to store to it anyway */
3687                                 if (ins->dreg != X86_EAX) {
3688                                         saved_eax = TRUE;
3689                                         amd64_push_reg (code, X86_EAX);
3690                                 }
3691                                 amd64_mov_reg_reg (code, X86_EAX, ins->sreg1, size);
3692                                 non_eax_reg = ins->sreg2;
3693                         }
3694                         if (ins->dreg == X86_EDX) {
3695                                 if (!saved_eax) {
3696                                         saved_eax = TRUE;
3697                                         amd64_push_reg (code, X86_EAX);
3698                                 }
3699                         } else {
3700                                 saved_edx = TRUE;
3701                                 amd64_push_reg (code, X86_EDX);
3702                         }
3703                         amd64_mul_reg_size (code, non_eax_reg, FALSE, size);
3704                         /* save before the check since pop and mov don't change the flags */
3705                         if (ins->dreg != X86_EAX)
3706                                 amd64_mov_reg_reg (code, ins->dreg, X86_EAX, size);
3707                         if (saved_edx)
3708                                 amd64_pop_reg (code, X86_EDX);
3709                         if (saved_eax)
3710                                 amd64_pop_reg (code, X86_EAX);
3711                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_O, FALSE, "OverflowException");
3712                         break;
3713                 }
3714                 case OP_IDIV:
3715                         amd64_cdq_size (code, 4);
3716                         amd64_div_reg_size (code, ins->sreg2, TRUE, 4);
3717                         break;
3718                 case OP_IDIV_UN:
3719                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
3720                         amd64_div_reg_size (code, ins->sreg2, FALSE, 4);
3721                         break;
3722                 case OP_IDIV_IMM:
3723                         amd64_mov_reg_imm (code, ins->sreg2, ins->inst_imm);
3724                         amd64_cdq_size (code, 4);
3725                         amd64_div_reg_size (code, ins->sreg2, TRUE, 4);
3726                         break;
3727                 case OP_IREM:
3728                         amd64_cdq_size (code, 4);
3729                         amd64_div_reg_size (code, ins->sreg2, TRUE, 4);
3730                         break;
3731                 case OP_IREM_UN:
3732                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RDX, AMD64_RDX);
3733                         amd64_div_reg_size (code, ins->sreg2, FALSE, 4);
3734                         break;
3735                 case OP_IREM_IMM:
3736                         amd64_mov_reg_imm (code, ins->sreg2, ins->inst_imm);
3737                         amd64_cdq_size (code, 4);
3738                         amd64_div_reg_size (code, ins->sreg2, TRUE, 4);
3739                         break;
3740
3741                 case OP_ICOMPARE:
3742                         amd64_alu_reg_reg_size (code, X86_CMP, ins->sreg1, ins->sreg2, 4);
3743                         break;
3744                 case OP_ICOMPARE_IMM:
3745                         amd64_alu_reg_imm_size (code, X86_CMP, ins->sreg1, ins->inst_imm, 4);
3746                         break;
3747
3748                 case OP_IBEQ:
3749                 case OP_IBLT:
3750                 case OP_IBGT:
3751                 case OP_IBGE:
3752                 case OP_IBLE:
3753                         EMIT_COND_BRANCH (ins, opcode_to_x86_cond (ins->opcode), TRUE);
3754                         break;
3755                 case OP_IBNE_UN:
3756                 case OP_IBLT_UN:
3757                 case OP_IBGT_UN:
3758                 case OP_IBGE_UN:
3759                 case OP_IBLE_UN:
3760                         EMIT_COND_BRANCH (ins, opcode_to_x86_cond (ins->opcode), FALSE);
3761                         break;
3762                 case OP_COND_EXC_IOV:
3763                         EMIT_COND_SYSTEM_EXCEPTION (opcode_to_x86_cond (ins->opcode),
3764                                                                                 TRUE, ins->inst_p1);
3765                         break;
3766                 case OP_COND_EXC_IC:
3767                         EMIT_COND_SYSTEM_EXCEPTION (opcode_to_x86_cond (ins->opcode),
3768                                                                                 FALSE, ins->inst_p1);
3769                         break;
3770                 case CEE_NOT:
3771                         amd64_not_reg (code, ins->sreg1);
3772                         break;
3773                 case CEE_NEG:
3774                         amd64_neg_reg (code, ins->sreg1);
3775                         break;
3776                 case OP_SEXT_I1:
3777                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, FALSE);
3778                         break;
3779                 case OP_SEXT_I2:
3780                         amd64_widen_reg (code, ins->dreg, ins->sreg1, TRUE, TRUE);
3781                         break;
3782                 case OP_ICONST:
3783                 case OP_I8CONST:
3784                         if ((((guint64)ins->inst_c0) >> 32) == 0)
3785                                 amd64_mov_reg_imm_size (code, ins->dreg, ins->inst_c0, 4);
3786                         else
3787                                 amd64_mov_reg_imm_size (code, ins->dreg, ins->inst_c0, 8);
3788                         break;
3789                 case OP_AOTCONST:
3790                         mono_add_patch_info (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
3791                         amd64_mov_reg_membase (code, ins->dreg, AMD64_RIP, 0, 8);
3792                         break;
3793                 case CEE_CONV_I4:
3794                 case CEE_CONV_U4:
3795                 case OP_MOVE:
3796                 case OP_SETREG:
3797                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, sizeof (gpointer));
3798                         break;
3799                 case OP_AMD64_SET_XMMREG_R4: {
3800                         if (use_sse2) {
3801                                 amd64_sse_cvtsd2ss_reg_reg (code, ins->dreg, ins->sreg1);
3802                         }
3803                         else {
3804                                 amd64_fst_membase (code, AMD64_RSP, -8, FALSE, TRUE);
3805                                 /* ins->dreg is set to -1 by the reg allocator */
3806                                 amd64_movss_reg_membase (code, ins->unused, AMD64_RSP, -8);
3807                         }
3808                         break;
3809                 }
3810                 case OP_AMD64_SET_XMMREG_R8: {
3811                         if (use_sse2) {
3812                                 if (ins->dreg != ins->sreg1)
3813                                         amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
3814                         }
3815                         else {
3816                                 amd64_fst_membase (code, AMD64_RSP, -8, TRUE, TRUE);
3817                                 /* ins->dreg is set to -1 by the reg allocator */
3818                                 amd64_movsd_reg_membase (code, ins->unused, AMD64_RSP, -8);
3819                         }
3820                         break;
3821                 }
3822                 case CEE_JMP: {
3823                         /*
3824                          * Note: this 'frame destruction' logic is useful for tail calls, too.
3825                          * Keep in sync with the code in emit_epilog.
3826                          */
3827                         int pos = 0, i;
3828
3829                         /* FIXME: no tracing support... */
3830                         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
3831                                 code = mono_arch_instrument_epilog (cfg, mono_profiler_method_leave, code, FALSE);
3832
3833                         g_assert (!cfg->method->save_lmf);
3834
3835                         code = emit_load_volatile_arguments (cfg, code);
3836
3837                         for (i = 0; i < AMD64_NREG; ++i)
3838                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i)))
3839                                         pos -= sizeof (gpointer);
3840
3841                         if (pos)
3842                                 amd64_lea_membase (code, AMD64_RSP, AMD64_RBP, pos);
3843
3844                         /* Pop registers in reverse order */
3845                         for (i = AMD64_NREG - 1; i > 0; --i)
3846                                 if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
3847                                         amd64_pop_reg (code, i);
3848                                 }
3849
3850                         amd64_leave (code);
3851                         offset = code - cfg->native_code;
3852                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_METHOD_JUMP, ins->inst_p0);
3853                         if (mono_compile_aot)
3854                                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_RIP, 0, 8);
3855                         else
3856                                 amd64_set_reg_template (code, AMD64_R11);
3857                         amd64_jump_reg (code, AMD64_R11);
3858                         break;
3859                 }
3860                 case OP_CHECK_THIS:
3861                         /* ensure ins->sreg1 is not NULL */
3862                         amd64_alu_membase_imm (code, X86_CMP, ins->sreg1, 0, 0);
3863                         break;
3864                 case OP_ARGLIST: {
3865                         amd64_lea_membase (code, AMD64_R11, AMD64_RBP, cfg->sig_cookie);
3866                         amd64_mov_membase_reg (code, ins->sreg1, 0, AMD64_R11, 8);
3867                         break;
3868                 }
3869                 case OP_FCALL:
3870                 case OP_LCALL:
3871                 case OP_VCALL:
3872                 case OP_VOIDCALL:
3873                 case CEE_CALL:
3874                         call = (MonoCallInst*)ins;
3875                         /*
3876                          * The AMD64 ABI forces callers to know about varargs.
3877                          */
3878                         if ((call->signature->call_convention == MONO_CALL_VARARG) && (call->signature->pinvoke))
3879                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
3880                         else if ((cfg->method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) && (cfg->method->klass->image != mono_defaults.corlib)) {
3881                                 /*
3882                                  * Since the unmanaged calling convention doesn't contain a
3883                                  * 'vararg' entry, we have to treat every pinvoke call as a
3884                                  * potential vararg call.
3885                                  */
3886                                 guint32 nregs, i;
3887                                 nregs = 0;
3888                                 for (i = 0; i < AMD64_XMM_NREG; ++i)
3889                                         if (call->used_fregs & (1 << i))
3890                                                 nregs ++;
3891                                 if (!nregs)
3892                                         amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
3893                                 else
3894                                         amd64_mov_reg_imm (code, AMD64_RAX, nregs);
3895                         }
3896
3897                         if (ins->flags & MONO_INST_HAS_METHOD)
3898                                 code = emit_call (cfg, code, MONO_PATCH_INFO_METHOD, call->method);
3899                         else
3900                                 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, call->fptr);
3901                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention))
3902                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
3903                         code = emit_move_return_value (cfg, ins, code);
3904                         break;
3905                 case OP_FCALL_REG:
3906                 case OP_LCALL_REG:
3907                 case OP_VCALL_REG:
3908                 case OP_VOIDCALL_REG:
3909                 case OP_CALL_REG:
3910                         call = (MonoCallInst*)ins;
3911
3912                         if (AMD64_IS_ARGUMENT_REG (ins->sreg1)) {
3913                                 amd64_mov_reg_reg (code, AMD64_R11, ins->sreg1, 8);
3914                                 ins->sreg1 = AMD64_R11;
3915                         }
3916
3917                         /*
3918                          * The AMD64 ABI forces callers to know about varargs.
3919                          */
3920                         if ((call->signature->call_convention == MONO_CALL_VARARG) && (call->signature->pinvoke)) {
3921                                 if (ins->sreg1 == AMD64_RAX) {
3922                                         amd64_mov_reg_reg (code, AMD64_R11, AMD64_RAX, 8);
3923                                         ins->sreg1 = AMD64_R11;
3924                                 }
3925                                 amd64_alu_reg_reg (code, X86_XOR, AMD64_RAX, AMD64_RAX);
3926                         }
3927                         amd64_call_reg (code, ins->sreg1);
3928                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention))
3929                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
3930                         code = emit_move_return_value (cfg, ins, code);
3931                         break;
3932                 case OP_FCALL_MEMBASE:
3933                 case OP_LCALL_MEMBASE:
3934                 case OP_VCALL_MEMBASE:
3935                 case OP_VOIDCALL_MEMBASE:
3936                 case OP_CALL_MEMBASE:
3937                         call = (MonoCallInst*)ins;
3938
3939                         if (AMD64_IS_ARGUMENT_REG (ins->sreg1)) {
3940                                 /*
3941                                  * Can't use R11 because it is clobbered by the trampoline
3942                                  * code, and the reg value is needed by get_vcall_slot_addr.
3943                                  */
3944                                 amd64_mov_reg_reg (code, AMD64_RAX, ins->sreg1, 8);
3945                                 ins->sreg1 = AMD64_RAX;
3946                         }
3947
3948                         amd64_call_membase (code, ins->sreg1, ins->inst_offset);
3949                         if (call->stack_usage && !CALLCONV_IS_STDCALL (call->signature->call_convention))
3950                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, call->stack_usage);
3951                         code = emit_move_return_value (cfg, ins, code);
3952                         break;
3953                 case OP_OUTARG:
3954                 case OP_X86_PUSH:
3955                         amd64_push_reg (code, ins->sreg1);
3956                         break;
3957                 case OP_X86_PUSH_IMM:
3958                         g_assert (amd64_is_imm32 (ins->inst_imm));
3959                         amd64_push_imm (code, ins->inst_imm);
3960                         break;
3961                 case OP_X86_PUSH_MEMBASE:
3962                         amd64_push_membase (code, ins->inst_basereg, ins->inst_offset);
3963                         break;
3964                 case OP_X86_PUSH_OBJ:
3965                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, ins->inst_imm);
3966                         amd64_push_reg (code, AMD64_RDI);
3967                         amd64_push_reg (code, AMD64_RSI);
3968                         amd64_push_reg (code, AMD64_RCX);
3969                         if (ins->inst_offset)
3970                                 amd64_lea_membase (code, AMD64_RSI, ins->inst_basereg, ins->inst_offset);
3971                         else
3972                                 amd64_mov_reg_reg (code, AMD64_RSI, ins->inst_basereg, 8);
3973                         amd64_lea_membase (code, AMD64_RDI, AMD64_RSP, 3 * 8);
3974                         amd64_mov_reg_imm (code, AMD64_RCX, (ins->inst_imm >> 3));
3975                         amd64_cld (code);
3976                         amd64_prefix (code, X86_REP_PREFIX);
3977                         amd64_movsd (code);
3978                         amd64_pop_reg (code, AMD64_RCX);
3979                         amd64_pop_reg (code, AMD64_RSI);
3980                         amd64_pop_reg (code, AMD64_RDI);
3981                         break;
3982                 case OP_X86_LEA:
3983                         amd64_lea_memindex (code, ins->dreg, ins->sreg1, ins->inst_imm, ins->sreg2, ins->unused);
3984                         break;
3985                 case OP_X86_LEA_MEMBASE:
3986                         amd64_lea_membase (code, ins->dreg, ins->sreg1, ins->inst_imm);
3987                         break;
3988                 case OP_X86_XCHG:
3989                         amd64_xchg_reg_reg (code, ins->sreg1, ins->sreg2, 4);
3990                         break;
3991                 case OP_LOCALLOC:
3992                         /* keep alignment */
3993                         amd64_alu_reg_imm (code, X86_ADD, ins->sreg1, MONO_ARCH_FRAME_ALIGNMENT - 1);
3994                         amd64_alu_reg_imm (code, X86_AND, ins->sreg1, ~(MONO_ARCH_FRAME_ALIGNMENT - 1));
3995                         code = mono_emit_stack_alloc (code, ins);
3996                         amd64_mov_reg_reg (code, ins->dreg, AMD64_RSP, 8);
3997                         break;
3998                 case CEE_RET:
3999                         amd64_ret (code);
4000                         break;
4001                 case CEE_THROW: {
4002                         amd64_mov_reg_reg (code, AMD64_RDI, ins->sreg1, 8);
4003                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4004                                              (gpointer)"mono_arch_throw_exception");
4005                         break;
4006                 }
4007                 case OP_RETHROW: {
4008                         amd64_mov_reg_reg (code, AMD64_RDI, ins->sreg1, 8);
4009                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4010                                              (gpointer)"mono_arch_rethrow_exception");
4011                         break;
4012                 }
4013                 case OP_CALL_HANDLER:
4014                         /* Align stack */
4015                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
4016                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_BB, ins->inst_target_bb);
4017                         amd64_call_imm (code, 0);
4018                         /* Restore stack alignment */
4019                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
4020                         break;
4021                 case OP_LABEL:
4022                         ins->inst_c0 = code - cfg->native_code;
4023                         break;
4024                 case CEE_BR:
4025                         //g_print ("target: %p, next: %p, curr: %p, last: %p\n", ins->inst_target_bb, bb->next_bb, ins, bb->last_ins);
4026                         //if ((ins->inst_target_bb == bb->next_bb) && ins == bb->last_ins)
4027                         //break;
4028                         if (ins->flags & MONO_INST_BRLABEL) {
4029                                 if (ins->inst_i0->inst_c0) {
4030                                         amd64_jump_code (code, cfg->native_code + ins->inst_i0->inst_c0);
4031                                 } else {
4032                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_LABEL, ins->inst_i0);
4033                                         if ((cfg->opt & MONO_OPT_BRANCH) &&
4034                                             x86_is_imm8 (ins->inst_i0->inst_c1 - cpos))
4035                                                 x86_jump8 (code, 0);
4036                                         else
4037                                                 x86_jump32 (code, 0);
4038                                 }
4039                         } else {
4040                                 if (ins->inst_target_bb->native_offset) {
4041                                         amd64_jump_code (code, cfg->native_code + ins->inst_target_bb->native_offset);
4042                                 } else {
4043                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb);
4044                                         if ((cfg->opt & MONO_OPT_BRANCH) &&
4045                                             x86_is_imm8 (ins->inst_target_bb->max_offset - cpos))
4046                                                 x86_jump8 (code, 0);
4047                                         else
4048                                                 x86_jump32 (code, 0);
4049                                 }
4050                         }
4051                         break;
4052                 case OP_BR_REG:
4053                         amd64_jump_reg (code, ins->sreg1);
4054                         break;
4055                 case OP_CEQ:
4056                 case OP_ICEQ:
4057                         amd64_set_reg (code, X86_CC_EQ, ins->dreg, TRUE);
4058                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
4059                         break;
4060                 case OP_CLT:
4061                 case OP_ICLT:
4062                         amd64_set_reg (code, X86_CC_LT, ins->dreg, TRUE);
4063                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
4064                         break;
4065                 case OP_CLT_UN:
4066                 case OP_ICLT_UN:
4067                         amd64_set_reg (code, X86_CC_LT, ins->dreg, FALSE);
4068                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
4069                         break;
4070                 case OP_CGT:
4071                 case OP_ICGT:
4072                         amd64_set_reg (code, X86_CC_GT, ins->dreg, TRUE);
4073                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
4074                         break;
4075                 case OP_CGT_UN:
4076                 case OP_ICGT_UN:
4077                         amd64_set_reg (code, X86_CC_GT, ins->dreg, FALSE);
4078                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
4079                         break;
4080                 case OP_COND_EXC_EQ:
4081                 case OP_COND_EXC_NE_UN:
4082                 case OP_COND_EXC_LT:
4083                 case OP_COND_EXC_LT_UN:
4084                 case OP_COND_EXC_GT:
4085                 case OP_COND_EXC_GT_UN:
4086                 case OP_COND_EXC_GE:
4087                 case OP_COND_EXC_GE_UN:
4088                 case OP_COND_EXC_LE:
4089                 case OP_COND_EXC_LE_UN:
4090                 case OP_COND_EXC_OV:
4091                 case OP_COND_EXC_NO:
4092                 case OP_COND_EXC_C:
4093                 case OP_COND_EXC_NC:
4094                         EMIT_COND_SYSTEM_EXCEPTION (branch_cc_table [ins->opcode - OP_COND_EXC_EQ],
4095                                                     (ins->opcode < OP_COND_EXC_NE_UN), ins->inst_p1);
4096                         break;
4097                 case CEE_BEQ:
4098                 case CEE_BNE_UN:
4099                 case CEE_BLT:
4100                 case CEE_BLT_UN:
4101                 case CEE_BGT:
4102                 case CEE_BGT_UN:
4103                 case CEE_BGE:
4104                 case CEE_BGE_UN:
4105                 case CEE_BLE:
4106                 case CEE_BLE_UN:
4107                         EMIT_COND_BRANCH (ins, branch_cc_table [ins->opcode - CEE_BEQ], (ins->opcode < CEE_BNE_UN));
4108                         break;
4109
4110                 /* floating point opcodes */
4111                 case OP_R8CONST: {
4112                         double d = *(double *)ins->inst_p0;
4113
4114                         if (use_sse2) {
4115                                 if ((d == 0.0) && (mono_signbit (d) == 0)) {
4116                                         amd64_sse_xorpd_reg_reg (code, ins->dreg, ins->dreg);
4117                                 }
4118                                 else {
4119                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, ins->inst_p0);
4120                                         amd64_sse_movsd_reg_membase (code, ins->dreg, AMD64_RIP, 0);
4121                                 }
4122                         }
4123                         else if ((d == 0.0) && (mono_signbit (d) == 0)) {
4124                                 amd64_fldz (code);
4125                         } else if (d == 1.0) {
4126                                 x86_fld1 (code);
4127                         } else {
4128                                 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R8, ins->inst_p0);
4129                                 amd64_fld_membase (code, AMD64_RIP, 0, TRUE);
4130                         }
4131                         break;
4132                 }
4133                 case OP_R4CONST: {
4134                         float f = *(float *)ins->inst_p0;
4135
4136                         if (use_sse2) {
4137                                 if ((f == 0.0) && (mono_signbit (f) == 0)) {
4138                                         amd64_sse_xorpd_reg_reg (code, ins->dreg, ins->dreg);
4139                                 }
4140                                 else {
4141                                         mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R4, ins->inst_p0);
4142                                         amd64_sse_movss_reg_membase (code, ins->dreg, AMD64_RIP, 0);
4143                                         amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
4144                                 }
4145                         }
4146                         else if ((f == 0.0) && (mono_signbit (f) == 0)) {
4147                                 amd64_fldz (code);
4148                         } else if (f == 1.0) {
4149                                 x86_fld1 (code);
4150                         } else {
4151                                 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_R4, ins->inst_p0);
4152                                 amd64_fld_membase (code, AMD64_RIP, 0, FALSE);
4153                         }
4154                         break;
4155                 }
4156                 case OP_STORER8_MEMBASE_REG:
4157                         if (use_sse2)
4158                                 amd64_sse_movsd_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, ins->sreg1);
4159                         else
4160                                 amd64_fst_membase (code, ins->inst_destbasereg, ins->inst_offset, TRUE, TRUE);
4161                         break;
4162                 case OP_LOADR8_SPILL_MEMBASE:
4163                         if (use_sse2)
4164                                 g_assert_not_reached ();
4165                         amd64_fld_membase (code, ins->inst_basereg, ins->inst_offset, TRUE);
4166                         amd64_fxch (code, 1);
4167                         break;
4168                 case OP_LOADR8_MEMBASE:
4169                         if (use_sse2)
4170                                 amd64_sse_movsd_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
4171                         else
4172                                 amd64_fld_membase (code, ins->inst_basereg, ins->inst_offset, TRUE);
4173                         break;
4174                 case OP_STORER4_MEMBASE_REG:
4175                         if (use_sse2) {
4176                                 /* This requires a double->single conversion */
4177                                 amd64_sse_cvtsd2ss_reg_reg (code, AMD64_XMM15, ins->sreg1);
4178                                 amd64_sse_movss_membase_reg (code, ins->inst_destbasereg, ins->inst_offset, AMD64_XMM15);
4179                         }
4180                         else
4181                                 amd64_fst_membase (code, ins->inst_destbasereg, ins->inst_offset, FALSE, TRUE);
4182                         break;
4183                 case OP_LOADR4_MEMBASE:
4184                         if (use_sse2) {
4185                                 amd64_sse_movss_reg_membase (code, ins->dreg, ins->inst_basereg, ins->inst_offset);
4186                                 amd64_sse_cvtss2sd_reg_reg (code, ins->dreg, ins->dreg);
4187                         }
4188                         else
4189                                 amd64_fld_membase (code, ins->inst_basereg, ins->inst_offset, FALSE);
4190                         break;
4191                 case CEE_CONV_R4: /* FIXME: change precision */
4192                 case CEE_CONV_R8:
4193                         if (use_sse2)
4194                                 amd64_sse_cvtsi2sd_reg_reg_size (code, ins->dreg, ins->sreg1, 4);
4195                         else {
4196                                 amd64_push_reg (code, ins->sreg1);
4197                                 amd64_fild_membase (code, AMD64_RSP, 0, FALSE);
4198                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
4199                         }
4200                         break;
4201                 case CEE_CONV_R_UN:
4202                         /* Emulated */
4203                         g_assert_not_reached ();
4204                         break;
4205                 case OP_LCONV_TO_R4: /* FIXME: change precision */
4206                 case OP_LCONV_TO_R8:
4207                         if (use_sse2)
4208                                 amd64_sse_cvtsi2sd_reg_reg (code, ins->dreg, ins->sreg1);
4209                         else {
4210                                 amd64_push_reg (code, ins->sreg1);
4211                                 amd64_fild_membase (code, AMD64_RSP, 0, TRUE);
4212                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
4213                         }
4214                         break;
4215                 case OP_X86_FP_LOAD_I8:
4216                         if (use_sse2)
4217                                 g_assert_not_reached ();
4218                         amd64_fild_membase (code, ins->inst_basereg, ins->inst_offset, TRUE);
4219                         break;
4220                 case OP_X86_FP_LOAD_I4:
4221                         if (use_sse2)
4222                                 g_assert_not_reached ();
4223                         amd64_fild_membase (code, ins->inst_basereg, ins->inst_offset, FALSE);
4224                         break;
4225                 case OP_FCONV_TO_I1:
4226                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 1, TRUE);
4227                         break;
4228                 case OP_FCONV_TO_U1:
4229                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 1, FALSE);
4230                         break;
4231                 case OP_FCONV_TO_I2:
4232                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 2, TRUE);
4233                         break;
4234                 case OP_FCONV_TO_U2:
4235                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 2, FALSE);
4236                         break;
4237                 case OP_FCONV_TO_I4:
4238                 case OP_FCONV_TO_I:
4239                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 4, TRUE);
4240                         break;
4241                 case OP_FCONV_TO_I8:
4242                         code = emit_float_to_int (cfg, code, ins->dreg, ins->sreg1, 8, TRUE);
4243                         break;
4244                 case OP_LCONV_TO_R_UN: {
4245                         static guint8 mn[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x40 };
4246                         guint8 *br;
4247
4248                         if (use_sse2)
4249                                 g_assert_not_reached ();
4250
4251                         /* load 64bit integer to FP stack */
4252                         amd64_push_imm (code, 0);
4253                         amd64_push_reg (code, ins->sreg2);
4254                         amd64_push_reg (code, ins->sreg1);
4255                         amd64_fild_membase (code, AMD64_RSP, 0, TRUE);
4256                         /* store as 80bit FP value */
4257                         x86_fst80_membase (code, AMD64_RSP, 0);
4258
4259                         /* test if lreg is negative */
4260                         amd64_test_reg_reg (code, ins->sreg2, ins->sreg2);
4261                         br = code; x86_branch8 (code, X86_CC_GEZ, 0, TRUE);
4262
4263                         /* add correction constant mn */
4264                         x86_fld80_mem (code, mn);
4265                         x86_fld80_membase (code, AMD64_RSP, 0);
4266                         amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
4267                         x86_fst80_membase (code, AMD64_RSP, 0);
4268
4269                         amd64_patch (br, code);
4270
4271                         x86_fld80_membase (code, AMD64_RSP, 0);
4272                         amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 12);
4273
4274                         break;
4275                 }
4276                 case OP_LCONV_TO_OVF_I: {
4277                         guint8 *br [3], *label [1];
4278
4279                         if (use_sse2)
4280                                 g_assert_not_reached ();
4281
4282                         /*
4283                          * Valid ints: 0xffffffff:8000000 to 00000000:0x7f000000
4284                          */
4285                         amd64_test_reg_reg (code, ins->sreg1, ins->sreg1);
4286
4287                         /* If the low word top bit is set, see if we are negative */
4288                         br [0] = code; x86_branch8 (code, X86_CC_LT, 0, TRUE);
4289                         /* We are not negative (no top bit set, check for our top word to be zero */
4290                         amd64_test_reg_reg (code, ins->sreg2, ins->sreg2);
4291                         br [1] = code; x86_branch8 (code, X86_CC_EQ, 0, TRUE);
4292                         label [0] = code;
4293
4294                         /* throw exception */
4295                         mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_EXC, "OverflowException");
4296                         x86_jump32 (code, 0);
4297
4298                         amd64_patch (br [0], code);
4299                         /* our top bit is set, check that top word is 0xfffffff */
4300                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg2, 0xffffffff);
4301
4302                         amd64_patch (br [1], code);
4303                         /* nope, emit exception */
4304                         br [2] = code; x86_branch8 (code, X86_CC_NE, 0, TRUE);
4305                         amd64_patch (br [2], label [0]);
4306
4307                         if (ins->dreg != ins->sreg1)
4308                                 amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, 4);
4309                         break;
4310                 }
4311                 case CEE_CONV_OVF_U4:
4312                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, 0);
4313                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_LT, TRUE, "OverflowException");
4314                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, 8);
4315                         break;
4316                 case CEE_CONV_OVF_I4_UN:
4317                         amd64_alu_reg_imm (code, X86_CMP, ins->sreg1, 0x7fffffff);
4318                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_GT, FALSE, "OverflowException");
4319                         amd64_mov_reg_reg (code, ins->dreg, ins->sreg1, 8);
4320                         break;
4321                 case OP_FMOVE:
4322                         if (use_sse2 && (ins->dreg != ins->sreg1))
4323                                 amd64_sse_movsd_reg_reg (code, ins->dreg, ins->sreg1);
4324                         break;
4325                 case OP_FADD:
4326                         if (use_sse2)
4327                                 amd64_sse_addsd_reg_reg (code, ins->dreg, ins->sreg2);
4328                         else
4329                                 amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
4330                         break;
4331                 case OP_FSUB:
4332                         if (use_sse2)
4333                                 amd64_sse_subsd_reg_reg (code, ins->dreg, ins->sreg2);
4334                         else
4335                                 amd64_fp_op_reg (code, X86_FSUB, 1, TRUE);
4336                         break;
4337                 case OP_FMUL:
4338                         if (use_sse2)
4339                                 amd64_sse_mulsd_reg_reg (code, ins->dreg, ins->sreg2);
4340                         else
4341                                 amd64_fp_op_reg (code, X86_FMUL, 1, TRUE);
4342                         break;
4343                 case OP_FDIV:
4344                         if (use_sse2)
4345                                 amd64_sse_divsd_reg_reg (code, ins->dreg, ins->sreg2);
4346                         else
4347                                 amd64_fp_op_reg (code, X86_FDIV, 1, TRUE);
4348                         break;
4349                 case OP_FNEG:
4350                         if (use_sse2) {
4351                                 amd64_mov_reg_imm_size (code, AMD64_R11, 0x8000000000000000, 8);
4352                                 amd64_push_reg (code, AMD64_R11);
4353                                 amd64_push_reg (code, AMD64_R11);
4354                                 amd64_sse_xorpd_reg_membase (code, ins->dreg, AMD64_RSP, 0);
4355                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 16);
4356                         }
4357                         else
4358                                 amd64_fchs (code);
4359                         break;
4360                 case OP_SIN:
4361                         if (use_sse2) {
4362                                 EMIT_SSE2_FPFUNC (code, fsin, ins->dreg, ins->sreg1);
4363                         }
4364                         else {
4365                                 amd64_fsin (code);
4366                                 amd64_fldz (code);
4367                                 amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
4368                         }
4369                         break;
4370                 case OP_COS:
4371                         if (use_sse2) {
4372                                 EMIT_SSE2_FPFUNC (code, fcos, ins->dreg, ins->sreg1);
4373                         }
4374                         else {
4375                                 amd64_fcos (code);
4376                                 amd64_fldz (code);
4377                                 amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
4378                         }
4379                         break;
4380                 case OP_ABS:
4381                         if (use_sse2) {
4382                                 EMIT_SSE2_FPFUNC (code, fabs, ins->dreg, ins->sreg1);
4383                         }
4384                         else
4385                                 amd64_fabs (code);
4386                         break;
4387                 case OP_TAN: {
4388                         /*
4389                          * it really doesn't make sense to inline all this code,
4390                          * it's here just to show that things may not be as simple
4391                          * as they appear.
4392                          */
4393                         guchar *check_pos, *end_tan, *pop_jump;
4394                         if (use_sse2)
4395                                 g_assert_not_reached ();
4396                         amd64_push_reg (code, AMD64_RAX);
4397                         amd64_fptan (code);
4398                         amd64_fnstsw (code);
4399                         amd64_test_reg_imm (code, AMD64_RAX, X86_FP_C2);
4400                         check_pos = code;
4401                         x86_branch8 (code, X86_CC_NE, 0, FALSE);
4402                         amd64_fstp (code, 0); /* pop the 1.0 */
4403                         end_tan = code;
4404                         x86_jump8 (code, 0);
4405                         amd64_fldpi (code);
4406                         amd64_fp_op (code, X86_FADD, 0);
4407                         amd64_fxch (code, 1);
4408                         x86_fprem1 (code);
4409                         amd64_fstsw (code);
4410                         amd64_test_reg_imm (code, AMD64_RAX, X86_FP_C2);
4411                         pop_jump = code;
4412                         x86_branch8 (code, X86_CC_NE, 0, FALSE);
4413                         amd64_fstp (code, 1);
4414                         amd64_fptan (code);
4415                         amd64_patch (pop_jump, code);
4416                         amd64_fstp (code, 0); /* pop the 1.0 */
4417                         amd64_patch (check_pos, code);
4418                         amd64_patch (end_tan, code);
4419                         amd64_fldz (code);
4420                         amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
4421                         amd64_pop_reg (code, AMD64_RAX);
4422                         break;
4423                 }
4424                 case OP_ATAN:
4425                         if (use_sse2)
4426                                 g_assert_not_reached ();
4427                         x86_fld1 (code);
4428                         amd64_fpatan (code);
4429                         amd64_fldz (code);
4430                         amd64_fp_op_reg (code, X86_FADD, 1, TRUE);
4431                         break;
4432                 case OP_SQRT:
4433                         if (use_sse2) {
4434                                 EMIT_SSE2_FPFUNC (code, fsqrt, ins->dreg, ins->sreg1);
4435                         }
4436                         else
4437                                 amd64_fsqrt (code);
4438                         break;
4439                 case OP_X86_FPOP:
4440                         if (!use_sse2)
4441                                 amd64_fstp (code, 0);
4442                         break;
4443                 case OP_FREM: {
4444                         guint8 *l1, *l2;
4445
4446                         if (use_sse2)
4447                                 g_assert_not_reached ();
4448                         amd64_push_reg (code, AMD64_RAX);
4449                         /* we need to exchange ST(0) with ST(1) */
4450                         amd64_fxch (code, 1);
4451
4452                         /* this requires a loop, because fprem somtimes
4453                          * returns a partial remainder */
4454                         l1 = code;
4455                         /* looks like MS is using fprem instead of the IEEE compatible fprem1 */
4456                         /* x86_fprem1 (code); */
4457                         amd64_fprem (code);
4458                         amd64_fnstsw (code);
4459                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_C2);
4460                         l2 = code + 2;
4461                         x86_branch8 (code, X86_CC_NE, l1 - l2, FALSE);
4462
4463                         /* pop result */
4464                         amd64_fstp (code, 1);
4465
4466                         amd64_pop_reg (code, AMD64_RAX);
4467                         break;
4468                 }
4469                 case OP_FCOMPARE:
4470                         if (use_sse2) {
4471                                 /*
4472                                  * The two arguments are swapped because the fbranch instructions
4473                                  * depend on this for the non-sse case to work.
4474                                  */
4475                                 amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
4476                                 break;
4477                         }
4478                         if (cfg->opt & MONO_OPT_FCMOV) {
4479                                 amd64_fcomip (code, 1);
4480                                 amd64_fstp (code, 0);
4481                                 break;
4482                         }
4483                         /* this overwrites EAX */
4484                         EMIT_FPCOMPARE(code);
4485                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_CC_MASK);
4486                         break;
4487                 case OP_FCEQ:
4488                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4489                                 /* zeroing the register at the start results in
4490                                  * shorter and faster code (we can also remove the widening op)
4491                                  */
4492                                 guchar *unordered_check;
4493                                 amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
4494
4495                                 if (use_sse2)
4496                                         amd64_sse_comisd_reg_reg (code, ins->sreg1, ins->sreg2);
4497                                 else {
4498                                         amd64_fcomip (code, 1);
4499                                         amd64_fstp (code, 0);
4500                                 }
4501                                 unordered_check = code;
4502                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
4503                                 amd64_set_reg (code, X86_CC_EQ, ins->dreg, FALSE);
4504                                 amd64_patch (unordered_check, code);
4505                                 break;
4506                         }
4507                         if (ins->dreg != AMD64_RAX)
4508                                 amd64_push_reg (code, AMD64_RAX);
4509
4510                         EMIT_FPCOMPARE(code);
4511                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_CC_MASK);
4512                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0x4000);
4513                         amd64_set_reg (code, X86_CC_EQ, ins->dreg, TRUE);
4514                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
4515
4516                         if (ins->dreg != AMD64_RAX)
4517                                 amd64_pop_reg (code, AMD64_RAX);
4518                         break;
4519                 case OP_FCLT:
4520                 case OP_FCLT_UN:
4521                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4522                                 /* zeroing the register at the start results in
4523                                  * shorter and faster code (we can also remove the widening op)
4524                                  */
4525                                 amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
4526                                 if (use_sse2)
4527                                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
4528                                 else {
4529                                         amd64_fcomip (code, 1);
4530                                         amd64_fstp (code, 0);
4531                                 }
4532                                 if (ins->opcode == OP_FCLT_UN) {
4533                                         guchar *unordered_check = code;
4534                                         guchar *jump_to_end;
4535                                         x86_branch8 (code, X86_CC_P, 0, FALSE);
4536                                         amd64_set_reg (code, X86_CC_GT, ins->dreg, FALSE);
4537                                         jump_to_end = code;
4538                                         x86_jump8 (code, 0);
4539                                         amd64_patch (unordered_check, code);
4540                                         amd64_inc_reg (code, ins->dreg);
4541                                         amd64_patch (jump_to_end, code);
4542                                 } else {
4543                                         amd64_set_reg (code, X86_CC_GT, ins->dreg, FALSE);
4544                                 }
4545                                 break;
4546                         }
4547                         if (ins->dreg != AMD64_RAX)
4548                                 amd64_push_reg (code, AMD64_RAX);
4549
4550                         EMIT_FPCOMPARE(code);
4551                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_CC_MASK);
4552                         if (ins->opcode == OP_FCLT_UN) {
4553                                 guchar *is_not_zero_check, *end_jump;
4554                                 is_not_zero_check = code;
4555                                 x86_branch8 (code, X86_CC_NZ, 0, TRUE);
4556                                 end_jump = code;
4557                                 x86_jump8 (code, 0);
4558                                 amd64_patch (is_not_zero_check, code);
4559                                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_CC_MASK);
4560
4561                                 amd64_patch (end_jump, code);
4562                         }
4563                         amd64_set_reg (code, X86_CC_EQ, ins->dreg, TRUE);
4564                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
4565
4566                         if (ins->dreg != AMD64_RAX)
4567                                 amd64_pop_reg (code, AMD64_RAX);
4568                         break;
4569                 case OP_FCGT:
4570                 case OP_FCGT_UN:
4571                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4572                                 /* zeroing the register at the start results in
4573                                  * shorter and faster code (we can also remove the widening op)
4574                                  */
4575                                 guchar *unordered_check;
4576                                 amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
4577                                 if (use_sse2)
4578                                         amd64_sse_comisd_reg_reg (code, ins->sreg2, ins->sreg1);
4579                                 else {
4580                                         amd64_fcomip (code, 1);
4581                                         amd64_fstp (code, 0);
4582                                 }
4583                                 if (ins->opcode == OP_FCGT) {
4584                                         unordered_check = code;
4585                                         x86_branch8 (code, X86_CC_P, 0, FALSE);
4586                                         amd64_set_reg (code, X86_CC_LT, ins->dreg, FALSE);
4587                                         amd64_patch (unordered_check, code);
4588                                 } else {
4589                                         amd64_set_reg (code, X86_CC_LT, ins->dreg, FALSE);
4590                                 }
4591                                 break;
4592                         }
4593                         if (ins->dreg != AMD64_RAX)
4594                                 amd64_push_reg (code, AMD64_RAX);
4595
4596                         EMIT_FPCOMPARE(code);
4597                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, X86_FP_CC_MASK);
4598                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4599                         if (ins->opcode == OP_FCGT_UN) {
4600                                 guchar *is_not_zero_check, *end_jump;
4601                                 is_not_zero_check = code;
4602                                 x86_branch8 (code, X86_CC_NZ, 0, TRUE);
4603                                 end_jump = code;
4604                                 x86_jump8 (code, 0);
4605                                 amd64_patch (is_not_zero_check, code);
4606                                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_CC_MASK);
4607
4608                                 amd64_patch (end_jump, code);
4609                         }
4610                         amd64_set_reg (code, X86_CC_EQ, ins->dreg, TRUE);
4611                         amd64_widen_reg (code, ins->dreg, ins->dreg, FALSE, FALSE);
4612
4613                         if (ins->dreg != AMD64_RAX)
4614                                 amd64_pop_reg (code, AMD64_RAX);
4615                         break;
4616                 case OP_FCLT_MEMBASE:
4617                 case OP_FCGT_MEMBASE:
4618                 case OP_FCLT_UN_MEMBASE:
4619                 case OP_FCGT_UN_MEMBASE:
4620                 case OP_FCEQ_MEMBASE: {
4621                         guchar *unordered_check, *jump_to_end;
4622                         int x86_cond;
4623                         g_assert (use_sse2);
4624
4625                         amd64_alu_reg_reg (code, X86_XOR, ins->dreg, ins->dreg);
4626                         amd64_sse_comisd_reg_membase (code, ins->sreg1, ins->sreg2, ins->inst_offset);
4627
4628                         switch (ins->opcode) {
4629                         case OP_FCEQ_MEMBASE:
4630                                 x86_cond = X86_CC_EQ;
4631                                 break;
4632                         case OP_FCLT_MEMBASE:
4633                         case OP_FCLT_UN_MEMBASE:
4634                                 x86_cond = X86_CC_LT;
4635                                 break;
4636                         case OP_FCGT_MEMBASE:
4637                         case OP_FCGT_UN_MEMBASE:
4638                                 x86_cond = X86_CC_GT;
4639                                 break;
4640                         default:
4641                                 g_assert_not_reached ();
4642                         }
4643
4644                         unordered_check = code;
4645                         x86_branch8 (code, X86_CC_P, 0, FALSE);
4646                         amd64_set_reg (code, x86_cond, ins->dreg, FALSE);
4647
4648                         switch (ins->opcode) {
4649                         case OP_FCEQ_MEMBASE:
4650                         case OP_FCLT_MEMBASE:
4651                         case OP_FCGT_MEMBASE:
4652                                 amd64_patch (unordered_check, code);
4653                                 break;
4654                         case OP_FCLT_UN_MEMBASE:
4655                         case OP_FCGT_UN_MEMBASE:
4656                                 jump_to_end = code;
4657                                 x86_jump8 (code, 0);
4658                                 amd64_patch (unordered_check, code);
4659                                 amd64_inc_reg (code, ins->dreg);
4660                                 amd64_patch (jump_to_end, code);
4661                                 break;
4662                         default:
4663                                 break;
4664                         }
4665                         break;
4666                 }
4667                 case OP_FBEQ:
4668                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4669                                 guchar *jump = code;
4670                                 x86_branch8 (code, X86_CC_P, 0, TRUE);
4671                                 EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4672                                 amd64_patch (jump, code);
4673                                 break;
4674                         }
4675                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0x4000);
4676                         EMIT_COND_BRANCH (ins, X86_CC_EQ, TRUE);
4677                         break;
4678                 case OP_FBNE_UN:
4679                         /* Branch if C013 != 100 */
4680                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4681                                 /* branch if !ZF or (PF|CF) */
4682                                 EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
4683                                 EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
4684                                 EMIT_COND_BRANCH (ins, X86_CC_B, FALSE);
4685                                 break;
4686                         }
4687                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C3);
4688                         EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
4689                         break;
4690                 case OP_FBLT:
4691                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4692                                 EMIT_COND_BRANCH (ins, X86_CC_GT, FALSE);
4693                                 break;
4694                         }
4695                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4696                         break;
4697                 case OP_FBLT_UN:
4698                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4699                                 EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
4700                                 EMIT_COND_BRANCH (ins, X86_CC_GT, FALSE);
4701                                 break;
4702                         }
4703                         if (ins->opcode == OP_FBLT_UN) {
4704                                 guchar *is_not_zero_check, *end_jump;
4705                                 is_not_zero_check = code;
4706                                 x86_branch8 (code, X86_CC_NZ, 0, TRUE);
4707                                 end_jump = code;
4708                                 x86_jump8 (code, 0);
4709                                 amd64_patch (is_not_zero_check, code);
4710                                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_CC_MASK);
4711
4712                                 amd64_patch (end_jump, code);
4713                         }
4714                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4715                         break;
4716                 case OP_FBGT:
4717                 case OP_FBGT_UN:
4718                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4719                                 EMIT_COND_BRANCH (ins, X86_CC_LT, FALSE);
4720                                 break;
4721                         }
4722                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4723                         if (ins->opcode == OP_FBGT_UN) {
4724                                 guchar *is_not_zero_check, *end_jump;
4725                                 is_not_zero_check = code;
4726                                 x86_branch8 (code, X86_CC_NZ, 0, TRUE);
4727                                 end_jump = code;
4728                                 x86_jump8 (code, 0);
4729                                 amd64_patch (is_not_zero_check, code);
4730                                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_CC_MASK);
4731
4732                                 amd64_patch (end_jump, code);
4733                         }
4734                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4735                         break;
4736                 case OP_FBGE:
4737                         /* Branch if C013 == 100 or 001 */
4738                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4739                                 guchar *br1;
4740
4741                                 /* skip branch if C1=1 */
4742                                 br1 = code;
4743                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
4744                                 /* branch if (C0 | C3) = 1 */
4745                                 EMIT_COND_BRANCH (ins, X86_CC_BE, FALSE);
4746                                 amd64_patch (br1, code);
4747                                 break;
4748                         }
4749                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4750                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4751                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C3);
4752                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4753                         break;
4754                 case OP_FBGE_UN:
4755                         /* Branch if C013 == 000 */
4756                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4757                                 EMIT_COND_BRANCH (ins, X86_CC_LE, FALSE);
4758                                 break;
4759                         }
4760                         EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
4761                         break;
4762                 case OP_FBLE:
4763                         /* Branch if C013=000 or 100 */
4764                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4765                                 guchar *br1;
4766
4767                                 /* skip branch if C1=1 */
4768                                 br1 = code;
4769                                 x86_branch8 (code, X86_CC_P, 0, FALSE);
4770                                 /* branch if C0=0 */
4771                                 EMIT_COND_BRANCH (ins, X86_CC_NB, FALSE);
4772                                 amd64_patch (br1, code);
4773                                 break;
4774                         }
4775                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, (X86_FP_C0|X86_FP_C1));
4776                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0);
4777                         EMIT_COND_BRANCH (ins, X86_CC_EQ, FALSE);
4778                         break;
4779                 case OP_FBLE_UN:
4780                         /* Branch if C013 != 001 */
4781                         if (use_sse2 || (cfg->opt & MONO_OPT_FCMOV)) {
4782                                 EMIT_COND_BRANCH (ins, X86_CC_P, FALSE);
4783                                 EMIT_COND_BRANCH (ins, X86_CC_GE, FALSE);
4784                                 break;
4785                         }
4786                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4787                         EMIT_COND_BRANCH (ins, X86_CC_NE, FALSE);
4788                         break;
4789                 case CEE_CKFINITE: {
4790                         if (use_sse2) {
4791                                 /* Transfer value to the fp stack */
4792                                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 16);
4793                                 amd64_movsd_membase_reg (code, AMD64_RSP, 0, ins->sreg1);
4794                                 amd64_fld_membase (code, AMD64_RSP, 0, TRUE);
4795                         }
4796                         amd64_push_reg (code, AMD64_RAX);
4797                         amd64_fxam (code);
4798                         amd64_fnstsw (code);
4799                         amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, 0x4100);
4800                         amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, X86_FP_C0);
4801                         amd64_pop_reg (code, AMD64_RAX);
4802                         if (use_sse2) {
4803                                 amd64_fstp (code, 0);
4804                         }
4805                         EMIT_COND_SYSTEM_EXCEPTION (X86_CC_EQ, FALSE, "ArithmeticException");
4806                         if (use_sse2)
4807                                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 16);
4808                         break;
4809                 }
4810                 case OP_TLS_GET: {
4811                         x86_prefix (code, X86_FS_PREFIX);
4812                         amd64_mov_reg_mem (code, ins->dreg, ins->inst_offset, 8);
4813                         break;
4814                 }
4815                 case OP_ATOMIC_ADD_I4:
4816                 case OP_ATOMIC_ADD_I8: {
4817                         int dreg = ins->dreg;
4818                         guint32 size = (ins->opcode == OP_ATOMIC_ADD_I4) ? 4 : 8;
4819
4820                         if (dreg == ins->inst_basereg)
4821                                 dreg = AMD64_R11;
4822
4823                         if (dreg != ins->sreg2)
4824                                 amd64_mov_reg_reg (code, ins->dreg, ins->sreg2, size);
4825
4826                         x86_prefix (code, X86_LOCK_PREFIX);
4827                         amd64_xadd_membase_reg (code, ins->inst_basereg, ins->inst_offset, dreg, size);
4828
4829                         if (dreg != ins->dreg)
4830                                 amd64_mov_reg_reg (code, ins->dreg, dreg, size);
4831
4832                         break;
4833                 }
4834                 case OP_ATOMIC_ADD_NEW_I4:
4835                 case OP_ATOMIC_ADD_NEW_I8: {
4836                         int dreg = ins->dreg;
4837                         guint32 size = (ins->opcode == OP_ATOMIC_ADD_NEW_I4) ? 4 : 8;
4838
4839                         if ((dreg == ins->sreg2) || (dreg == ins->inst_basereg))
4840                                 dreg = AMD64_R11;
4841
4842                         amd64_mov_reg_reg (code, dreg, ins->sreg2, size);
4843                         amd64_prefix (code, X86_LOCK_PREFIX);
4844                         amd64_xadd_membase_reg (code, ins->inst_basereg, ins->inst_offset, dreg, size);
4845                         /* dreg contains the old value, add with sreg2 value */
4846                         amd64_alu_reg_reg_size (code, X86_ADD, dreg, ins->sreg2, size);
4847
4848                         if (ins->dreg != dreg)
4849                                 amd64_mov_reg_reg (code, ins->dreg, dreg, size);
4850
4851                         break;
4852                 }
4853                 case OP_ATOMIC_EXCHANGE_I4:
4854                 case OP_ATOMIC_EXCHANGE_I8: {
4855                         guchar *br[2];
4856                         int sreg2 = ins->sreg2;
4857                         int breg = ins->inst_basereg;
4858                         guint32 size = (ins->opcode == OP_ATOMIC_EXCHANGE_I4) ? 4 : 8;
4859
4860                         /*
4861                          * See http://msdn.microsoft.com/msdnmag/issues/0700/Win32/ for
4862                          * an explanation of how this works.
4863                          */
4864
4865                         /* cmpxchg uses eax as comperand, need to make sure we can use it
4866                          * hack to overcome limits in x86 reg allocator
4867                          * (req: dreg == eax and sreg2 != eax and breg != eax)
4868                          */
4869                         if (ins->dreg != AMD64_RAX)
4870                                 amd64_push_reg (code, AMD64_RAX);
4871
4872                         /* We need the EAX reg for the cmpxchg */
4873                         if (ins->sreg2 == AMD64_RAX) {
4874                                 amd64_push_reg (code, AMD64_RDX);
4875                                 amd64_mov_reg_reg (code, AMD64_RDX, AMD64_RAX, size);
4876                                 sreg2 = AMD64_RDX;
4877                         }
4878
4879                         if (breg == AMD64_RAX) {
4880                                 amd64_mov_reg_reg (code, AMD64_R11, AMD64_RAX, size);
4881                                 breg = AMD64_R11;
4882                         }
4883
4884                         amd64_mov_reg_membase (code, AMD64_RAX, breg, ins->inst_offset, size);
4885
4886                         br [0] = code; amd64_prefix (code, X86_LOCK_PREFIX);
4887                         amd64_cmpxchg_membase_reg_size (code, breg, ins->inst_offset, sreg2, size);
4888                         br [1] = code; amd64_branch8 (code, X86_CC_NE, -1, FALSE);
4889                         amd64_patch (br [1], br [0]);
4890
4891                         if (ins->dreg != AMD64_RAX) {
4892                                 amd64_mov_reg_reg (code, ins->dreg, AMD64_RAX, size);
4893                                 amd64_pop_reg (code, AMD64_RAX);
4894                         }
4895
4896                         if (ins->sreg2 != sreg2)
4897                                 amd64_pop_reg (code, AMD64_RDX);
4898
4899                         break;
4900                 }
4901                 default:
4902                         g_warning ("unknown opcode %s in %s()\n", mono_inst_name (ins->opcode), __FUNCTION__);
4903                         g_assert_not_reached ();
4904                 }
4905
4906                 if ((code - cfg->native_code - offset) > max_len) {
4907                         g_warning ("wrong maximal instruction length of instruction %s (expected %d, got %ld)",
4908                                    mono_inst_name (ins->opcode), max_len, code - cfg->native_code - offset);
4909                         g_assert_not_reached ();
4910                 }
4911
4912                 cpos += max_len;
4913
4914                 last_ins = ins;
4915                 last_offset = offset;
4916
4917                 ins = ins->next;
4918         }
4919
4920         cfg->code_len = code - cfg->native_code;
4921 }
4922
4923 void
4924 mono_arch_register_lowlevel_calls (void)
4925 {
4926 }
4927
4928 void
4929 mono_arch_patch_code (MonoMethod *method, MonoDomain *domain, guint8 *code, MonoJumpInfo *ji, gboolean run_cctors)
4930 {
4931         MonoJumpInfo *patch_info;
4932
4933         for (patch_info = ji; patch_info; patch_info = patch_info->next) {
4934                 unsigned char *ip = patch_info->ip.i + code;
4935                 const unsigned char *target;
4936
4937                 target = mono_resolve_patch_target (method, domain, code, patch_info, run_cctors);
4938
4939                 if (mono_compile_aot) {
4940                         switch (patch_info->type) {
4941                         case MONO_PATCH_INFO_BB:
4942                         case MONO_PATCH_INFO_LABEL:
4943                                 break;
4944                         default: {
4945                                 /* Just to make code run at aot time work */
4946                                 const unsigned char **tmp;
4947
4948                                 mono_domain_lock (domain);
4949                                 tmp = mono_code_manager_reserve (domain->code_mp, sizeof (gpointer));
4950                                 mono_domain_unlock (domain);
4951
4952                                 *tmp = target;
4953                                 target = (const unsigned char*)(guint64)((guint8*)tmp - (guint8*)ip);
4954                                 break;
4955                         }
4956                         }
4957                 }
4958
4959                 switch (patch_info->type) {
4960                 case MONO_PATCH_INFO_NONE:
4961                         continue;
4962                 case MONO_PATCH_INFO_CLASS_INIT: {
4963                         /* Might already been changed to a nop */
4964                         guint8* ip2 = ip;
4965                         if (mono_compile_aot)
4966                                 amd64_call_membase (ip2, AMD64_RIP, 0);
4967                         else {
4968                                 amd64_call_code (ip2, 0);
4969                         }
4970                         break;
4971                 }
4972                 case MONO_PATCH_INFO_METHOD_REL:
4973                 case MONO_PATCH_INFO_R8:
4974                 case MONO_PATCH_INFO_R4:
4975                         g_assert_not_reached ();
4976                         continue;
4977                 case MONO_PATCH_INFO_BB:
4978                         break;
4979                 default:
4980                         break;
4981                 }
4982                 amd64_patch (ip, (gpointer)target);
4983         }
4984 }
4985
4986 guint8 *
4987 mono_arch_emit_prolog (MonoCompile *cfg)
4988 {
4989         MonoMethod *method = cfg->method;
4990         MonoBasicBlock *bb;
4991         MonoMethodSignature *sig;
4992         MonoInst *inst;
4993         int alloc_size, pos, max_offset, i, quad;
4994         guint8 *code;
4995         CallInfo *cinfo;
4996
4997         cfg->code_size =  MAX (((MonoMethodNormal *)method)->header->code_size * 4, 512);
4998         code = cfg->native_code = g_malloc (cfg->code_size);
4999
5000         amd64_push_reg (code, AMD64_RBP);
5001         amd64_mov_reg_reg (code, AMD64_RBP, AMD64_RSP, sizeof (gpointer));
5002
5003         /* Stack alignment check */
5004 #if 0
5005         {
5006                 amd64_mov_reg_reg (code, AMD64_RAX, AMD64_RSP, 8);
5007                 amd64_alu_reg_imm (code, X86_AND, AMD64_RAX, 0xf);
5008                 amd64_alu_reg_imm (code, X86_CMP, AMD64_RAX, 0);
5009                 x86_branch8 (code, X86_CC_EQ, 2, FALSE);
5010                 amd64_breakpoint (code);
5011         }
5012 #endif
5013
5014         alloc_size = ALIGN_TO (cfg->stack_offset, MONO_ARCH_FRAME_ALIGNMENT);
5015         pos = 0;
5016
5017         if (method->save_lmf) {
5018                 gint32 lmf_offset;
5019
5020                 pos = ALIGN_TO (pos + sizeof (MonoLMF), 16);
5021
5022                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, pos);
5023
5024                 lmf_offset = - cfg->arch.lmf_offset;
5025
5026                 /* Save ip */
5027                 amd64_lea_membase (code, AMD64_R11, AMD64_RIP, 0);
5028                 amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rip), AMD64_R11, 8);
5029                 /* Save fp */
5030                 amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, ebp), AMD64_RBP, 8);
5031                 /* Save method */
5032                 /* FIXME: add a relocation for this */
5033                 if (IS_IMM32 (cfg->method))
5034                         amd64_mov_membase_imm (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, method), (guint64)cfg->method, 8);
5035                 else {
5036                         amd64_mov_reg_imm (code, AMD64_R11, cfg->method);
5037                         amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, method), AMD64_R11, 8);
5038                 }
5039                 /* Save callee saved regs */
5040                 amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbx), AMD64_RBX, 8);
5041                 amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r12), AMD64_R12, 8);
5042                 amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r13), AMD64_R13, 8);
5043                 amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r14), AMD64_R14, 8);
5044                 amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r15), AMD64_R15, 8);
5045         } else {
5046
5047                 for (i = 0; i < AMD64_NREG; ++i)
5048                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
5049                                 amd64_push_reg (code, i);
5050                                 pos += sizeof (gpointer);
5051                         }
5052         }
5053
5054         alloc_size -= pos;
5055
5056         if (alloc_size) {
5057                 /* See mono_emit_stack_alloc */
5058 #if defined(PLATFORM_WIN32) || defined(MONO_ARCH_SIGSEGV_ON_ALTSTACK)
5059                 guint32 remaining_size = alloc_size;
5060                 while (remaining_size >= 0x1000) {
5061                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 0x1000);
5062                         amd64_test_membase_reg (code, AMD64_RSP, 0, AMD64_RSP);
5063                         remaining_size -= 0x1000;
5064                 }
5065                 if (remaining_size)
5066                         amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, remaining_size);
5067 #else
5068                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, alloc_size);
5069 #endif
5070         }
5071
5072         /* compute max_offset in order to use short forward jumps */
5073         max_offset = 0;
5074         if (cfg->opt & MONO_OPT_BRANCH) {
5075                 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
5076                         MonoInst *ins = bb->code;
5077                         bb->max_offset = max_offset;
5078
5079                         if (cfg->prof_options & MONO_PROFILE_COVERAGE)
5080                                 max_offset += 6;
5081                         /* max alignment for loops */
5082                         if ((cfg->opt & MONO_OPT_LOOP) && bb_is_loop_start (bb))
5083                                 max_offset += LOOP_ALIGNMENT;
5084
5085                         while (ins) {
5086                                 if (ins->opcode == OP_LABEL)
5087                                         ins->inst_c1 = max_offset;
5088
5089                                 max_offset += ((guint8 *)ins_spec [ins->opcode])[MONO_INST_LEN];
5090                                 ins = ins->next;
5091                         }
5092                 }
5093         }
5094
5095         sig = mono_method_signature (method);
5096         pos = 0;
5097
5098         cinfo = get_call_info (sig, FALSE);
5099
5100         if (sig->ret->type != MONO_TYPE_VOID) {
5101                 if ((cinfo->ret.storage == ArgInIReg) && (cfg->ret->opcode != OP_REGVAR)) {
5102                         /* Save volatile arguments to the stack */
5103                         amd64_mov_membase_reg (code, cfg->ret->inst_basereg, cfg->ret->inst_offset, cinfo->ret.reg, 8);
5104                 }
5105         }
5106
5107         /* Keep this in sync with emit_load_volatile_arguments */
5108         for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
5109                 ArgInfo *ainfo = cinfo->args + i;
5110                 gint32 stack_offset;
5111                 MonoType *arg_type;
5112                 inst = cfg->varinfo [i];
5113
5114                 if (sig->hasthis && (i == 0))
5115                         arg_type = &mono_defaults.object_class->byval_arg;
5116                 else
5117                         arg_type = sig->params [i - sig->hasthis];
5118
5119                 stack_offset = ainfo->offset + ARGS_OFFSET;
5120
5121                 /* Save volatile arguments to the stack */
5122                 if (inst->opcode != OP_REGVAR) {
5123                         switch (ainfo->storage) {
5124                         case ArgInIReg: {
5125                                 guint32 size = 8;
5126
5127                                 /* FIXME: I1 etc */
5128                                 /*
5129                                 if (stack_offset & 0x1)
5130                                         size = 1;
5131                                 else if (stack_offset & 0x2)
5132                                         size = 2;
5133                                 else if (stack_offset & 0x4)
5134                                         size = 4;
5135                                 else
5136                                         size = 8;
5137                                 */
5138                                 amd64_mov_membase_reg (code, inst->inst_basereg, inst->inst_offset, ainfo->reg, size);
5139                                 break;
5140                         }
5141                         case ArgInFloatSSEReg:
5142                                 amd64_movss_membase_reg (code, inst->inst_basereg, inst->inst_offset, ainfo->reg);
5143                                 break;
5144                         case ArgInDoubleSSEReg:
5145                                 amd64_movsd_membase_reg (code, inst->inst_basereg, inst->inst_offset, ainfo->reg);
5146                                 break;
5147                         case ArgValuetypeInReg:
5148                                 for (quad = 0; quad < 2; quad ++) {
5149                                         switch (ainfo->pair_storage [quad]) {
5150                                         case ArgInIReg:
5151                                                 amd64_mov_membase_reg (code, inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad], sizeof (gpointer));
5152                                                 break;
5153                                         case ArgInFloatSSEReg:
5154                                                 amd64_movss_membase_reg (code, inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad]);
5155                                                 break;
5156                                         case ArgInDoubleSSEReg:
5157                                                 amd64_movsd_membase_reg (code, inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)), ainfo->pair_regs [quad]);
5158                                                 break;
5159                                         case ArgNone:
5160                                                 break;
5161                                         default:
5162                                                 g_assert_not_reached ();
5163                                         }
5164                                 }
5165                                 break;
5166                         default:
5167                                 break;
5168                         }
5169                 }
5170
5171                 if (inst->opcode == OP_REGVAR) {
5172                         /* Argument allocated to (non-volatile) register */
5173                         switch (ainfo->storage) {
5174                         case ArgInIReg:
5175                                 amd64_mov_reg_reg (code, inst->dreg, ainfo->reg, 8);
5176                                 break;
5177                         case ArgOnStack:
5178                                 amd64_mov_reg_membase (code, inst->dreg, AMD64_RBP, ARGS_OFFSET + ainfo->offset, 8);
5179                                 break;
5180                         default:
5181                                 g_assert_not_reached ();
5182                         }
5183                 }
5184         }
5185
5186         if (method->save_lmf) {
5187                 gint32 lmf_offset;
5188
5189                 if (lmf_tls_offset != -1) {
5190                         /* Load lmf quicky using the FS register */
5191                         x86_prefix (code, X86_FS_PREFIX);
5192                         amd64_mov_reg_mem (code, AMD64_RAX, lmf_tls_offset, 8);
5193                 }
5194                 else {
5195                         /*
5196                          * The call might clobber argument registers, but they are already
5197                          * saved to the stack/global regs.
5198                          */
5199
5200                         code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
5201                                                                  (gpointer)"mono_get_lmf_addr");
5202                 }
5203
5204                 lmf_offset = - cfg->arch.lmf_offset;
5205
5206                 /* Save lmf_addr */
5207                 amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, lmf_addr), AMD64_RAX, 8);
5208                 /* Save previous_lmf */
5209                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_RAX, 0, 8);
5210                 amd64_mov_membase_reg (code, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), AMD64_R11, 8);
5211                 /* Set new lmf */
5212                 amd64_lea_membase (code, AMD64_R11, AMD64_RBP, lmf_offset);
5213                 amd64_mov_membase_reg (code, AMD64_RAX, 0, AMD64_R11, 8);
5214         }
5215
5216
5217         g_free (cinfo);
5218
5219         if (mono_jit_trace_calls != NULL && mono_trace_eval (method))
5220                 code = mono_arch_instrument_prolog (cfg, mono_trace_enter_method, code, TRUE);
5221
5222         cfg->code_len = code - cfg->native_code;
5223
5224         g_assert (cfg->code_len < cfg->code_size);
5225
5226         return code;
5227 }
5228
5229 void
5230 mono_arch_emit_epilog (MonoCompile *cfg)
5231 {
5232         MonoMethod *method = cfg->method;
5233         int quad, pos, i;
5234         guint8 *code;
5235         int max_epilog_size = 16;
5236         CallInfo *cinfo;
5237
5238         if (cfg->method->save_lmf)
5239                 max_epilog_size += 256;
5240
5241         if (mono_jit_trace_calls != NULL)
5242                 max_epilog_size += 50;
5243
5244         if (cfg->prof_options & MONO_PROFILE_ENTER_LEAVE)
5245                 max_epilog_size += 50;
5246
5247         max_epilog_size += (AMD64_NREG * 2);
5248
5249         while (cfg->code_len + max_epilog_size > (cfg->code_size - 16)) {
5250                 cfg->code_size *= 2;
5251                 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
5252                 mono_jit_stats.code_reallocs++;
5253         }
5254
5255         code = cfg->native_code + cfg->code_len;
5256
5257         if (mono_jit_trace_calls != NULL && mono_trace_eval (method))
5258                 code = mono_arch_instrument_epilog (cfg, mono_trace_leave_method, code, TRUE);
5259
5260         /* the code restoring the registers must be kept in sync with CEE_JMP */
5261         pos = 0;
5262
5263         if (method->save_lmf) {
5264                 gint32 lmf_offset = - cfg->arch.lmf_offset;
5265
5266                 /* Restore previous lmf */
5267                 amd64_mov_reg_membase (code, AMD64_RCX, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, previous_lmf), 8);
5268                 amd64_mov_reg_membase (code, AMD64_R11, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, lmf_addr), 8);
5269                 amd64_mov_membase_reg (code, AMD64_R11, 0, AMD64_RCX, 8);
5270
5271                 /* Restore caller saved regs */
5272                 if (cfg->used_int_regs & (1 << AMD64_RBX)) {
5273                         amd64_mov_reg_membase (code, AMD64_RBX, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, rbx), 8);
5274                 }
5275                 if (cfg->used_int_regs & (1 << AMD64_R12)) {
5276                         amd64_mov_reg_membase (code, AMD64_R12, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r12), 8);
5277                 }
5278                 if (cfg->used_int_regs & (1 << AMD64_R13)) {
5279                         amd64_mov_reg_membase (code, AMD64_R13, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r13), 8);
5280                 }
5281                 if (cfg->used_int_regs & (1 << AMD64_R14)) {
5282                         amd64_mov_reg_membase (code, AMD64_R14, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r14), 8);
5283                 }
5284                 if (cfg->used_int_regs & (1 << AMD64_R15)) {
5285                         amd64_mov_reg_membase (code, AMD64_R15, AMD64_RBP, lmf_offset + G_STRUCT_OFFSET (MonoLMF, r15), 8);
5286                 }
5287         } else {
5288
5289                 for (i = 0; i < AMD64_NREG; ++i)
5290                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i)))
5291                                 pos -= sizeof (gpointer);
5292
5293                 if (pos) {
5294                         if (pos == - sizeof (gpointer)) {
5295                                 /* Only one register, so avoid lea */
5296                                 for (i = AMD64_NREG - 1; i > 0; --i)
5297                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
5298                                                 amd64_mov_reg_membase (code, i, AMD64_RBP, pos, 8);
5299                                         }
5300                         }
5301                         else {
5302                                 amd64_lea_membase (code, AMD64_RSP, AMD64_RBP, pos);
5303
5304                                 /* Pop registers in reverse order */
5305                                 for (i = AMD64_NREG - 1; i > 0; --i)
5306                                         if (AMD64_IS_CALLEE_SAVED_REG (i) && (cfg->used_int_regs & (1 << i))) {
5307                                                 amd64_pop_reg (code, i);
5308                                         }
5309                         }
5310                 }
5311         }
5312
5313         /* Load returned vtypes into registers if needed */
5314         cinfo = get_call_info (mono_method_signature (method), FALSE);
5315         if (cinfo->ret.storage == ArgValuetypeInReg) {
5316                 ArgInfo *ainfo = &cinfo->ret;
5317                 MonoInst *inst = cfg->ret;
5318
5319                 for (quad = 0; quad < 2; quad ++) {
5320                         switch (ainfo->pair_storage [quad]) {
5321                         case ArgInIReg:
5322                                 amd64_mov_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)), sizeof (gpointer));
5323                                 break;
5324                         case ArgInFloatSSEReg:
5325                                 amd64_movss_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)));
5326                                 break;
5327                         case ArgInDoubleSSEReg:
5328                                 amd64_movsd_reg_membase (code, ainfo->pair_regs [quad], inst->inst_basereg, inst->inst_offset + (quad * sizeof (gpointer)));
5329                                 break;
5330                         case ArgNone:
5331                                 break;
5332                         default:
5333                                 g_assert_not_reached ();
5334                         }
5335                 }
5336         }
5337         g_free (cinfo);
5338
5339         amd64_leave (code);
5340         amd64_ret (code);
5341
5342         cfg->code_len = code - cfg->native_code;
5343
5344         g_assert (cfg->code_len < cfg->code_size);
5345
5346 }
5347
5348 void
5349 mono_arch_emit_exceptions (MonoCompile *cfg)
5350 {
5351         MonoJumpInfo *patch_info;
5352         int nthrows, i;
5353         guint8 *code;
5354         MonoClass *exc_classes [16];
5355         guint8 *exc_throw_start [16], *exc_throw_end [16];
5356         guint32 code_size = 0;
5357
5358         /* Compute needed space */
5359         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
5360                 if (patch_info->type == MONO_PATCH_INFO_EXC)
5361                         code_size += 40;
5362                 if (patch_info->type == MONO_PATCH_INFO_R8)
5363                         code_size += 8 + 7; /* sizeof (double) + alignment */
5364                 if (patch_info->type == MONO_PATCH_INFO_R4)
5365                         code_size += 4 + 7; /* sizeof (float) + alignment */
5366         }
5367
5368         while (cfg->code_len + code_size > (cfg->code_size - 16)) {
5369                 cfg->code_size *= 2;
5370                 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
5371                 mono_jit_stats.code_reallocs++;
5372         }
5373
5374         code = cfg->native_code + cfg->code_len;
5375
5376         /* add code to raise exceptions */
5377         nthrows = 0;
5378         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
5379                 switch (patch_info->type) {
5380                 case MONO_PATCH_INFO_EXC: {
5381                         MonoClass *exc_class;
5382                         guint8 *buf, *buf2;
5383                         guint32 throw_ip;
5384
5385                         amd64_patch (patch_info->ip.i + cfg->native_code, code);
5386
5387                         exc_class = mono_class_from_name (mono_defaults.corlib, "System", patch_info->data.name);
5388                         g_assert (exc_class);
5389                         throw_ip = patch_info->ip.i;
5390
5391                         //x86_breakpoint (code);
5392                         /* Find a throw sequence for the same exception class */
5393                         for (i = 0; i < nthrows; ++i)
5394                                 if (exc_classes [i] == exc_class)
5395                                         break;
5396                         if (i < nthrows) {
5397                                 amd64_mov_reg_imm (code, AMD64_RSI, (exc_throw_end [i] - cfg->native_code) - throw_ip);
5398                                 x86_jump_code (code, exc_throw_start [i]);
5399                                 patch_info->type = MONO_PATCH_INFO_NONE;
5400                         }
5401                         else {
5402                                 buf = code;
5403                                 amd64_mov_reg_imm_size (code, AMD64_RSI, 0xf0f0f0f0, 4);
5404                                 buf2 = code;
5405
5406                                 if (nthrows < 16) {
5407                                         exc_classes [nthrows] = exc_class;
5408                                         exc_throw_start [nthrows] = code;
5409                                 }
5410
5411                                 amd64_mov_reg_imm (code, AMD64_RDI, exc_class->type_token);
5412                                 patch_info->data.name = "mono_arch_throw_corlib_exception";
5413                                 patch_info->type = MONO_PATCH_INFO_INTERNAL_METHOD;
5414                                 patch_info->ip.i = code - cfg->native_code;
5415
5416                                 if (mono_compile_aot) {
5417                                         amd64_mov_reg_membase (code, GP_SCRATCH_REG, AMD64_RIP, 0, 8);
5418                                         amd64_call_reg (code, GP_SCRATCH_REG);
5419                                 } else {
5420                                         /* The callee is in memory allocated using the code manager */
5421                                         amd64_call_code (code, 0);
5422                                 }
5423
5424                                 amd64_mov_reg_imm (buf, AMD64_RSI, (code - cfg->native_code) - throw_ip);
5425                                 while (buf < buf2)
5426                                         x86_nop (buf);
5427
5428                                 if (nthrows < 16) {
5429                                         exc_throw_end [nthrows] = code;
5430                                         nthrows ++;
5431                                 }
5432                         }
5433                         break;
5434                 }
5435                 default:
5436                         /* do nothing */
5437                         break;
5438                 }
5439         }
5440
5441         /* Handle relocations with RIP relative addressing */
5442         for (patch_info = cfg->patch_info; patch_info; patch_info = patch_info->next) {
5443                 gboolean remove = FALSE;
5444
5445                 switch (patch_info->type) {
5446                 case MONO_PATCH_INFO_R8: {
5447                         guint8 *pos;
5448
5449                         code = (guint8*)ALIGN_TO (code, 8);
5450
5451                         pos = cfg->native_code + patch_info->ip.i;
5452
5453                         *(double*)code = *(double*)patch_info->data.target;
5454
5455                         if (use_sse2)
5456                                 *(guint32*)(pos + 4) = (guint8*)code - pos - 8;
5457                         else
5458                                 *(guint32*)(pos + 3) = (guint8*)code - pos - 7;
5459                         code += 8;
5460
5461                         remove = TRUE;
5462                         break;
5463                 }
5464                 case MONO_PATCH_INFO_R4: {
5465                         guint8 *pos;
5466
5467                         code = (guint8*)ALIGN_TO (code, 8);
5468
5469                         pos = cfg->native_code + patch_info->ip.i;
5470
5471                         *(float*)code = *(float*)patch_info->data.target;
5472
5473                         if (use_sse2)
5474                                 *(guint32*)(pos + 4) = (guint8*)code - pos - 8;
5475                         else
5476                                 *(guint32*)(pos + 3) = (guint8*)code - pos - 7;
5477                         code += 4;
5478
5479                         remove = TRUE;
5480                         break;
5481                 }
5482                 default:
5483                         break;
5484                 }
5485
5486                 if (remove) {
5487                         if (patch_info == cfg->patch_info)
5488                                 cfg->patch_info = patch_info->next;
5489                         else {
5490                                 MonoJumpInfo *tmp;
5491
5492                                 for (tmp = cfg->patch_info; tmp->next != patch_info; tmp = tmp->next)
5493                                         ;
5494                                 tmp->next = patch_info->next;
5495                         }
5496                 }
5497         }
5498
5499         cfg->code_len = code - cfg->native_code;
5500
5501         g_assert (cfg->code_len < cfg->code_size);
5502
5503 }
5504
5505 void*
5506 mono_arch_instrument_prolog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
5507 {
5508         guchar *code = p;
5509         CallInfo *cinfo;
5510         MonoMethodSignature *sig;
5511         MonoInst *inst;
5512         int i, n, stack_area = 0;
5513
5514         /* Keep this in sync with mono_arch_get_argument_info */
5515
5516         if (enable_arguments) {
5517                 /* Allocate a new area on the stack and save arguments there */
5518                 sig = mono_method_signature (cfg->method);
5519
5520                 cinfo = get_call_info (sig, FALSE);
5521
5522                 n = sig->param_count + sig->hasthis;
5523
5524                 stack_area = ALIGN_TO (n * 8, 16);
5525
5526                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, stack_area);
5527
5528                 for (i = 0; i < n; ++i) {
5529                         inst = cfg->varinfo [i];
5530
5531                         if (inst->opcode == OP_REGVAR)
5532                                 amd64_mov_membase_reg (code, AMD64_RSP, (i * 8), inst->dreg, 8);
5533                         else {
5534                                 amd64_mov_reg_membase (code, AMD64_R11, inst->inst_basereg, inst->inst_offset, 8);
5535                                 amd64_mov_membase_reg (code, AMD64_RSP, (i * 8), AMD64_R11, 8);
5536                         }
5537                 }
5538         }
5539
5540         mono_add_patch_info (cfg, code-cfg->native_code, MONO_PATCH_INFO_METHODCONST, cfg->method);
5541         amd64_set_reg_template (code, AMD64_RDI);
5542         amd64_mov_reg_reg (code, AMD64_RSI, AMD64_RSP, 8);
5543         code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)func);
5544
5545         if (enable_arguments) {
5546                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, stack_area);
5547
5548                 g_free (cinfo);
5549         }
5550
5551         return code;
5552 }
5553
5554 enum {
5555         SAVE_NONE,
5556         SAVE_STRUCT,
5557         SAVE_EAX,
5558         SAVE_EAX_EDX,
5559         SAVE_XMM
5560 };
5561
5562 void*
5563 mono_arch_instrument_epilog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
5564 {
5565         guchar *code = p;
5566         int save_mode = SAVE_NONE;
5567         MonoMethod *method = cfg->method;
5568         int rtype = mono_type_get_underlying_type (mono_method_signature (method)->ret)->type;
5569
5570         switch (rtype) {
5571         case MONO_TYPE_VOID:
5572                 /* special case string .ctor icall */
5573                 if (strcmp (".ctor", method->name) && method->klass == mono_defaults.string_class)
5574                         save_mode = SAVE_EAX;
5575                 else
5576                         save_mode = SAVE_NONE;
5577                 break;
5578         case MONO_TYPE_I8:
5579         case MONO_TYPE_U8:
5580                 save_mode = SAVE_EAX;
5581                 break;
5582         case MONO_TYPE_R4:
5583         case MONO_TYPE_R8:
5584                 save_mode = SAVE_XMM;
5585                 break;
5586         case MONO_TYPE_VALUETYPE:
5587                 save_mode = SAVE_STRUCT;
5588                 break;
5589         default:
5590                 save_mode = SAVE_EAX;
5591                 break;
5592         }
5593
5594         /* Save the result and copy it into the proper argument register */
5595         switch (save_mode) {
5596         case SAVE_EAX:
5597                 amd64_push_reg (code, AMD64_RAX);
5598                 /* Align stack */
5599                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
5600                 if (enable_arguments)
5601                         amd64_mov_reg_reg (code, AMD64_RSI, AMD64_RAX, 8);
5602                 break;
5603         case SAVE_STRUCT:
5604                 /* FIXME: */
5605                 if (enable_arguments)
5606                         amd64_mov_reg_imm (code, AMD64_RSI, 0);
5607                 break;
5608         case SAVE_XMM:
5609                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
5610                 amd64_movsd_membase_reg (code, AMD64_RSP, 0, AMD64_XMM0);
5611                 /* Align stack */
5612                 amd64_alu_reg_imm (code, X86_SUB, AMD64_RSP, 8);
5613                 /*
5614                  * The result is already in the proper argument register so no copying
5615                  * needed.
5616                  */
5617                 break;
5618         case SAVE_NONE:
5619                 break;
5620         default:
5621                 g_assert_not_reached ();
5622         }
5623
5624         /* Set %al since this is a varargs call */
5625         if (save_mode == SAVE_XMM)
5626                 amd64_mov_reg_imm (code, AMD64_RAX, 1);
5627         else
5628                 amd64_mov_reg_imm (code, AMD64_RAX, 0);
5629
5630         mono_add_patch_info (cfg, code-cfg->native_code, MONO_PATCH_INFO_METHODCONST, method);
5631         amd64_set_reg_template (code, AMD64_RDI);
5632         code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)func);
5633
5634         /* Restore result */
5635         switch (save_mode) {
5636         case SAVE_EAX:
5637                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
5638                 amd64_pop_reg (code, AMD64_RAX);
5639                 break;
5640         case SAVE_STRUCT:
5641                 /* FIXME: */
5642                 break;
5643         case SAVE_XMM:
5644                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
5645                 amd64_movsd_reg_membase (code, AMD64_XMM0, AMD64_RSP, 0);
5646                 amd64_alu_reg_imm (code, X86_ADD, AMD64_RSP, 8);
5647                 break;
5648         case SAVE_NONE:
5649                 break;
5650         default:
5651                 g_assert_not_reached ();
5652         }
5653
5654         return code;
5655 }
5656
5657 void
5658 mono_arch_flush_icache (guint8 *code, gint size)
5659 {
5660         /* Not needed */
5661 }
5662
5663 void
5664 mono_arch_flush_register_windows (void)
5665 {
5666 }
5667
5668 gboolean
5669 mono_arch_is_inst_imm (gint64 imm)
5670 {
5671         return amd64_is_imm32 (imm);
5672 }
5673
5674 #define IS_REX(inst) (((inst) >= 0x40) && ((inst) <= 0x4f))
5675
5676 static int reg_to_ucontext_reg [] = {
5677         REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP, REG_RSI, REG_RDI,
5678         REG_R8, REG_R9, REG_R10, REG_R11, REG_R12, REG_R13, REG_R14, REG_R15,
5679         REG_RIP
5680 };
5681
5682 /*
5683  * Determine whenever the trap whose info is in SIGINFO is caused by
5684  * integer overflow.
5685  */
5686 gboolean
5687 mono_arch_is_int_overflow (void *sigctx, void *info)
5688 {
5689         ucontext_t *ctx = (ucontext_t*)sigctx;
5690         guint8* rip;
5691         int reg;
5692
5693         rip = (guint8*)ctx->uc_mcontext.gregs [REG_RIP];
5694
5695         if (IS_REX (rip [0])) {
5696                 reg = amd64_rex_b (rip [0]);
5697                 rip ++;
5698         }
5699         else
5700                 reg = 0;
5701
5702         if ((rip [0] == 0xf7) && (x86_modrm_mod (rip [1]) == 0x3) && (x86_modrm_reg (rip [1]) == 0x7)) {
5703                 /* idiv REG */
5704                 reg += x86_modrm_rm (rip [1]);
5705
5706                 if (ctx->uc_mcontext.gregs [reg_to_ucontext_reg [reg]] == -1)
5707                         return TRUE;
5708         }
5709
5710         return FALSE;
5711 }
5712
5713 guint32
5714 mono_arch_get_patch_offset (guint8 *code)
5715 {
5716         return 3;
5717 }
5718
5719 gpointer*
5720 mono_arch_get_vcall_slot_addr (guint8* code, gpointer *regs)
5721 {
5722         guint32 reg;
5723         guint32 disp;
5724         guint8 rex = 0;
5725
5726         /* go to the start of the call instruction
5727          *
5728          * address_byte = (m << 6) | (o << 3) | reg
5729          * call opcode: 0xff address_byte displacement
5730          * 0xff m=1,o=2 imm8
5731          * 0xff m=2,o=2 imm32
5732          */
5733         code -= 7;
5734
5735         /*
5736          * A given byte sequence can match more than case here, so we have to be
5737          * really careful about the ordering of the cases. Longer sequences
5738          * come first.
5739          */
5740         if ((code [0] == 0x41) && (code [1] == 0xff) && (code [2] == 0x15)) {
5741                 /* call OFFSET(%rip) */
5742                 return NULL;
5743         }
5744         else if ((code [1] == 0xff) && (amd64_modrm_reg (code [2]) == 0x2) && (amd64_modrm_mod (code [2]) == 0x2)) {
5745                 /* call *[reg+disp32] */
5746                 if (IS_REX (code [0]))
5747                         rex = code [0];
5748                 reg = amd64_modrm_rm (code [2]);
5749                 disp = *(guint32*)(code + 3);
5750                 //printf ("B: [%%r%d+0x%x]\n", reg, disp);
5751         }
5752         else if (code [2] == 0xe8) {
5753                 /* call <ADDR> */
5754                 return NULL;
5755         }
5756         else if (IS_REX (code [4]) && (code [5] == 0xff) && (amd64_modrm_reg (code [6]) == 0x2) && (amd64_modrm_mod (code [6]) == 0x3)) {
5757                 /* call *%reg */
5758                 return NULL;
5759         }
5760         else if ((code [4] == 0xff) && (amd64_modrm_reg (code [5]) == 0x2) && (amd64_modrm_mod (code [5]) == 0x1)) {
5761                 /* call *[reg+disp8] */
5762                 if (IS_REX (code [3]))
5763                         rex = code [3];
5764                 reg = amd64_modrm_rm (code [5]);
5765                 disp = *(guint8*)(code + 6);
5766                 //printf ("B: [%%r%d+0x%x]\n", reg, disp);
5767         }
5768         else if ((code [5] == 0xff) && (amd64_modrm_reg (code [6]) == 0x2) && (amd64_modrm_mod (code [6]) == 0x0)) {
5769                         /*
5770                          * This is a interface call: should check the above code can't catch it earlier
5771                          * 8b 40 30   mov    0x30(%eax),%eax
5772                          * ff 10      call   *(%eax)
5773                          */
5774                 if (IS_REX (code [4]))
5775                         rex = code [4];
5776                 reg = amd64_modrm_rm (code [6]);
5777                 disp = 0;
5778         }
5779         else
5780                 g_assert_not_reached ();
5781
5782         reg += amd64_rex_b (rex);
5783
5784         /* R11 is clobbered by the trampoline code */
5785         g_assert (reg != AMD64_R11);
5786
5787         return (gpointer)(((guint64)(regs [reg])) + disp);
5788 }
5789
5790 gpointer*
5791 mono_arch_get_delegate_method_ptr_addr (guint8* code, gpointer *regs)
5792 {
5793         guint32 reg;
5794         guint32 disp;
5795
5796         code -= 10;
5797
5798         if (IS_REX (code [0]) && (code [1] == 0x8b) && (code [3] == 0x48) && (code [4] == 0x8b) && (code [5] == 0x40) && (code [7] == 0x48) && (code [8] == 0xff) && (code [9] == 0xd0)) {
5799                 /* mov REG, %rax; mov <OFFSET>(%rax), %rax; call *%rax */
5800                 reg = amd64_rex_b (code [0]) + amd64_modrm_rm (code [2]);
5801                 disp = code [6];
5802
5803                 if (reg == AMD64_RAX)
5804                         return NULL;
5805                 else
5806                         return (gpointer*)(((guint64)(regs [reg])) + disp);
5807         }
5808
5809         return NULL;
5810 }
5811
5812 /*
5813  * Support for fast access to the thread-local lmf structure using the GS
5814  * segment register on NPTL + kernel 2.6.x.
5815  */
5816
5817 static gboolean tls_offset_inited = FALSE;
5818
5819 /* code should be simply return <tls var>; */
5820 static int
5821 read_tls_offset_from_method (void* method)
5822 {
5823         guint8 *code = (guint8*)method;
5824
5825         /*
5826          * Determine the offset of mono_lfm_addr inside the TLS structures
5827          * by disassembling the function above.
5828          */
5829         /* This is generated by gcc 3.3.2 */
5830         if ((code [0] == 0x55) && (code [1] == 0x48) && (code [2] == 0x89) &&
5831                 (code [3] == 0xe5) && (code [4] == 0x64) && (code [5] == 0x48) &&
5832                 (code [6] == 0x8b) && (code [7] == 0x04) && (code [8] == 0x25) &&
5833                 (code [9] == 0x00) && (code [10] == 0x00) && (code [11] == 0x00) &&
5834                 (code [12] == 0x0) && (code [13] == 0x48) && (code [14] == 0x8b) &&
5835                 (code [15] == 0x80)) {
5836                 return *(gint32*)&(code [16]);
5837         } else if
5838                 /* This is generated by gcc-3.3.2 with -O=2 */
5839                 /* mov fs:0, %rax ; mov <offset>(%rax), %rax ; retq */
5840                 ((code [0] == 0x64) && (code [1] == 0x48) && (code [2] == 0x8b) &&
5841                  (code [3] == 0x04) && (code [4] == 0x25) &&
5842                  (code [9] == 0x48) && (code [10] == 0x8b) && (code [11] == 0x80) &&
5843                  (code [16] == 0xc3)) {
5844                         return *(gint32*)&(code [12]);
5845         } else if
5846                 /* This is generated by gcc-3.4.1 */
5847                 ((code [0] == 0x55) && (code [1] == 0x48) && (code [2] == 0x89) &&
5848                  (code [3] == 0xe5) && (code [4] == 0x64) && (code [5] == 0x48) &&
5849                  (code [6] == 0x8b) && (code [7] == 0x04) && (code [8] == 0x25) &&
5850                  (code [13] == 0xc9) && (code [14] == 0xc3)) {
5851                         return *(gint32*)&(code [9]);
5852         } else if
5853                 /* This is generated by gcc-3.4.1 with -O=2 */
5854                 ((code [0] == 0x64) && (code [1] == 0x48) && (code [2] == 0x8b) &&
5855                  (code [3] == 0x04) && (code [4] == 0x25)) {
5856                 return *(gint32*)&(code [5]);
5857         }
5858
5859         return -1;
5860 }
5861
5862 #ifdef MONO_ARCH_SIGSEGV_ON_ALTSTACK
5863
5864 static void
5865 setup_stack (MonoJitTlsData *tls)
5866 {
5867         pthread_t self = pthread_self();
5868         pthread_attr_t attr;
5869         size_t stsize = 0;
5870         struct sigaltstack sa;
5871         guint8 *staddr = NULL;
5872         guint8 *current = (guint8*)&staddr;
5873
5874         if (mono_running_on_valgrind ())
5875                 return;
5876
5877         /* Determine stack boundaries */
5878 #ifdef HAVE_PTHREAD_GETATTR_NP
5879         pthread_getattr_np( self, &attr );
5880 #else
5881 #ifdef HAVE_PTHREAD_ATTR_GET_NP
5882         pthread_attr_get_np( self, &attr );
5883 #elif defined(sun)
5884         pthread_attr_init( &attr );
5885         pthread_attr_getstacksize( &attr, &stsize );
5886 #else
5887 #error "Not implemented"
5888 #endif
5889 #endif
5890 #ifndef sun
5891         pthread_attr_getstack( &attr, (void**)&staddr, &stsize );
5892 #endif
5893
5894         g_assert (staddr);
5895
5896         g_assert ((current > staddr) && (current < staddr + stsize));
5897
5898         tls->end_of_stack = staddr + stsize;
5899
5900         /*
5901          * threads created by nptl does not seem to have a guard page, and
5902          * since the main thread is not created by us, we can't even set one.
5903          * Increasing stsize fools the SIGSEGV signal handler into thinking this
5904          * is a stack overflow exception.
5905          */
5906         tls->stack_size = stsize + getpagesize ();
5907
5908         /* Setup an alternate signal stack */
5909         tls->signal_stack = mmap (0, SIGNAL_STACK_SIZE, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
5910         tls->signal_stack_size = SIGNAL_STACK_SIZE;
5911
5912         g_assert (tls->signal_stack);
5913
5914         sa.ss_sp = tls->signal_stack;
5915         sa.ss_size = SIGNAL_STACK_SIZE;
5916         sa.ss_flags = SS_ONSTACK;
5917         sigaltstack (&sa, NULL);
5918 }
5919
5920 #endif
5921
5922 void
5923 mono_arch_setup_jit_tls_data (MonoJitTlsData *tls)
5924 {
5925         if (!tls_offset_inited) {
5926                 tls_offset_inited = TRUE;
5927
5928                 lmf_tls_offset = read_tls_offset_from_method (mono_get_lmf_addr);
5929                 appdomain_tls_offset = read_tls_offset_from_method (mono_domain_get);
5930                 thread_tls_offset = read_tls_offset_from_method (mono_thread_current);
5931         }
5932
5933 #ifdef MONO_ARCH_SIGSEGV_ON_ALTSTACK
5934         setup_stack (tls);
5935 #endif
5936 }
5937
5938 void
5939 mono_arch_free_jit_tls_data (MonoJitTlsData *tls)
5940 {
5941 #ifdef MONO_ARCH_SIGSEGV_ON_ALTSTACK
5942         struct sigaltstack sa;
5943
5944         sa.ss_sp = tls->signal_stack;
5945         sa.ss_size = SIGNAL_STACK_SIZE;
5946         sa.ss_flags = SS_DISABLE;
5947         sigaltstack  (&sa, NULL);
5948
5949         if (tls->signal_stack)
5950                 munmap (tls->signal_stack, SIGNAL_STACK_SIZE);
5951 #endif
5952 }
5953
5954 void
5955 mono_arch_emit_this_vret_args (MonoCompile *cfg, MonoCallInst *inst, int this_reg, int this_type, int vt_reg)
5956 {
5957         MonoCallInst *call = (MonoCallInst*)inst;
5958         int out_reg = param_regs [0];
5959         guint64 regpair;
5960
5961         if (vt_reg != -1) {
5962                 CallInfo * cinfo = get_call_info (inst->signature, FALSE);
5963                 MonoInst *vtarg;
5964
5965                 if (cinfo->ret.storage == ArgValuetypeInReg) {
5966                         /*
5967                          * The valuetype is in RAX:RDX after the call, need to be copied to
5968                          * the stack. Push the address here, so the call instruction can
5969                          * access it.
5970                          */
5971                         MONO_INST_NEW (cfg, vtarg, OP_X86_PUSH);
5972                         vtarg->sreg1 = vt_reg;
5973                         mono_bblock_add_inst (cfg->cbb, vtarg);
5974
5975                         /* Align stack */
5976                         MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SUB_IMM, X86_ESP, X86_ESP, 8);
5977                 }
5978                 else {
5979                         MONO_INST_NEW (cfg, vtarg, OP_SETREG);
5980                         vtarg->sreg1 = vt_reg;
5981                         vtarg->dreg = mono_regstate_next_int (cfg->rs);
5982                         mono_bblock_add_inst (cfg->cbb, vtarg);
5983
5984                         regpair = (((guint64)out_reg) << 32) + vtarg->dreg;
5985                         call->out_ireg_args = g_slist_append (call->out_ireg_args, (gpointer)(regpair));
5986
5987                         out_reg = param_regs [1];
5988                 }
5989
5990                 g_free (cinfo);
5991         }
5992
5993         /* add the this argument */
5994         if (this_reg != -1) {
5995                 MonoInst *this;
5996                 MONO_INST_NEW (cfg, this, OP_SETREG);
5997                 this->type = this_type;
5998                 this->sreg1 = this_reg;
5999                 this->dreg = mono_regstate_next_int (cfg->rs);
6000                 mono_bblock_add_inst (cfg->cbb, this);
6001
6002                 regpair = (((guint64)out_reg) << 32) + this->dreg;
6003                 call->out_ireg_args = g_slist_append (call->out_ireg_args, (gpointer)(regpair));
6004         }
6005 }
6006
6007 MonoInst*
6008 mono_arch_get_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args)
6009 {
6010         MonoInst *ins = NULL;
6011
6012         if (cmethod->klass == mono_defaults.math_class) {
6013                 if (strcmp (cmethod->name, "Sin") == 0) {
6014                         MONO_INST_NEW (cfg, ins, OP_SIN);
6015                         ins->inst_i0 = args [0];
6016                 } else if (strcmp (cmethod->name, "Cos") == 0) {
6017                         MONO_INST_NEW (cfg, ins, OP_COS);
6018                         ins->inst_i0 = args [0];
6019                 } else if (strcmp (cmethod->name, "Tan") == 0) {
6020                         if (use_sse2)
6021                                 return ins;
6022                         MONO_INST_NEW (cfg, ins, OP_TAN);
6023                         ins->inst_i0 = args [0];
6024                 } else if (strcmp (cmethod->name, "Atan") == 0) {
6025                         if (use_sse2)
6026                                 return ins;
6027                         MONO_INST_NEW (cfg, ins, OP_ATAN);
6028                         ins->inst_i0 = args [0];
6029                 } else if (strcmp (cmethod->name, "Sqrt") == 0) {
6030                         MONO_INST_NEW (cfg, ins, OP_SQRT);
6031                         ins->inst_i0 = args [0];
6032                 } else if (strcmp (cmethod->name, "Abs") == 0 && fsig->params [0]->type == MONO_TYPE_R8) {
6033                         MONO_INST_NEW (cfg, ins, OP_ABS);
6034                         ins->inst_i0 = args [0];
6035                 }
6036 #if 0
6037                 /* OP_FREM is not IEEE compatible */
6038                 else if (strcmp (cmethod->name, "IEEERemainder") == 0) {
6039                         MONO_INST_NEW (cfg, ins, OP_FREM);
6040                         ins->inst_i0 = args [0];
6041                         ins->inst_i1 = args [1];
6042                 }
6043 #endif
6044         } else if(cmethod->klass->image == mono_defaults.corlib &&
6045                            (strcmp (cmethod->klass->name_space, "System.Threading") == 0) &&
6046                            (strcmp (cmethod->klass->name, "Interlocked") == 0)) {
6047
6048                 if (strcmp (cmethod->name, "Increment") == 0) {
6049                         MonoInst *ins_iconst;
6050                         guint32 opcode;
6051
6052                         if (fsig->params [0]->type == MONO_TYPE_I4)
6053                                 opcode = OP_ATOMIC_ADD_NEW_I4;
6054                         else if (fsig->params [0]->type == MONO_TYPE_I8)
6055                                 opcode = OP_ATOMIC_ADD_NEW_I8;
6056                         else
6057                                 g_assert_not_reached ();
6058                         MONO_INST_NEW (cfg, ins, opcode);
6059                         MONO_INST_NEW (cfg, ins_iconst, OP_ICONST);
6060                         ins_iconst->inst_c0 = 1;
6061
6062                         ins->inst_i0 = args [0];
6063                         ins->inst_i1 = ins_iconst;
6064                 } else if (strcmp (cmethod->name, "Decrement") == 0) {
6065                         MonoInst *ins_iconst;
6066                         guint32 opcode;
6067
6068                         if (fsig->params [0]->type == MONO_TYPE_I4)
6069                                 opcode = OP_ATOMIC_ADD_NEW_I4;
6070                         else if (fsig->params [0]->type == MONO_TYPE_I8)
6071                                 opcode = OP_ATOMIC_ADD_NEW_I8;
6072                         else
6073                                 g_assert_not_reached ();
6074                         MONO_INST_NEW (cfg, ins, opcode);
6075                         MONO_INST_NEW (cfg, ins_iconst, OP_ICONST);
6076                         ins_iconst->inst_c0 = -1;
6077
6078                         ins->inst_i0 = args [0];
6079                         ins->inst_i1 = ins_iconst;
6080                 } else if (strcmp (cmethod->name, "Add") == 0) {
6081                         guint32 opcode;
6082
6083                         if (fsig->params [0]->type == MONO_TYPE_I4)
6084                                 opcode = OP_ATOMIC_ADD_I4;
6085                         else if (fsig->params [0]->type == MONO_TYPE_I8)
6086                                 opcode = OP_ATOMIC_ADD_I8;
6087                         else
6088                                 g_assert_not_reached ();
6089
6090                         MONO_INST_NEW (cfg, ins, opcode);
6091
6092                         ins->inst_i0 = args [0];
6093                         ins->inst_i1 = args [1];
6094                 } else if (strcmp (cmethod->name, "Exchange") == 0) {
6095                         guint32 opcode;
6096
6097                         if (fsig->params [0]->type == MONO_TYPE_I4)
6098                                 opcode = OP_ATOMIC_EXCHANGE_I4;
6099                         else if ((fsig->params [0]->type == MONO_TYPE_I8) ||
6100                                          (fsig->params [0]->type == MONO_TYPE_I) ||
6101                                          (fsig->params [0]->type == MONO_TYPE_OBJECT))
6102                                 opcode = OP_ATOMIC_EXCHANGE_I8;
6103                         else
6104                                 return NULL;
6105
6106                         MONO_INST_NEW (cfg, ins, opcode);
6107
6108                         ins->inst_i0 = args [0];
6109                         ins->inst_i1 = args [1];
6110                 } else if (strcmp (cmethod->name, "Read") == 0 && (fsig->params [0]->type == MONO_TYPE_I8)) {
6111                         /* 64 bit reads are already atomic */
6112                         MONO_INST_NEW (cfg, ins, CEE_LDIND_I8);
6113                         ins->inst_i0 = args [0];
6114                 }
6115
6116                 /*
6117                  * Can't implement CompareExchange methods this way since they have
6118                  * three arguments.
6119                  */
6120         }
6121
6122         return ins;
6123 }
6124
6125 gboolean
6126 mono_arch_print_tree (MonoInst *tree, int arity)
6127 {
6128         return 0;
6129 }
6130
6131 MonoInst* mono_arch_get_domain_intrinsic (MonoCompile* cfg)
6132 {
6133         MonoInst* ins;
6134
6135         if (appdomain_tls_offset == -1)
6136                 return NULL;
6137
6138         MONO_INST_NEW (cfg, ins, OP_TLS_GET);
6139         ins->inst_offset = appdomain_tls_offset;
6140         return ins;
6141 }
6142
6143 MonoInst* mono_arch_get_thread_intrinsic (MonoCompile* cfg)
6144 {
6145         MonoInst* ins;
6146
6147         if (thread_tls_offset == -1)
6148                 return NULL;
6149
6150         MONO_INST_NEW (cfg, ins, OP_TLS_GET);
6151         ins->inst_offset = thread_tls_offset;
6152         return ins;
6153 }