mono/mini/mini-llvm.c

   1 /*
   2  * mini-llvm.c: llvm "Backend" for the mono JIT
   3  *
   4  * (C) 2009 Novell, Inc.
   5  */
   6
   7 #include "mini.h"
   8 #include <mono/metadata/debug-helpers.h>
   9 #include <mono/metadata/mempool-internals.h>
  10
  11 #ifndef __STDC_LIMIT_MACROS
  12 #define __STDC_LIMIT_MACROS
  13 #endif
  14 #ifndef __STDC_CONSTANT_MACROS
  15 #define __STDC_CONSTANT_MACROS
  16 #endif
  17
  18 #include "llvm-c/Core.h"
  19 #include "llvm-c/ExecutionEngine.h"
  20 #include "llvm-c/BitWriter.h"
  21 #include "llvm-c/Analysis.h"
  22
  23 #include "mini-llvm-cpp.h"
  24
  25  /*
  26   * Information associated by mono with LLVM modules.
  27   */
  28 typedef struct {
  29         LLVMModuleRef module;
  30         LLVMValueRef throw, throw_corlib_exception;
  31         GHashTable *llvm_types;
  32         LLVMValueRef got_var;
  33         const char *got_symbol;
  34         GHashTable *plt_entries;
  35 } MonoLLVMModule;
  36
  37 /*
  38  * Information associated by the backend with mono basic blocks.
  39  */
  40 typedef struct {
  41         LLVMBasicBlockRef bblock, end_bblock;
  42         LLVMValueRef finally_ind;
  43         gboolean added, invoke_target;
  44         /*
  45          * If this bblock is the start of a finally clause, this is a list of bblocks it
  46          * needs to branch to in ENDFINALLY.
  47          */
  48         GSList *call_handler_return_bbs;
  49         LLVMValueRef endfinally_switch;
  50         GSList *phi_nodes;
  51 } BBInfo;
  52
  53 /*
  54  * Structure containing emit state
  55  */
  56 typedef struct {
  57         MonoMemPool *mempool;
  58
  59         /* Maps method names to the corresponding LLVMValueRef */
  60         GHashTable *emitted_method_decls;
  61
  62         MonoCompile *cfg;
  63         LLVMValueRef lmethod;
  64         MonoLLVMModule *lmodule;
  65         LLVMModuleRef module;
  66         BBInfo *bblocks;
  67         int sindex, default_index, ex_index;
  68         LLVMBuilderRef builder;
  69         LLVMValueRef *values, *addresses;
  70         MonoType **vreg_cli_types;
  71         LLVMCallInfo *linfo;
  72         MonoMethodSignature *sig;
  73         GSList *builders;
  74         GHashTable *region_to_handler;
  75         LLVMBuilderRef alloca_builder;
  76         LLVMValueRef last_alloca;
  77
  78         char temp_name [32];
  79 } EmitContext;
  80
  81 typedef struct {
  82         MonoBasicBlock *bb;
  83         MonoInst *phi;
  84         MonoBasicBlock *in_bb;
  85         int sreg;
  86 } PhiNode;
  87
  88 /*
  89  * Instruction metadata
  90  * This is the same as ins_info, but LREG != IREG.
  91  */
  92 #ifdef MINI_OP
  93 #undef MINI_OP
  94 #endif
  95 #ifdef MINI_OP3
  96 #undef MINI_OP3
  97 #endif
  98 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
  99 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
 100 #define NONE ' '
 101 #define IREG 'i'
 102 #define FREG 'f'
 103 #define VREG 'v'
 104 #define XREG 'x'
 105 #define LREG 'l'
 106 /* keep in sync with the enum in mini.h */
 107 const char
 108 llvm_ins_info[] = {
 109 #include "mini-ops.h"
 110 };
 111 #undef MINI_OP
 112 #undef MINI_OP3
 113
 114 #if SIZEOF_VOID_P == 4
 115 #define GET_LONG_IMM(ins) (((guint64)(ins)->inst_ms_word << 32) | (guint64)(guint32)(ins)->inst_ls_word)
 116 #else
 117 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
 118 #endif
 119
 120 #define LLVM_INS_INFO(opcode) (&llvm_ins_info [((opcode) - OP_START - 1) * 4])
 121
 122 #if 0
 123 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
 124 #else
 125 #define TRACE_FAILURE(msg)
 126 #endif
 127
 128 #define LLVM_FAILURE(ctx, reason) do { \
 129         TRACE_FAILURE (reason); \
 130         (ctx)->cfg->exception_message = g_strdup (reason); \
 131         (ctx)->cfg->disable_llvm = TRUE; \
 132         goto FAILURE; \
 133 } while (0)
 134
 135 #define CHECK_FAILURE(ctx) do { \
 136     if ((ctx)->cfg->disable_llvm) \
 137                 goto FAILURE; \
 138 } while (0)
 139
 140 static LLVMIntPredicate cond_to_llvm_cond [] = {
 141         LLVMIntEQ,
 142         LLVMIntNE,
 143         LLVMIntSLE,
 144         LLVMIntSGE,
 145         LLVMIntSLT,
 146         LLVMIntSGT,
 147         LLVMIntULE,
 148         LLVMIntUGE,
 149         LLVMIntULT,
 150         LLVMIntUGT,
 151 };
 152
 153 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
 154         LLVMRealOEQ,
 155         LLVMRealUNE,
 156         LLVMRealOLE,
 157         LLVMRealOGE,
 158         LLVMRealOLT,
 159         LLVMRealOGT,
 160         LLVMRealULE,
 161         LLVMRealUGE,
 162         LLVMRealULT,
 163         LLVMRealUGT,
 164 };
 165
 166 static LLVMExecutionEngineRef ee;
 167 static guint32 current_cfg_tls_id;
 168
 169 static MonoLLVMModule jit_module, aot_module;
 170 static gboolean jit_module_inited;
 171
 172 static void init_jit_module (void);
 173
 174 /*
 175  * IntPtrType:
 176  *
 177  *   The LLVM type with width == sizeof (gpointer)
 178  */
 179 static LLVMTypeRef
 180 IntPtrType (void)
 181 {
 182         return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
 183 }
 184
 185 /*
 186  * get_vtype_size:
 187  *
 188  *   Return the size of the LLVM representation of the vtype T.
 189  */
 190 static guint32
 191 get_vtype_size (MonoType *t)
 192 {
 193         int size;
 194
 195         size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
 196
 197         while (size < sizeof (gpointer) && mono_is_power_of_two (size) == -1)
 198                 size ++;
 199
 200         return size;
 201 }
 202
 203 /*
 204  * simd_class_to_llvm_type:
 205  *
 206  *   Return the LLVM type corresponding to the Mono.SIMD class KLASS
 207  */
 208 static LLVMTypeRef
 209 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
 210 {
 211         if (!strcmp (klass->name, "Vector2d")) {
 212                 return LLVMVectorType (LLVMDoubleType (), 2);
 213         } else if (!strcmp (klass->name, "Vector2l")) {
 214                 return LLVMVectorType (LLVMInt64Type (), 2);
 215         } else if (!strcmp (klass->name, "Vector2ul")) {
 216                 return LLVMVectorType (LLVMInt64Type (), 2);
 217         } else if (!strcmp (klass->name, "Vector4i")) {
 218                 return LLVMVectorType (LLVMInt32Type (), 4);
 219         } else if (!strcmp (klass->name, "Vector4ui")) {
 220                 return LLVMVectorType (LLVMInt32Type (), 4);
 221         } else if (!strcmp (klass->name, "Vector4f")) {
 222                 return LLVMVectorType (LLVMFloatType (), 4);
 223         } else if (!strcmp (klass->name, "Vector8s")) {
 224                 return LLVMVectorType (LLVMInt16Type (), 8);
 225         } else if (!strcmp (klass->name, "Vector8us")) {
 226                 return LLVMVectorType (LLVMInt16Type (), 8);
 227         } else if (!strcmp (klass->name, "Vector16sb")) {
 228                 return LLVMVectorType (LLVMInt8Type (), 16);
 229         } else if (!strcmp (klass->name, "Vector16b")) {
 230                 return LLVMVectorType (LLVMInt8Type (), 16);
 231         } else {
 232                 printf ("%s\n", klass->name);
 233                 NOT_IMPLEMENTED;
 234                 return NULL;
 235         }
 236 }
 237
 238 /*
 239  * type_to_llvm_type:
 240  *
 241  *   Return the LLVM type corresponding to T.
 242  */
 243 static LLVMTypeRef
 244 type_to_llvm_type (EmitContext *ctx, MonoType *t)
 245 {
 246         if (t->byref)
 247                 return LLVMPointerType (LLVMInt8Type (), 0);
 248         switch (t->type) {
 249         case MONO_TYPE_VOID:
 250                 return LLVMVoidType ();
 251         case MONO_TYPE_I1:
 252                 return LLVMInt8Type ();
 253         case MONO_TYPE_I2:
 254                 return LLVMInt16Type ();
 255         case MONO_TYPE_I4:
 256                 return LLVMInt32Type ();
 257         case MONO_TYPE_U1:
 258                 return LLVMInt8Type ();
 259         case MONO_TYPE_U2:
 260                 return LLVMInt16Type ();
 261         case MONO_TYPE_U4:
 262                 return LLVMInt32Type ();
 263         case MONO_TYPE_BOOLEAN:
 264                 return LLVMInt8Type ();
 265         case MONO_TYPE_I8:
 266         case MONO_TYPE_U8:
 267                 return LLVMInt64Type ();
 268         case MONO_TYPE_CHAR:
 269                 return LLVMInt16Type ();
 270         case MONO_TYPE_R4:
 271                 return LLVMFloatType ();
 272         case MONO_TYPE_R8:
 273                 return LLVMDoubleType ();
 274         case MONO_TYPE_I:
 275         case MONO_TYPE_U:
 276                 return IntPtrType ();
 277         case MONO_TYPE_OBJECT:
 278         case MONO_TYPE_CLASS:
 279         case MONO_TYPE_ARRAY:
 280         case MONO_TYPE_SZARRAY:
 281         case MONO_TYPE_STRING:
 282         case MONO_TYPE_PTR:
 283                 return LLVMPointerType (IntPtrType (), 0);
 284         case MONO_TYPE_VAR:
 285         case MONO_TYPE_MVAR:
 286                 /* Because of generic sharing */
 287                 return IntPtrType ();
 288         case MONO_TYPE_GENERICINST:
 289                 if (!mono_type_generic_inst_is_valuetype (t))
 290                         return IntPtrType ();
 291                 /* Fall through */
 292         case MONO_TYPE_VALUETYPE:
 293         case MONO_TYPE_TYPEDBYREF: {
 294                 MonoClass *klass;
 295                 LLVMTypeRef ltype;
 296
 297                 klass = mono_class_from_mono_type (t);
 298
 299                 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
 300                         return simd_class_to_llvm_type (ctx, klass);
 301
 302                 if (klass->enumtype)
 303                         return type_to_llvm_type (ctx, mono_class_enum_basetype (klass));
 304                 ltype = g_hash_table_lookup (ctx->lmodule->llvm_types, klass);
 305                 if (!ltype) {
 306                         int i, size;
 307                         LLVMTypeRef *eltypes;
 308
 309                         size = get_vtype_size (t);
 310
 311                         eltypes = g_new (LLVMTypeRef, size);
 312                         for (i = 0; i < size; ++i)
 313                                 eltypes [i] = LLVMInt8Type ();
 314
 315                         ltype = LLVMStructType (eltypes, size, FALSE);
 316                         g_hash_table_insert (ctx->lmodule->llvm_types, klass, ltype);
 317                         g_free (eltypes);
 318                 }
 319                 return ltype;
 320         }
 321
 322         default:
 323                 printf ("X: %d\n", t->type);
 324                 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
 325                 ctx->cfg->disable_llvm = TRUE;
 326                 return NULL;
 327         }
 328 }
 329
 330 /*
 331  * type_is_unsigned:
 332  *
 333  *   Return whenever T is an unsigned int type.
 334  */
 335 static gboolean
 336 type_is_unsigned (EmitContext *ctx, MonoType *t)
 337 {
 338         if (t->byref)
 339                 return FALSE;
 340         switch (t->type) {
 341         case MONO_TYPE_U1:
 342         case MONO_TYPE_U2:
 343         case MONO_TYPE_U4:
 344         case MONO_TYPE_U8:
 345                 return TRUE;
 346         default:
 347                 return FALSE;
 348         }
 349 }
 350
 351 /*
 352  * type_to_llvm_arg_type:
 353  *
 354  *   Same as type_to_llvm_type, but treat i8/i16 as i32.
 355  */
 356 static LLVMTypeRef
 357 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
 358 {
 359         LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
 360
 361         if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
 362                 /*
 363                  * LLVM generates code which only sets the lower bits, while JITted
 364                  * code expects all the bits to be set.
 365                  */
 366                 ptype = LLVMInt32Type ();
 367         }
 368
 369         return ptype;
 370 }
 371
 372 /*
 373  * llvm_type_to_stack_type:
 374  *
 375  *   Return the LLVM type which needs to be used when a value of type TYPE is pushed
 376  * on the IL stack.
 377  */
 378 static G_GNUC_UNUSED LLVMTypeRef
 379 llvm_type_to_stack_type (LLVMTypeRef type)
 380 {
 381         if (type == NULL)
 382                 return NULL;
 383         if (type == LLVMInt8Type ())
 384                 return LLVMInt32Type ();
 385         else if (type == LLVMInt16Type ())
 386                 return LLVMInt32Type ();
 387         else if (type == LLVMFloatType ())
 388                 return LLVMDoubleType ();
 389         else
 390                 return type;
 391 }
 392
 393 /*
 394  * regtype_to_llvm_type:
 395  *
 396  *   Return the LLVM type corresponding to the regtype C used in instruction
 397  * descriptions.
 398  */
 399 static LLVMTypeRef
 400 regtype_to_llvm_type (char c)
 401 {
 402         switch (c) {
 403         case 'i':
 404                 return LLVMInt32Type ();
 405         case 'l':
 406                 return LLVMInt64Type ();
 407         case 'f':
 408                 return LLVMDoubleType ();
 409         default:
 410                 return NULL;
 411         }
 412 }
 413
 414 /*
 415  * op_to_llvm_type:
 416  *
 417  *   Return the LLVM type corresponding to the unary/binary opcode OPCODE.
 418  */
 419 static LLVMTypeRef
 420 op_to_llvm_type (int opcode)
 421 {
 422         switch (opcode) {
 423         case OP_ICONV_TO_I1:
 424         case OP_LCONV_TO_I1:
 425                 return LLVMInt8Type ();
 426         case OP_ICONV_TO_U1:
 427         case OP_LCONV_TO_U1:
 428                 return LLVMInt8Type ();
 429         case OP_ICONV_TO_I2:
 430         case OP_LCONV_TO_I2:
 431                 return LLVMInt16Type ();
 432         case OP_ICONV_TO_U2:
 433         case OP_LCONV_TO_U2:
 434                 return LLVMInt16Type ();
 435         case OP_ICONV_TO_I4:
 436         case OP_LCONV_TO_I4:
 437                 return LLVMInt32Type ();
 438         case OP_ICONV_TO_U4:
 439         case OP_LCONV_TO_U4:
 440                 return LLVMInt32Type ();
 441         case OP_ICONV_TO_I8:
 442                 return LLVMInt64Type ();
 443         case OP_ICONV_TO_R4:
 444                 return LLVMFloatType ();
 445         case OP_ICONV_TO_R8:
 446                 return LLVMDoubleType ();
 447         case OP_ICONV_TO_U8:
 448                 return LLVMInt64Type ();
 449         case OP_FCONV_TO_I4:
 450                 return LLVMInt32Type ();
 451         case OP_FCONV_TO_I8:
 452                 return LLVMInt64Type ();
 453         case OP_FCONV_TO_I1:
 454         case OP_FCONV_TO_U1:
 455                 return LLVMInt8Type ();
 456         case OP_FCONV_TO_I2:
 457         case OP_FCONV_TO_U2:
 458                 return LLVMInt16Type ();
 459         case OP_FCONV_TO_I:
 460         case OP_FCONV_TO_U:
 461                 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
 462         case OP_IADD_OVF:
 463         case OP_IADD_OVF_UN:
 464         case OP_ISUB_OVF:
 465         case OP_ISUB_OVF_UN:
 466         case OP_IMUL_OVF:
 467         case OP_IMUL_OVF_UN:
 468                 return LLVMInt32Type ();
 469         case OP_LADD_OVF:
 470         case OP_LADD_OVF_UN:
 471         case OP_LSUB_OVF:
 472         case OP_LSUB_OVF_UN:
 473         case OP_LMUL_OVF:
 474         case OP_LMUL_OVF_UN:
 475                 return LLVMInt64Type ();
 476         default:
 477                 printf ("%s\n", mono_inst_name (opcode));
 478                 g_assert_not_reached ();
 479                 return NULL;
 480         }
 481 }
 482
 483 /*
 484  * load_store_to_llvm_type:
 485  *
 486  *   Return the size/sign/zero extension corresponding to the load/store opcode
 487  * OPCODE.
 488  */
 489 static LLVMTypeRef
 490 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
 491 {
 492         *sext = FALSE;
 493         *zext = FALSE;
 494
 495         switch (opcode) {
 496         case OP_LOADI1_MEMBASE:
 497         case OP_STOREI1_MEMBASE_REG:
 498         case OP_STOREI1_MEMBASE_IMM:
 499                 *size = 1;
 500                 *sext = TRUE;
 501                 return LLVMInt8Type ();
 502         case OP_LOADU1_MEMBASE:
 503         case OP_LOADU1_MEM:
 504                 *size = 1;
 505                 *zext = TRUE;
 506                 return LLVMInt8Type ();
 507         case OP_LOADI2_MEMBASE:
 508         case OP_STOREI2_MEMBASE_REG:
 509         case OP_STOREI2_MEMBASE_IMM:
 510                 *size = 2;
 511                 *sext = TRUE;
 512                 return LLVMInt16Type ();
 513         case OP_LOADU2_MEMBASE:
 514         case OP_LOADU2_MEM:
 515                 *size = 2;
 516                 *zext = TRUE;
 517                 return LLVMInt16Type ();
 518         case OP_LOADI4_MEMBASE:
 519         case OP_LOADU4_MEMBASE:
 520         case OP_LOADI4_MEM:
 521         case OP_LOADU4_MEM:
 522         case OP_STOREI4_MEMBASE_REG:
 523         case OP_STOREI4_MEMBASE_IMM:
 524                 *size = 4;
 525                 return LLVMInt32Type ();
 526         case OP_LOADI8_MEMBASE:
 527         case OP_LOADI8_MEM:
 528         case OP_STOREI8_MEMBASE_REG:
 529         case OP_STOREI8_MEMBASE_IMM:
 530                 *size = 8;
 531                 return LLVMInt64Type ();
 532         case OP_LOADR4_MEMBASE:
 533         case OP_STORER4_MEMBASE_REG:
 534                 *size = 4;
 535                 return LLVMFloatType ();
 536         case OP_LOADR8_MEMBASE:
 537         case OP_STORER8_MEMBASE_REG:
 538                 *size = 8;
 539                 return LLVMDoubleType ();
 540         case OP_LOAD_MEMBASE:
 541         case OP_LOAD_MEM:
 542         case OP_STORE_MEMBASE_REG:
 543         case OP_STORE_MEMBASE_IMM:
 544                 *size = sizeof (gpointer);
 545                 return IntPtrType ();
 546         default:
 547                 g_assert_not_reached ();
 548                 return NULL;
 549         }
 550 }
 551
 552 /*
 553  * ovf_op_to_intrins:
 554  *
 555  *   Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
 556  */
 557 static const char*
 558 ovf_op_to_intrins (int opcode)
 559 {
 560         switch (opcode) {
 561         case OP_IADD_OVF:
 562                 return "llvm.sadd.with.overflow.i32";
 563         case OP_IADD_OVF_UN:
 564                 return "llvm.uadd.with.overflow.i32";
 565         case OP_ISUB_OVF:
 566                 return "llvm.ssub.with.overflow.i32";
 567         case OP_ISUB_OVF_UN:
 568                 return "llvm.usub.with.overflow.i32";
 569         case OP_IMUL_OVF:
 570                 return "llvm.smul.with.overflow.i32";
 571         case OP_IMUL_OVF_UN:
 572                 return "llvm.umul.with.overflow.i32";
 573         case OP_LADD_OVF:
 574                 return "llvm.sadd.with.overflow.i64";
 575         case OP_LADD_OVF_UN:
 576                 return "llvm.uadd.with.overflow.i64";
 577         case OP_LSUB_OVF:
 578                 return "llvm.ssub.with.overflow.i64";
 579         case OP_LSUB_OVF_UN:
 580                 return "llvm.usub.with.overflow.i64";
 581         case OP_LMUL_OVF:
 582                 return "llvm.smul.with.overflow.i64";
 583         case OP_LMUL_OVF_UN:
 584                 return "llvm.umul.with.overflow.i64";
 585         default:
 586                 g_assert_not_reached ();
 587                 return NULL;
 588         }
 589 }
 590
 591 static const char*
 592 simd_op_to_intrins (int opcode)
 593 {
 594         switch (opcode) {
 595 #if defined(TARGET_X86) || defined(TARGET_AMD64)
 596         case OP_MINPD:
 597                 return "llvm.x86.sse2.min.pd";
 598         case OP_MINPS:
 599                 return "llvm.x86.sse2.min.ps";
 600         case OP_PMIND_UN:
 601                 return "llvm.x86.sse41.pminud";
 602         case OP_PMINW_UN:
 603                 return "llvm.x86.sse41.pminuw";
 604         case OP_PMINB_UN:
 605                 return "llvm.x86.sse41.pminub";
 606         case OP_MAXPD:
 607                 return "llvm.x86.sse2.max.pd";
 608         case OP_MAXPS:
 609                 return "llvm.x86.sse2.max.ps";
 610         case OP_PMAXD_UN:
 611                 return "llvm.x86.sse41.pmaxud";
 612         case OP_PMAXW_UN:
 613                 return "llvm.x86.sse41.pmaxuw";
 614         case OP_PMAXB_UN:
 615                 return "llvm.x86.sse41.pmaxub";
 616 #endif
 617         default:
 618                 g_assert_not_reached ();
 619                 return NULL;
 620         }
 621 }
 622
 623 /*
 624  * get_bb:
 625  *
 626  *   Return the LLVM basic block corresponding to BB.
 627  */
 628 static LLVMBasicBlockRef
 629 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
 630 {
 631         char bb_name [128];
 632
 633         if (ctx->bblocks [bb->block_num].bblock == NULL) {
 634                 sprintf (bb_name, "BB%d", bb->block_num);
 635
 636                 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
 637                 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
 638         }
 639
 640         return ctx->bblocks [bb->block_num].bblock;
 641 }
 642
 643 /*
 644  * get_end_bb:
 645  *
 646  *   Return the last LLVM bblock corresponding to BB.
 647  * This might not be equal to the bb returned by get_bb () since we need to generate
 648  * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
 649  */
 650 static LLVMBasicBlockRef
 651 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
 652 {
 653         get_bb (ctx, bb);
 654         return ctx->bblocks [bb->block_num].end_bblock;
 655 }
 656
 657 static LLVMBasicBlockRef
 658 gen_bb (EmitContext *ctx, const char *prefix)
 659 {
 660         char bb_name [128];
 661
 662         sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
 663         return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
 664 }
 665
 666 /*
 667  * resolve_patch:
 668  *
 669  *   Return the target of the patch identified by TYPE and TARGET.
 670  */
 671 static gpointer
 672 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
 673 {
 674         MonoJumpInfo ji;
 675
 676         memset (&ji, 0, sizeof (ji));
 677         ji.type = type;
 678         ji.data.target = target;
 679
 680         return mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
 681 }
 682
 683 /*
 684  * convert_full:
 685  *
 686  *   Emit code to convert the LLVM value V to DTYPE.
 687  */
 688 static LLVMValueRef
 689 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
 690 {
 691         LLVMTypeRef stype = LLVMTypeOf (v);
 692
 693         if (stype != dtype) {
 694                 gboolean ext = FALSE;
 695
 696                 /* Extend */
 697                 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
 698                         ext = TRUE;
 699                 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
 700                         ext = TRUE;
 701                 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
 702                         ext = TRUE;
 703
 704                 if (ext)
 705                         return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
 706
 707                 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
 708                         return LLVMBuildFPExt (ctx->builder, v, dtype, "");
 709
 710                 /* Trunc */
 711                 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
 712                         return LLVMBuildTrunc (ctx->builder, v, dtype, "");
 713                 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
 714                         return LLVMBuildTrunc (ctx->builder, v, dtype, "");
 715                 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
 716                         return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
 717
 718                 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
 719                         return LLVMBuildBitCast (ctx->builder, v, dtype, "");
 720                 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
 721                         return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
 722                 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
 723                         return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
 724
 725 #ifdef MONO_ARCH_SOFT_FLOAT
 726                 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
 727                         return LLVMBuildBitCast (ctx->builder, v, dtype, "");
 728                 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
 729                         return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
 730 #endif
 731
 732                 LLVMDumpValue (v);
 733                 LLVMDumpValue (LLVMConstNull (dtype));
 734                 g_assert_not_reached ();
 735                 return NULL;
 736         } else {
 737                 return v;
 738         }
 739 }
 740
 741 static LLVMValueRef
 742 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
 743 {
 744         return convert_full (ctx, v, dtype, FALSE);
 745 }
 746
 747 /*
 748  * emit_volatile_load:
 749  *
 750  *   If vreg is volatile, emit a load from its address.
 751  */
 752 static LLVMValueRef
 753 emit_volatile_load (EmitContext *ctx, int vreg)
 754 {
 755         MonoType *t;
 756
 757         LLVMValueRef v = LLVMBuildLoad (ctx->builder, ctx->addresses [vreg], "");
 758         t = ctx->vreg_cli_types [vreg];
 759         if (t && !t->byref) {
 760                 /*
 761                  * Might have to zero extend since llvm doesn't have
 762                  * unsigned types.
 763                  */
 764                 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2)
 765                         v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
 766                 else if (t->type == MONO_TYPE_U8)
 767                         v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
 768         }
 769
 770         return v;
 771 }
 772
 773 /*
 774  * emit_volatile_store:
 775  *
 776  *   If VREG is volatile, emit a store from its value to its address.
 777  */
 778 static void
 779 emit_volatile_store (EmitContext *ctx, int vreg)
 780 {
 781         MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
 782
 783         if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
 784                 g_assert (ctx->addresses [vreg]);
 785                 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
 786         }
 787 }
 788
 789 /*
 790  * sig_to_llvm_sig:
 791  *
 792  *   Return the LLVM signature corresponding to the mono signature SIG using the
 793  * calling convention information in CINFO.
 794  */
 795 static LLVMTypeRef
 796 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
 797 {
 798         LLVMTypeRef ret_type;
 799         LLVMTypeRef *param_types = NULL;
 800         LLVMTypeRef res;
 801         int i, j, pindex;
 802         gboolean vretaddr = FALSE;
 803
 804         ret_type = type_to_llvm_type (ctx, sig->ret);
 805         CHECK_FAILURE (ctx);
 806
 807         if (cinfo && cinfo->ret.storage == LLVMArgVtypeInReg) {
 808                 /* LLVM models this by returning an aggregate value */
 809                 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
 810                         LLVMTypeRef members [2];
 811
 812                         members [0] = IntPtrType ();
 813                         ret_type = LLVMStructType (members, 1, FALSE);
 814                 } else {
 815                         g_assert_not_reached ();
 816                 }
 817         } else if (cinfo && MONO_TYPE_ISSTRUCT (sig->ret)) {
 818                 g_assert (cinfo->ret.storage == LLVMArgVtypeRetAddr);
 819                 vretaddr = TRUE;
 820         }
 821
 822         param_types = g_new0 (LLVMTypeRef, (sig->param_count * 2) + 2);
 823         pindex = 0;
 824         if (vretaddr) {
 825                 ret_type = LLVMVoidType ();
 826                 param_types [pindex ++] = IntPtrType ();
 827         }
 828         if (sig->hasthis)
 829                 param_types [pindex ++] = IntPtrType ();
 830         for (i = 0; i < sig->param_count; ++i) {
 831                 if (cinfo && cinfo->args [i + sig->hasthis].storage == LLVMArgVtypeInReg) {
 832                         for (j = 0; j < 2; ++j) {
 833                                 switch (cinfo->args [i + sig->hasthis].pair_storage [j]) {
 834                                 case LLVMArgInIReg:
 835                                         param_types [pindex ++] = LLVMIntType (sizeof (gpointer) * 8);
 836                                         break;
 837                                 case LLVMArgNone:
 838                                         break;
 839                                 default:
 840                                         g_assert_not_reached ();
 841                                 }
 842                         }
 843                 } else if (cinfo && cinfo->args [i + sig->hasthis].storage == LLVMArgVtypeByVal) {
 844                         param_types [pindex] = type_to_llvm_arg_type (ctx, sig->params [i]);
 845                         CHECK_FAILURE (ctx);
 846                         param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
 847                         pindex ++;
 848                 } else {
 849                         param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
 850                 }
 851         }
 852         CHECK_FAILURE (ctx);
 853
 854         res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
 855         g_free (param_types);
 856
 857         return res;
 858
 859  FAILURE:
 860         g_free (param_types);
 861
 862         return NULL;
 863 }
 864
 865 /*
 866  * LLVMFunctionType1:
 867  *
 868  *   Create an LLVM function type from the arguments.
 869  */
 870 static G_GNUC_UNUSED LLVMTypeRef
 871 LLVMFunctionType1(LLVMTypeRef ReturnType,
 872                                   LLVMTypeRef ParamType1,
 873                                   int IsVarArg)
 874 {
 875         LLVMTypeRef param_types [1];
 876
 877         param_types [0] = ParamType1;
 878
 879         return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
 880 }
 881
 882 /*
 883  * LLVMFunctionType2:
 884  *
 885  *   Create an LLVM function type from the arguments.
 886  */
 887 static LLVMTypeRef
 888 LLVMFunctionType2(LLVMTypeRef ReturnType,
 889                                   LLVMTypeRef ParamType1,
 890                                   LLVMTypeRef ParamType2,
 891                                   int IsVarArg)
 892 {
 893         LLVMTypeRef param_types [2];
 894
 895         param_types [0] = ParamType1;
 896         param_types [1] = ParamType2;
 897
 898         return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
 899 }
 900
 901 /*
 902  * LLVMFunctionType3:
 903  *
 904  *   Create an LLVM function type from the arguments.
 905  */
 906 static LLVMTypeRef
 907 LLVMFunctionType3(LLVMTypeRef ReturnType,
 908                                   LLVMTypeRef ParamType1,
 909                                   LLVMTypeRef ParamType2,
 910                                   LLVMTypeRef ParamType3,
 911                                   int IsVarArg)
 912 {
 913         LLVMTypeRef param_types [3];
 914
 915         param_types [0] = ParamType1;
 916         param_types [1] = ParamType2;
 917         param_types [2] = ParamType3;
 918
 919         return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
 920 }
 921
 922 /*
 923  * create_builder:
 924  *
 925  *   Create an LLVM builder and remember it so it can be freed later.
 926  */
 927 static LLVMBuilderRef
 928 create_builder (EmitContext *ctx)
 929 {
 930         LLVMBuilderRef builder = LLVMCreateBuilder ();
 931
 932         ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
 933
 934         return builder;
 935 }
 936
 937 static LLVMValueRef
 938 get_plt_entry (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
 939 {
 940         char *callee_name = mono_aot_get_plt_symbol (type, data);
 941         LLVMValueRef callee;
 942
 943         if (!callee_name)
 944                 return NULL;
 945
 946         // FIXME: Locking
 947         callee = g_hash_table_lookup (ctx->lmodule->plt_entries, callee_name);
 948         if (!callee) {
 949                 callee = LLVMAddFunction (ctx->module, callee_name, llvm_sig);
 950
 951                 LLVMSetVisibility (callee, LLVMHiddenVisibility);
 952
 953                 g_hash_table_insert (ctx->lmodule->plt_entries, (char*)callee_name, callee);
 954         }
 955
 956         return callee;
 957 }
 958
 959 static void
 960 emit_cond_throw_pos (EmitContext *ctx)
 961 {
 962 }
 963
 964 /*
 965  * emit_call:
 966  *
 967  *   Emit an LLVM call or invoke instruction depending on whenever the call is inside
 968  * a try region.
 969  */
 970 static LLVMValueRef
 971 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
 972 {
 973         MonoCompile *cfg = ctx->cfg;
 974         LLVMValueRef lcall;
 975         LLVMBuilderRef builder = *builder_ref;
 976
 977         // FIXME: Nested clauses
 978         if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY)) {
 979                 MonoMethodHeader *header = mono_method_get_header (cfg->method);
 980                 // FIXME: Add a macro for this
 981                 int clause_index = (bb->region >> 8) - 1;
 982                 MonoExceptionClause *ec = &header->clauses [clause_index];
 983                 MonoBasicBlock *tblock;
 984                 LLVMBasicBlockRef ex_bb, noex_bb;
 985
 986                 /*
 987                  * Have to use an invoke instead of a call, branching to the
 988                  * handler bblock of the clause containing this bblock.
 989                  */
 990
 991                 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
 992
 993                 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
 994                 g_assert (tblock);
 995
 996                 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
 997
 998                 ex_bb = get_bb (ctx, tblock);
 999
1000                 noex_bb = gen_bb (ctx, "NOEX_BB");
1001
1002                 /* Use an invoke */
1003                 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
1004
1005                 builder = ctx->builder = create_builder (ctx);
1006                 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1007
1008                 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1009         } else {
1010                 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
1011                 ctx->builder = builder;
1012         }
1013
1014         *builder_ref = ctx->builder;
1015
1016         return lcall;
1017 }
1018
1019 /*
1020  * emit_cond_system_exception:
1021  *
1022  *   Emit code to throw the exception EXC_TYPE if the condition CMP is false.
1023  */
1024 static void
1025 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
1026 {
1027         LLVMBasicBlockRef ex_bb, noex_bb;
1028         LLVMBuilderRef builder;
1029         MonoClass *exc_class;
1030         LLVMValueRef args [2];
1031
1032         ex_bb = gen_bb (ctx, "EX_BB");
1033         noex_bb = gen_bb (ctx, "NOEX_BB");
1034
1035         LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
1036
1037         exc_class = mono_class_from_name (mono_defaults.corlib, "System", exc_type);
1038         g_assert (exc_class);
1039
1040         /* Emit exception throwing code */
1041         builder = create_builder (ctx);
1042         LLVMPositionBuilderAtEnd (builder, ex_bb);
1043
1044         if (!ctx->lmodule->throw_corlib_exception) {
1045                 LLVMValueRef callee;
1046                 LLVMTypeRef sig;
1047
1048                 MonoMethodSignature *throw_sig = mono_metadata_signature_alloc (mono_defaults.corlib, 2);
1049                 throw_sig->ret = &mono_defaults.void_class->byval_arg;
1050                 throw_sig->params [0] = &mono_defaults.int32_class->byval_arg;
1051                 throw_sig->params [1] = &mono_defaults.int32_class->byval_arg;
1052                 sig = sig_to_llvm_sig (ctx, throw_sig, NULL);
1053
1054                 if (ctx->cfg->compile_aot) {
1055                         callee = get_plt_entry (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_corlib_exception");
1056                 } else {
1057                         callee = LLVMAddFunction (ctx->module, "throw_corlib_exception", sig_to_llvm_sig (ctx, throw_sig, NULL));
1058
1059 #ifdef TARGET_X86
1060                         /*
1061                          * LLVM generated code doesn't push the arguments, so we need another
1062                          * throw trampoline.
1063                          */
1064                         LLVMAddGlobalMapping (ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_llvm_throw_corlib_exception"));
1065 #else
1066                         LLVMAddGlobalMapping (ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_corlib_exception"));
1067 #endif
1068                 }
1069
1070                 mono_memory_barrier ();
1071                 ctx->lmodule->throw_corlib_exception = callee;
1072         }
1073
1074 #ifdef TARGET_X86
1075         args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
1076 #else
1077         args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token, FALSE);
1078 #endif
1079         /*
1080          * FIXME: The offset is 0, this is not a problem for exception handling
1081          * in general, because we don't llvm compile methods with handlers, its only
1082          * a problem for line numbers in stack traces.
1083          */
1084         args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1085         emit_call (ctx, bb, &builder, ctx->lmodule->throw_corlib_exception, args, 2);
1086
1087         LLVMBuildUnreachable (builder);
1088
1089         ctx->builder = create_builder (ctx);
1090         LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1091
1092         ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1093
1094         ctx->ex_index ++;
1095 }
1096
1097 /*
1098  * emit_reg_to_vtype:
1099  *
1100  *   Emit code to store the vtype in the registers REGS to the address ADDRESS.
1101  */
1102 static void
1103 emit_reg_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *regs)
1104 {
1105         int j, size;
1106
1107         size = get_vtype_size (t);
1108
1109         if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1110                 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
1111         }
1112
1113         for (j = 0; j < 2; ++j) {
1114                 LLVMValueRef index [2], addr;
1115                 int part_size = size > sizeof (gpointer) ? sizeof (gpointer) : size;
1116                 LLVMTypeRef part_type;
1117
1118                 if (ainfo->pair_storage [j] == LLVMArgNone)
1119                         continue;
1120
1121                 part_type = LLVMIntType (part_size * 8);
1122                 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1123                         index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
1124                         addr = LLVMBuildGEP (builder, address, index, 1, "");
1125                 } else {
1126                         index [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1127                         index [1] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
1128                         addr = LLVMBuildGEP (builder, address, index, 2, "");
1129                 }
1130                 switch (ainfo->pair_storage [j]) {
1131                 case LLVMArgInIReg:
1132                         LLVMBuildStore (builder, convert (ctx, regs [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
1133                         break;
1134                 case LLVMArgNone:
1135                         break;
1136                 default:
1137                         g_assert_not_reached ();
1138                 }
1139
1140                 size -= sizeof (gpointer);
1141         }
1142 }
1143
1144 /*
1145  * emit_vtype_to_reg:
1146  *
1147  *   Emit code to load a vtype at address ADDRESS into registers. Store the registers
1148  * into REGS, and the number of registers into NREGS.
1149  */
1150 static void
1151 emit_vtype_to_reg (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *regs, guint32 *nregs)
1152 {
1153         int pindex = 0;
1154         int j, size;
1155
1156         size = get_vtype_size (t);
1157
1158         if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1159                 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
1160         }
1161
1162         for (j = 0; j < 2; ++j) {
1163                 LLVMValueRef index [2], addr;
1164                 int partsize = size > sizeof (gpointer) ? sizeof (gpointer) : size;
1165
1166                 if (ainfo->pair_storage [j] == LLVMArgNone)
1167                         continue;
1168
1169                 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1170                         index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
1171                         addr = LLVMBuildGEP (builder, address, index, 1, "");
1172                 } else {
1173                         index [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1174                         index [1] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
1175                         addr = LLVMBuildGEP (builder, address, index, 2, "");
1176                 }
1177                 switch (ainfo->pair_storage [j]) {
1178                 case LLVMArgInIReg:
1179                         regs [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
1180                         break;
1181                 case LLVMArgNone:
1182                         break;
1183                 default:
1184                         g_assert_not_reached ();
1185                 }
1186                 size -= sizeof (gpointer);
1187         }
1188
1189         *nregs = pindex;
1190 }
1191
1192 static LLVMValueRef
1193 build_alloca (EmitContext *ctx, MonoType *t)
1194 {
1195         MonoClass *k = mono_class_from_mono_type (t);
1196         int align;
1197
1198         if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
1199                 align = 16;
1200         else
1201                 align = mono_class_min_align (k);
1202
1203         /* Sometimes align is not a power of 2 */
1204         while (mono_is_power_of_two (align) == -1)
1205                 align ++;
1206
1207         /*
1208          * Have to place all alloca's at the end of the entry bb, since otherwise they would
1209          * get executed every time control reaches them.
1210          */
1211         LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
1212
1213         ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, type_to_llvm_type (ctx, t), NULL, align, "");
1214         return ctx->last_alloca;
1215 }
1216
1217 /*
1218  * Put the global into the 'llvm.used' array to prevent it from being optimized away.
1219  */
1220 static void
1221 mark_as_used (LLVMModuleRef module, LLVMValueRef global)
1222 {
1223         LLVMTypeRef used_type;
1224         LLVMValueRef used, used_elem;
1225
1226         used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), 1);
1227         used = LLVMAddGlobal (module, used_type, "llvm.used");
1228         used_elem = LLVMConstBitCast (global, LLVMPointerType (LLVMInt8Type (), 0));
1229         LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), &used_elem, 1));
1230         LLVMSetLinkage (used, LLVMAppendingLinkage);
1231         LLVMSetSection (used, "llvm.metadata");
1232 }
1233
1234 /*
1235  * emit_entry_bb:
1236  *
1237  *   Emit code to load/convert arguments.
1238  */
1239 static void
1240 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder, int *pindexes)
1241 {
1242         int i, pindex;
1243         MonoCompile *cfg = ctx->cfg;
1244         MonoMethodSignature *sig = ctx->sig;
1245         LLVMCallInfo *linfo = ctx->linfo;
1246         MonoBasicBlock *bb;
1247
1248         ctx->alloca_builder = create_builder (ctx);
1249
1250         /*
1251          * Handle indirect/volatile variables by allocating memory for them
1252          * using 'alloca', and storing their address in a temporary.
1253          */
1254         for (i = 0; i < cfg->num_varinfo; ++i) {
1255                 MonoInst *var = cfg->varinfo [i];
1256                 LLVMTypeRef vtype;
1257
1258                 if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || MONO_TYPE_ISSTRUCT (var->inst_vtype)) {
1259                         vtype = type_to_llvm_type (ctx, var->inst_vtype);
1260                         CHECK_FAILURE (ctx);
1261                         /* Could be already created by an OP_VPHI */
1262                         if (!ctx->addresses [var->dreg])
1263                                 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
1264                         ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
1265                 }
1266         }
1267
1268         for (i = 0; i < sig->param_count; ++i) {
1269                 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
1270                 int reg = cfg->args [i + sig->hasthis]->dreg;
1271
1272                 if (ainfo->storage == LLVMArgVtypeInReg) {
1273                         LLVMValueRef regs [2];
1274
1275                         /*
1276                          * Emit code to save the argument from the registers to
1277                          * the real argument.
1278                          */
1279                         pindex = pindexes [i];
1280                         regs [0] = LLVMGetParam (ctx->lmethod, pindex);
1281                         if (ainfo->pair_storage [1] != LLVMArgNone)
1282                                 regs [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
1283                         else
1284                                 regs [1] = NULL;
1285
1286                         ctx->addresses [reg] = build_alloca (ctx, sig->params [i]);
1287
1288                         emit_reg_to_vtype (ctx, builder, sig->params [i], ctx->addresses [reg], ainfo, regs);
1289
1290                         if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->params [i]))) {
1291                                 /* Treat these as normal values */
1292                                 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
1293                         }
1294                 } else if (ainfo->storage == LLVMArgVtypeByVal) {
1295                         ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindexes [i]);
1296                 } else {
1297                         ctx->values [reg] = convert (ctx, ctx->values [reg], llvm_type_to_stack_type (type_to_llvm_type (ctx, sig->params [i])));
1298                 }
1299         }
1300
1301         if (cfg->vret_addr)
1302                 emit_volatile_store (ctx, cfg->vret_addr->dreg);
1303         if (sig->hasthis)
1304                 emit_volatile_store (ctx, cfg->args [0]->dreg);
1305         for (i = 0; i < sig->param_count; ++i)
1306                 if (!MONO_TYPE_ISSTRUCT (sig->params [i]))
1307                         emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
1308
1309         /*
1310          * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
1311          * it needs to continue normally, or return back to the exception handling system.
1312          */
1313         for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
1314                 if (bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER))
1315                         g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
1316                 if (bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER) && bb->in_scount == 0) {
1317                         LLVMValueRef val = LLVMBuildAlloca (builder, LLVMInt32Type (), "");
1318                         LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
1319
1320                         ctx->bblocks [bb->block_num].finally_ind = val;
1321                 }
1322         }
1323
1324  FAILURE:
1325         ;
1326 }
1327
1328 /* Have to export this for AOT */
1329 void
1330 mono_personality (void);
1331
1332 void
1333 mono_personality (void)
1334 {
1335         /* Not used */
1336         g_assert_not_reached ();
1337 }
1338
1339 /*
1340  * mono_llvm_emit_method:
1341  *
1342  *   Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
1343  */
1344 void
1345 mono_llvm_emit_method (MonoCompile *cfg)
1346 {
1347         EmitContext *ctx;
1348         MonoMethodSignature *sig;
1349         MonoBasicBlock *bb;
1350         LLVMTypeRef method_type;
1351         LLVMValueRef method = NULL, debug_alias = NULL;
1352         char *method_name, *debug_name = NULL;
1353         LLVMValueRef *values, *addresses;
1354         LLVMTypeRef *vreg_types;
1355         MonoType **vreg_cli_types;
1356         int i, max_block_num, pindex, bb_index;
1357         int *pindexes = NULL;
1358         gboolean last = FALSE;
1359         GPtrArray *phi_values;
1360         LLVMCallInfo *linfo;
1361         GSList *l;
1362         LLVMModuleRef module;
1363         gboolean *is_dead;
1364         gboolean *unreachable;
1365         BBInfo *bblocks;
1366         GPtrArray *bblock_list;
1367         MonoMethodHeader *header;
1368         MonoExceptionClause *clause;
1369
1370         /* The code below might acquire the loader lock, so use it for global locking */
1371         mono_loader_lock ();
1372
1373         /* Used to communicate with the callbacks */
1374         TlsSetValue (current_cfg_tls_id, cfg);
1375
1376         ctx = g_new0 (EmitContext, 1);
1377         ctx->cfg = cfg;
1378         ctx->mempool = cfg->mempool;
1379
1380         /*
1381          * This maps vregs to the LLVM instruction defining them
1382          */
1383         values = g_new0 (LLVMValueRef, cfg->next_vreg);
1384         /*
1385          * This maps vregs for volatile variables to the LLVM instruction defining their
1386          * address.
1387          */
1388         addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
1389         vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
1390         vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
1391         phi_values = g_ptr_array_new ();
1392         /*
1393          * This signals whenever the vreg was defined by a phi node with no input vars
1394          * (i.e. all its input bblocks end with NOT_REACHABLE).
1395          */
1396         is_dead = g_new0 (gboolean, cfg->next_vreg);
1397         /* Whenever the bblock is unreachable */
1398         unreachable = g_new0 (gboolean, cfg->max_block_num);
1399
1400         bblock_list = g_ptr_array_new ();
1401
1402         ctx->values = values;
1403         ctx->addresses = addresses;
1404         ctx->vreg_cli_types = vreg_cli_types;
1405         ctx->region_to_handler = g_hash_table_new (NULL, NULL);
1406
1407         if (cfg->compile_aot) {
1408                 ctx->lmodule = &aot_module;
1409                 method_name = mono_aot_get_method_name (cfg);
1410                 debug_name = mono_aot_get_method_debug_name (cfg);
1411         } else {
1412                 init_jit_module ();
1413                 ctx->lmodule = &jit_module;
1414                 method_name = mono_method_full_name (cfg->method, TRUE);
1415                 debug_name = NULL;
1416         }
1417
1418         module = ctx->module = ctx->lmodule->module;
1419
1420 #if 1
1421         {
1422                 static int count = 0;
1423                 count ++;
1424
1425                 if (getenv ("LLVM_COUNT")) {
1426                         if (count == atoi (getenv ("LLVM_COUNT"))) {
1427                                 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
1428                                 last = TRUE;
1429                         }
1430                         if (count > atoi (getenv ("LLVM_COUNT")))
1431                                 LLVM_FAILURE (ctx, "");
1432                 }
1433         }
1434 #endif
1435
1436         sig = mono_method_signature (cfg->method);
1437         ctx->sig = sig;
1438
1439         linfo = mono_arch_get_llvm_call_info (cfg, sig);
1440         ctx->linfo = linfo;
1441         CHECK_FAILURE (ctx);
1442
1443         method_type = sig_to_llvm_sig (ctx, sig, linfo);
1444         CHECK_FAILURE (ctx);
1445
1446         method = LLVMAddFunction (module, method_name, method_type);
1447         ctx->lmethod = method;
1448
1449         LLVMSetLinkage (method, LLVMPrivateLinkage);
1450
1451         if (cfg->method->save_lmf)
1452                 LLVM_FAILURE (ctx, "lmf");
1453
1454         if (sig->pinvoke)
1455                 LLVM_FAILURE (ctx, "pinvoke signature");
1456
1457         header = mono_method_get_header (cfg->method);
1458         for (i = 0; i < header->num_clauses; ++i) {
1459                 clause = &header->clauses [i];
1460                 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_NONE)
1461                         LLVM_FAILURE (ctx, "non-finally/catch clause.");
1462         }
1463
1464         /*
1465          * This maps parameter indexes in the original signature to the indexes in
1466          * the LLVM signature.
1467          */
1468         pindexes = g_new0 (int, sig->param_count);
1469         pindex = 0;
1470         if (cfg->vret_addr) {
1471                 values [cfg->vret_addr->dreg] = LLVMGetParam (method, pindex);
1472                 pindex ++;
1473         }
1474         if (sig->hasthis) {
1475                 values [cfg->args [0]->dreg] = LLVMGetParam (method, pindex);
1476                 pindex ++;
1477         }
1478         for (i = 0; i < sig->param_count; ++i) {
1479                 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex);
1480                 pindexes [i] = pindex;
1481                 if (linfo->args [i + sig->hasthis].storage == LLVMArgVtypeInReg) {
1482                         if (linfo->args [i + sig->hasthis].pair_storage [0] != LLVMArgNone)
1483                                 pindex ++;
1484                         if (linfo->args [i + sig->hasthis].pair_storage [1] != LLVMArgNone)
1485                                 pindex ++;
1486                 } else if (linfo->args [i + sig->hasthis].storage == LLVMArgVtypeByVal) {
1487                         LLVMAddAttribute (LLVMGetParam (method, pindex), LLVMByValAttribute);
1488                         pindex ++;
1489                 } else {
1490                         pindex ++;
1491                 }
1492         }
1493
1494         max_block_num = 0;
1495         for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
1496                 max_block_num = MAX (max_block_num, bb->block_num);
1497         ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
1498
1499         /* Add branches between non-consecutive bblocks */
1500         for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
1501                 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
1502                         bb->next_bb != bb->last_ins->inst_false_bb) {
1503
1504                         MonoInst *inst = mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
1505                         inst->opcode = OP_BR;
1506                         inst->inst_target_bb = bb->last_ins->inst_false_bb;
1507                         mono_bblock_add_inst (bb, inst);
1508                 }
1509         }
1510
1511         /*
1512          * Make a first pass over the code to precreate PHI nodes.
1513          */
1514         for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
1515                 MonoInst *ins;
1516                 LLVMBuilderRef builder;
1517                 char *dname;
1518                 char dname_buf[128];
1519
1520                 builder = create_builder (ctx);
1521
1522                 for (ins = bb->code; ins; ins = ins->next) {
1523                         switch (ins->opcode) {
1524                         case OP_PHI:
1525                         case OP_FPHI:
1526                         case OP_VPHI:
1527                         case OP_XPHI: {
1528                                 LLVMTypeRef phi_type = llvm_type_to_stack_type (type_to_llvm_type (ctx, &ins->klass->byval_arg));
1529
1530                                 CHECK_FAILURE (ctx);
1531
1532                                 if (ins->opcode == OP_VPHI) {
1533                                         /* Treat valuetype PHI nodes as operating on the address itself */
1534                                         g_assert (ins->klass);
1535                                         phi_type = LLVMPointerType (type_to_llvm_type (ctx, &ins->klass->byval_arg), 0);
1536                                 }
1537
1538                                 /*
1539                                  * Have to precreate these, as they can be referenced by
1540                                  * earlier instructions.
1541                                  */
1542                                 sprintf (dname_buf, "t%d", ins->dreg);
1543                                 dname = dname_buf;
1544                                 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
1545
1546                                 if (ins->opcode == OP_VPHI)
1547                                         addresses [ins->dreg] = values [ins->dreg];
1548
1549                                 g_ptr_array_add (phi_values, values [ins->dreg]);
1550
1551                                 /*
1552                                  * Set the expected type of the incoming arguments since these have
1553                                  * to have the same type.
1554                                  */
1555                                 for (i = 0; i < ins->inst_phi_args [0]; i++) {
1556                                         int sreg1 = ins->inst_phi_args [i + 1];
1557
1558                                         if (sreg1 != -1)
1559                                                 vreg_types [sreg1] = phi_type;
1560                                 }
1561                                 break;
1562                                 }
1563                         default:
1564                                 break;
1565                         }
1566                 }
1567         }
1568
1569         /*
1570          * Create an ordering for bblocks, use the depth first order first, then
1571          * put the exception handling bblocks last.
1572          */
1573         for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
1574                 bb = cfg->bblocks [bb_index];
1575                 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
1576                         g_ptr_array_add (bblock_list, bb);
1577                         bblocks [bb->block_num].added = TRUE;
1578                 }
1579         }
1580
1581         for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
1582                 if (!bblocks [bb->block_num].added)
1583                         g_ptr_array_add (bblock_list, bb);
1584         }
1585
1586         /*
1587          * Second pass: generate code.
1588          */
1589         for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
1590                 MonoInst *ins;
1591                 LLVMBasicBlockRef cbb;
1592                 LLVMBuilderRef builder;
1593                 gboolean has_terminator;
1594                 LLVMValueRef v;
1595                 LLVMValueRef lhs, rhs;
1596
1597                 bb = g_ptr_array_index (bblock_list, bb_index);
1598
1599                 if (!(bb == cfg->bb_entry || bb->in_count > 0))
1600                         continue;
1601
1602                 cbb = get_bb (ctx, bb);
1603                 builder = create_builder (ctx);
1604                 ctx->builder = builder;
1605                 LLVMPositionBuilderAtEnd (builder, cbb);
1606
1607                 if (bb == cfg->bb_entry)
1608                         emit_entry_bb (ctx, builder, pindexes);
1609                 CHECK_FAILURE (ctx);
1610
1611                 if (bb->flags & BB_EXCEPTION_HANDLER) {
1612                         LLVMTypeRef i8ptr;
1613                         LLVMValueRef eh_selector, eh_exception, personality, args [4];
1614                         MonoInst *exvar;
1615                         static gint32 mapping_inited;
1616                         static int ti_generator;
1617                         char ti_name [128];
1618                         MonoClass **ti;
1619                         LLVMValueRef type_info;
1620                         int clause_index;
1621
1622                         if (!bblocks [bb->block_num].invoke_target) {
1623                                 /*
1624                                  * LLVM asserts if llvm.eh.selector is called from a bblock which
1625                                  * doesn't have an invoke pointing at it.
1626                                  */
1627                                 LLVM_FAILURE (ctx, "handler without invokes");
1628                         }
1629
1630                         eh_selector = LLVMGetNamedFunction (module, "llvm.eh.selector");
1631
1632                         if (cfg->compile_aot) {
1633                                 /* Use a dummy personality function */
1634                                 personality = LLVMGetNamedFunction (module, "mono_aot_personality");
1635                                 g_assert (personality);
1636                         } else {
1637                                 personality = LLVMGetNamedFunction (module, "mono_personality");
1638                                 if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0)
1639                                         LLVMAddGlobalMapping (ee, personality, mono_personality);
1640                         }
1641
1642                         i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
1643
1644                         clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
1645
1646                         /*
1647                          * Create the type info
1648                          */
1649                         sprintf (ti_name, "type_info_%d", ti_generator);
1650                         ti_generator ++;
1651
1652                         if (cfg->compile_aot) {
1653                                 /* decode_eh_frame () in aot-runtime.c will decode this */
1654                                 type_info = LLVMAddGlobal (module, LLVMInt32Type (), ti_name);
1655                                 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
1656
1657                                 LLVMSetLinkage (type_info, LLVMPrivateLinkage);
1658                                 LLVMSetVisibility (type_info, LLVMHiddenVisibility);
1659
1660                                 /*
1661                                  * Enabling this causes llc to crash:
1662                                  * http://llvm.org/bugs/show_bug.cgi?id=6102
1663                                  */
1664                                 LLVM_FAILURE (ctx, "aot+clauses");
1665                         } else {
1666                                 /* exception_cb will decode this */
1667                                 ti = g_malloc (sizeof (gint32));
1668                                 *(gint32*)ti = clause_index;
1669
1670                                 type_info = LLVMAddGlobal (module, i8ptr, ti_name);
1671
1672                                 LLVMAddGlobalMapping (ee, type_info, ti);
1673                         }
1674
1675                         args [0] = LLVMConstNull (i8ptr);
1676                         args [1] = LLVMConstBitCast (personality, i8ptr);
1677                         args [2] = type_info;
1678                         LLVMBuildCall (builder, eh_selector, args, 3, "");
1679
1680                         /* Store the exception into the exvar */
1681                         if (bb->in_scount == 1) {
1682                                 g_assert (bb->in_scount == 1);
1683                                 exvar = bb->in_stack [0];
1684
1685                                 eh_exception = LLVMGetNamedFunction (module, "llvm.eh.exception");
1686
1687                                 // FIXME: This is shared with filter clauses ?
1688                                 g_assert (!values [exvar->dreg]);
1689                                 values [exvar->dreg] = LLVMBuildCall (builder, eh_exception, NULL, 0, "");
1690                                 emit_volatile_store (ctx, exvar->dreg);
1691                         }
1692                 }
1693
1694                 has_terminator = FALSE;
1695                 for (ins = bb->code; ins; ins = ins->next) {
1696                         const char *spec = LLVM_INS_INFO (ins->opcode);
1697                         char *dname = NULL;
1698                         char dname_buf [128];
1699
1700                         if (has_terminator)
1701                                 /* There could be instructions after a terminator, skip them */
1702                                 break;
1703
1704                         if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
1705                                 sprintf (dname_buf, "t%d", ins->dreg);
1706                                 dname = dname_buf;
1707                         }
1708
1709                         if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
1710                                 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
1711
1712                                 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1713                                         lhs = emit_volatile_load (ctx, ins->sreg1);
1714                                 } else {
1715                                         /* It is ok for SETRET to have an uninitialized argument */
1716                                         if (!values [ins->sreg1] && ins->opcode != OP_SETRET)
1717                                                 LLVM_FAILURE (ctx, "sreg1");
1718                                         lhs = values [ins->sreg1];
1719                                 }
1720                         } else {
1721                                 lhs = NULL;
1722                         }
1723
1724                         if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
1725                                 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
1726                                 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1727                                         rhs = emit_volatile_load (ctx, ins->sreg2);
1728                                 } else {
1729                                         if (!values [ins->sreg2])
1730                                                 LLVM_FAILURE (ctx, "sreg2");
1731                                         rhs = values [ins->sreg2];
1732                                 }
1733                         } else {
1734                                 rhs = NULL;
1735                         }
1736
1737                         //mono_print_ins (ins);
1738                         switch (ins->opcode) {
1739                         case OP_NOP:
1740                         case OP_NOT_NULL:
1741                         case OP_LIVERANGE_START:
1742                         case OP_LIVERANGE_END:
1743                                 break;
1744                         case OP_ICONST:
1745                                 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
1746                                 break;
1747                         case OP_I8CONST:
1748 #if SIZEOF_VOID_P == 4
1749                                 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
1750 #else
1751                                 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
1752 #endif
1753                                 break;
1754                         case OP_R8CONST:
1755                                 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), *(double*)ins->inst_p0);
1756                                 break;
1757                         case OP_R4CONST:
1758                                 values [ins->dreg] = LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0), LLVMDoubleType ());
1759                                 break;
1760                         case OP_BR:
1761                                 LLVMBuildBr (builder, get_bb (ctx, ins->inst_target_bb));
1762                                 has_terminator = TRUE;
1763                                 break;
1764                         case OP_SWITCH: {
1765                                 int i;
1766                                 LLVMValueRef v;
1767                                 char bb_name [128];
1768                                 LLVMBasicBlockRef new_bb;
1769                                 LLVMBuilderRef new_builder;
1770
1771                                 // The default branch is already handled
1772                                 // FIXME: Handle it here
1773
1774                                 /* Start new bblock */
1775                                 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
1776                                 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1777
1778                                 lhs = convert (ctx, lhs, LLVMInt32Type ());
1779                                 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
1780                                 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
1781                                         MonoBasicBlock *target_bb = ins->inst_many_bb [i];
1782
1783                                         LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
1784                                 }
1785
1786                                 new_builder = create_builder (ctx);
1787                                 LLVMPositionBuilderAtEnd (new_builder, new_bb);
1788                                 LLVMBuildUnreachable (new_builder);
1789
1790                                 has_terminator = TRUE;
1791                                 g_assert (!ins->next);
1792
1793                                 break;
1794                         }
1795
1796                         case OP_SETRET:
1797                                 if (linfo->ret.storage == LLVMArgVtypeInReg) {
1798                                         LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
1799                                         LLVMValueRef part1, retval;
1800                                         int size;
1801
1802                                         size = get_vtype_size (sig->ret);
1803
1804                                         g_assert (addresses [ins->sreg1]);
1805
1806                                         g_assert (linfo->ret.pair_storage [0] == LLVMArgInIReg);
1807                                         g_assert (linfo->ret.pair_storage [1] == LLVMArgNone);
1808
1809                                         part1 = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMIntType (size * 8), 0), ""), ""), IntPtrType ());
1810
1811                                         retval = LLVMBuildInsertValue (builder, LLVMGetUndef (ret_type), part1, 0, "");
1812
1813                                         LLVMBuildRet (builder, retval);
1814                                         break;
1815                                 }
1816
1817                                 if (linfo->ret.storage == LLVMArgVtypeRetAddr) {
1818                                         LLVMBuildRetVoid (builder);
1819                                         break;
1820                                 }
1821
1822                                 if (!lhs || is_dead [ins->sreg1]) {
1823                                         /*
1824                                          * The method did not set its return value, probably because it
1825                                          * ends with a throw.
1826                                          */
1827                                         if (cfg->vret_addr)
1828                                                 LLVMBuildRetVoid (builder);
1829                                         else
1830                                                 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
1831                                 } else {
1832                                         LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
1833                                 }
1834                                 has_terminator = TRUE;
1835                                 break;
1836                         case OP_ICOMPARE:
1837                         case OP_FCOMPARE:
1838                         case OP_LCOMPARE:
1839                         case OP_COMPARE:
1840                         case OP_ICOMPARE_IMM:
1841                         case OP_LCOMPARE_IMM:
1842                         case OP_COMPARE_IMM:
1843 #ifdef TARGET_AMD64
1844                         case OP_AMD64_ICOMPARE_MEMBASE_REG:
1845                         case OP_AMD64_ICOMPARE_MEMBASE_IMM:
1846 #endif
1847 #ifdef TARGET_X86
1848                         case OP_X86_COMPARE_MEMBASE_REG:
1849                         case OP_X86_COMPARE_MEMBASE_IMM:
1850 #endif
1851                         {
1852                                 CompRelation rel;
1853                                 LLVMValueRef cmp;
1854
1855                                 if (ins->next->opcode == OP_NOP)
1856                                         break;
1857
1858                                 if (ins->next->opcode == OP_BR)
1859                                         /* The comparison result is not needed */
1860                                         continue;
1861
1862                                 rel = mono_opcode_to_cond (ins->next->opcode);
1863
1864                                 /* Used for implementing bound checks */
1865 #ifdef TARGET_AMD64
1866                                 if ((ins->opcode == OP_AMD64_ICOMPARE_MEMBASE_REG) || (ins->opcode == OP_AMD64_ICOMPARE_MEMBASE_IMM)) {
1867                                         int size = 4;
1868                                         LLVMValueRef index;
1869                                         LLVMTypeRef t;
1870
1871                                         t = LLVMInt32Type ();
1872
1873                                         g_assert (ins->inst_offset % size == 0);
1874                                         index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
1875
1876                                         lhs = LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (t, 0)), &index, 1, ""), "");
1877                                 }
1878                                 if (ins->opcode == OP_AMD64_ICOMPARE_MEMBASE_IMM) {
1879                                         lhs = convert (ctx, lhs, LLVMInt32Type ());
1880                                         rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
1881                                 }
1882                                 if (ins->opcode == OP_AMD64_ICOMPARE_MEMBASE_REG)
1883                                         rhs = convert (ctx, rhs, LLVMInt32Type ());
1884 #endif
1885
1886 #ifdef TARGET_X86
1887                                 if ((ins->opcode == OP_X86_COMPARE_MEMBASE_REG) || (ins->opcode == OP_X86_COMPARE_MEMBASE_IMM)) {
1888                                         int size = 4;
1889                                         LLVMValueRef index;
1890                                         LLVMTypeRef t;
1891
1892                                         t = LLVMInt32Type ();
1893
1894                                         g_assert (ins->inst_offset % size == 0);
1895                                         index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
1896
1897                                         lhs = LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (t, 0)), &index, 1, ""), "");
1898                                 }
1899                                 if (ins->opcode == OP_X86_COMPARE_MEMBASE_IMM) {
1900                                         lhs = convert (ctx, lhs, LLVMInt32Type ());
1901                                         rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
1902                                 }
1903                                 if (ins->opcode == OP_X86_COMPARE_MEMBASE_REG)
1904                                         rhs = convert (ctx, rhs, LLVMInt32Type ());
1905 #endif
1906
1907                                 if (ins->opcode == OP_ICOMPARE_IMM) {
1908                                         lhs = convert (ctx, lhs, LLVMInt32Type ());
1909                                         rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
1910                                 }
1911                                 if (ins->opcode == OP_LCOMPARE_IMM) {
1912                                         lhs = convert (ctx, lhs, LLVMInt64Type ());
1913                                         rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
1914                                 }
1915                                 if (ins->opcode == OP_LCOMPARE) {
1916                                         lhs = convert (ctx, lhs, LLVMInt64Type ());
1917                                         rhs = convert (ctx, rhs, LLVMInt64Type ());
1918                                 }
1919                                 if (ins->opcode == OP_ICOMPARE) {
1920                                         lhs = convert (ctx, lhs, LLVMInt32Type ());
1921                                         rhs = convert (ctx, rhs, LLVMInt32Type ());
1922                                 }
1923
1924                                 if (lhs && rhs) {
1925                                         if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
1926                                                 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
1927                                         else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
1928                                                 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
1929                                 }
1930
1931                                 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
1932                                 if (ins->opcode == OP_FCOMPARE)
1933                                         cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
1934                                 else if (ins->opcode == OP_COMPARE_IMM)
1935                                         cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
1936                                 else if (ins->opcode == OP_COMPARE)
1937                                         cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
1938                                 else
1939                                         cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
1940
1941                                 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
1942                                         LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
1943                                         has_terminator = TRUE;
1944                                 } else if (MONO_IS_SETCC (ins->next)) {
1945                                         sprintf (dname_buf, "t%d", ins->next->dreg);
1946                                         dname = dname_buf;
1947                                         values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
1948
1949                                         /* Add stores for volatile variables */
1950                                         emit_volatile_store (ctx, ins->next->dreg);
1951                                 } else if (MONO_IS_COND_EXC (ins->next)) {
1952                                         //emit_cond_throw_pos (ctx);
1953                                         emit_cond_system_exception (ctx, bb, ins->next->inst_p1, cmp);
1954                                         builder = ctx->builder;
1955                                 } else {
1956                                         LLVM_FAILURE (ctx, "next");
1957                                 }
1958
1959                                 ins = ins->next;
1960                                 break;
1961                         }
1962                         case OP_FCEQ:
1963                         case OP_FCLT:
1964                         case OP_FCLT_UN:
1965                         case OP_FCGT:
1966                         case OP_FCGT_UN: {
1967                                 CompRelation rel;
1968                                 LLVMValueRef cmp;
1969
1970                                 rel = mono_opcode_to_cond (ins->opcode);
1971
1972                                 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
1973                                 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
1974                                 break;
1975                         }
1976                         case OP_PHI:
1977                         case OP_FPHI:
1978                         case OP_VPHI:
1979                         case OP_XPHI: {
1980                                 int i;
1981                                 gboolean empty = TRUE;
1982
1983                                 /* Check that all input bblocks really branch to us */
1984                                 for (i = 0; i < bb->in_count; ++i) {
1985                                         if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
1986                                                 ins->inst_phi_args [i + 1] = -1;
1987                                         else
1988                                                 empty = FALSE;
1989                                 }
1990
1991                                 if (empty) {
1992                                         /* LLVM doesn't like phi instructions with zero operands */
1993                                         is_dead [ins->dreg] = TRUE;
1994                                         break;
1995                                 }
1996
1997                                 /* Created earlier, insert it now */
1998                                 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
1999
2000                                 for (i = 0; i < ins->inst_phi_args [0]; i++) {
2001                                         int sreg1 = ins->inst_phi_args [i + 1];
2002                                         int count, j;
2003
2004                                         /*
2005                                          * Count the number of times the incoming bblock branches to us,
2006                                          * since llvm requires a separate entry for each.
2007                                          */
2008                                         if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
2009                                                 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
2010
2011                                                 count = 0;
2012                                                 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
2013                                                         if (switch_ins->inst_many_bb [j] == bb)
2014                                                                 count ++;
2015                                                 }
2016                                         } else {
2017                                                 count = 1;
2018                                         }
2019
2020                                         /* Remember for later */
2021                                         for (j = 0; j < count; ++j) {
2022                                                 PhiNode *node = mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
2023                                                 node->bb = bb;
2024                                                 node->phi = ins;
2025                                                 node->in_bb = bb->in_bb [i];
2026                                                 node->sreg = sreg1;
2027                                                 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
2028                                         }
2029                                 }
2030                                 break;
2031                         }
2032                         case OP_MOVE:
2033                         case OP_LMOVE:
2034                         case OP_XMOVE:
2035                                 g_assert (lhs);
2036                                 values [ins->dreg] = lhs;
2037                                 break;
2038                         case OP_FMOVE: {
2039                                 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
2040
2041                                 g_assert (lhs);
2042                                 values [ins->dreg] = lhs;
2043
2044                                 if (var && var->klass->byval_arg.type == MONO_TYPE_R4) {
2045                                         /*
2046                                          * This is added by the spilling pass in case of the JIT,
2047                                          * but we have to do it ourselves.
2048                                          */
2049                                         values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
2050                                 }
2051                                 break;
2052                         }
2053                         case OP_IADD:
2054                         case OP_ISUB:
2055                         case OP_IAND:
2056                         case OP_IMUL:
2057                         case OP_IDIV:
2058                         case OP_IDIV_UN:
2059                         case OP_IREM:
2060                         case OP_IREM_UN:
2061                         case OP_IOR:
2062                         case OP_IXOR:
2063                         case OP_ISHL:
2064                         case OP_ISHR:
2065                         case OP_ISHR_UN:
2066                         case OP_FADD:
2067                         case OP_FSUB:
2068                         case OP_FMUL:
2069                         case OP_FDIV:
2070                         case OP_LADD:
2071                         case OP_LSUB:
2072                         case OP_LMUL:
2073                         case OP_LDIV:
2074                         case OP_LDIV_UN:
2075                         case OP_LREM:
2076                         case OP_LREM_UN:
2077                         case OP_LAND:
2078                         case OP_LOR:
2079                         case OP_LXOR:
2080                         case OP_LSHL:
2081                         case OP_LSHR:
2082                         case OP_LSHR_UN:
2083                                 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
2084                                 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
2085
2086                                 switch (ins->opcode) {
2087                                 case OP_IADD:
2088                                 case OP_FADD:
2089                                 case OP_LADD:
2090                                         values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
2091                                         break;
2092                                 case OP_ISUB:
2093                                 case OP_FSUB:
2094                                 case OP_LSUB:
2095                                         values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
2096                                         break;
2097                                 case OP_IMUL:
2098                                 case OP_FMUL:
2099                                 case OP_LMUL:
2100                                         values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
2101                                         break;
2102                                 case OP_IREM:
2103                                 case OP_LREM:
2104                                         values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
2105                                         break;
2106                                 case OP_IREM_UN:
2107                                 case OP_LREM_UN:
2108                                         values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
2109                                         break;
2110                                 case OP_IDIV:
2111                                 case OP_LDIV:
2112                                         values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
2113                                         break;
2114                                 case OP_IDIV_UN:
2115                                 case OP_LDIV_UN:
2116                                         values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
2117                                         break;
2118                                 case OP_FDIV:
2119                                         values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
2120                                         break;
2121                                 case OP_IAND:
2122                                 case OP_LAND:
2123                                         values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
2124                                         break;
2125                                 case OP_IOR:
2126                                 case OP_LOR:
2127                                         values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
2128                                         break;
2129                                 case OP_IXOR:
2130                                 case OP_LXOR:
2131                                         values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
2132                                         break;
2133                                 case OP_ISHL:
2134                                 case OP_LSHL:
2135                                         values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
2136                                         break;
2137                                 case OP_ISHR:
2138                                 case OP_LSHR:
2139                                         values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
2140                                         break;
2141                                 case OP_ISHR_UN:
2142                                 case OP_LSHR_UN:
2143                                         values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
2144                                         break;
2145                                 default:
2146                                         g_assert_not_reached ();
2147                                 }
2148                                 break;
2149                         case OP_IADD_IMM:
2150                         case OP_ISUB_IMM:
2151                         case OP_IMUL_IMM:
2152                         case OP_IREM_IMM:
2153                         case OP_IREM_UN_IMM:
2154                         case OP_IDIV_IMM:
2155                         case OP_IDIV_UN_IMM:
2156                         case OP_IAND_IMM:
2157                         case OP_IOR_IMM:
2158                         case OP_IXOR_IMM:
2159                         case OP_ISHL_IMM:
2160                         case OP_ISHR_IMM:
2161                         case OP_ISHR_UN_IMM:
2162                         case OP_LADD_IMM:
2163                         case OP_LSUB_IMM:
2164                         case OP_LREM_IMM:
2165                         case OP_LAND_IMM:
2166                         case OP_LOR_IMM:
2167                         case OP_LXOR_IMM:
2168                         case OP_LSHL_IMM:
2169                         case OP_LSHR_IMM:
2170                         case OP_LSHR_UN_IMM:
2171                         case OP_ADD_IMM:
2172                         case OP_AND_IMM:
2173                         case OP_MUL_IMM:
2174                         case OP_SHL_IMM:
2175                         case OP_SHR_IMM: {
2176                                 LLVMValueRef imm;
2177
2178                                 if (spec [MONO_INST_SRC1] == 'l') {
2179                                         imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
2180                                 } else {
2181                                         imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
2182                                 }
2183
2184 #if SIZEOF_VOID_P == 4
2185                                 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
2186                                         imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
2187 #endif
2188
2189                                 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
2190                                         lhs = convert (ctx, lhs, IntPtrType ());
2191                                 imm = convert (ctx, imm, LLVMTypeOf (lhs));
2192                                 switch (ins->opcode) {
2193                                 case OP_IADD_IMM:
2194                                 case OP_LADD_IMM:
2195                                 case OP_ADD_IMM:
2196                                         values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
2197                                         break;
2198                                 case OP_ISUB_IMM:
2199                                 case OP_LSUB_IMM:
2200                                         values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
2201                                         break;
2202                                 case OP_IMUL_IMM:
2203                                 case OP_MUL_IMM:
2204                                         values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
2205                                         break;
2206                                 case OP_IDIV_IMM:
2207                                 case OP_LDIV_IMM:
2208                                         values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
2209                                         break;
2210                                 case OP_IDIV_UN_IMM:
2211                                 case OP_LDIV_UN_IMM:
2212                                         values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
2213                                         break;
2214                                 case OP_IREM_IMM:
2215                                 case OP_LREM_IMM:
2216                                         values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
2217                                         break;
2218                                 case OP_IREM_UN_IMM:
2219                                         values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
2220                                         break;
2221                                 case OP_IAND_IMM:
2222                                 case OP_LAND_IMM:
2223                                 case OP_AND_IMM:
2224                                         values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
2225                                         break;
2226                                 case OP_IOR_IMM:
2227                                 case OP_LOR_IMM:
2228                                         values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
2229                                         break;
2230                                 case OP_IXOR_IMM:
2231                                 case OP_LXOR_IMM:
2232                                         values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
2233                                         break;
2234                                 case OP_ISHL_IMM:
2235                                 case OP_LSHL_IMM:
2236                                 case OP_SHL_IMM:
2237                                         values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
2238                                         break;
2239                                 case OP_ISHR_IMM:
2240                                 case OP_LSHR_IMM:
2241                                 case OP_SHR_IMM:
2242                                         values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
2243                                         break;
2244                                 case OP_ISHR_UN_IMM:
2245                                         /* This is used to implement conv.u4, so the lhs could be an i8 */
2246                                         lhs = convert (ctx, lhs, LLVMInt32Type ());
2247                                         imm = convert (ctx, imm, LLVMInt32Type ());
2248                                         values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
2249                                         break;
2250                                 case OP_LSHR_UN_IMM:
2251                                         values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
2252                                         break;
2253                                 default:
2254                                         g_assert_not_reached ();
2255                                 }
2256                                 break;
2257                         }
2258                         case OP_INEG:
2259                                 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
2260                                 break;
2261                         case OP_LNEG:
2262                                 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
2263                                 break;
2264                         case OP_FNEG:
2265                                 lhs = convert (ctx, lhs, LLVMDoubleType ());
2266                                 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
2267                                 break;
2268                         case OP_INOT: {
2269                                 guint32 v = 0xffffffff;
2270                                 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), lhs, dname);
2271                                 break;
2272                         }
2273                         case OP_LNOT: {
2274                                 guint64 v = 0xffffffffffffffffLL;
2275                                 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
2276                                 break;
2277                         }
2278 #if defined(TARGET_X86) || defined(TARGET_AMD64)
2279                         case OP_X86_LEA: {
2280                                 LLVMValueRef v1, v2;
2281
2282                                 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), (1 << ins->backend.shift_amount), FALSE), "");
2283                                 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
2284                                 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
2285                                 break;
2286                         }
2287 #endif
2288
2289                         case OP_ICONV_TO_I1:
2290                         case OP_ICONV_TO_I2:
2291                         case OP_ICONV_TO_I4:
2292                         case OP_ICONV_TO_U1:
2293                         case OP_ICONV_TO_U2:
2294                         case OP_ICONV_TO_U4:
2295                         case OP_LCONV_TO_I1:
2296                         case OP_LCONV_TO_I2:
2297                         case OP_LCONV_TO_U1:
2298                         case OP_LCONV_TO_U2:
2299                         case OP_LCONV_TO_U4: {
2300                                 gboolean sign;
2301
2302                                 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
2303
2304                                 /* Have to do two casts since our vregs have type int */
2305                                 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
2306                                 if (sign)
2307                                         values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
2308                                 else
2309                                         values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
2310                                 break;
2311                         }
2312                         case OP_ICONV_TO_I8:
2313                                 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
2314                                 break;
2315                         case OP_ICONV_TO_U8:
2316                                 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
2317                                 break;
2318                         case OP_FCONV_TO_I4:
2319                                 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
2320                                 break;
2321                         case OP_FCONV_TO_I1:
2322                                 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
2323                                 break;
2324                         case OP_FCONV_TO_U1:
2325                                 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
2326                                 break;
2327                         case OP_FCONV_TO_I2:
2328                                 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
2329                                 break;
2330                         case OP_FCONV_TO_U2:
2331                                 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
2332                                 break;
2333                         case OP_FCONV_TO_I8:
2334                                 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
2335                                 break;
2336                         case OP_FCONV_TO_I:
2337                                 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
2338                                 break;
2339                         case OP_ICONV_TO_R8:
2340                         case OP_LCONV_TO_R8:
2341                                 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
2342                                 break;
2343                         case OP_LCONV_TO_R_UN:
2344                                 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
2345                                 break;
2346 #if SIZEOF_VOID_P == 4
2347                         case OP_LCONV_TO_U:
2348 #endif
2349                         case OP_LCONV_TO_I4:
2350                                 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
2351                                 break;
2352                         case OP_ICONV_TO_R4:
2353                         case OP_LCONV_TO_R4:
2354                                 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
2355                                 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
2356                                 break;
2357                         case OP_FCONV_TO_R4:
2358                                 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
2359                                 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
2360                                 break;
2361                         case OP_SEXT_I4:
2362                                 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
2363                                 break;
2364                         case OP_ZEXT_I4:
2365                                 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
2366                                 break;
2367                         case OP_TRUNC_I4:
2368                                 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
2369                                 break;
2370                         case OP_LOCALLOC_IMM: {
2371                                 LLVMValueRef v;
2372
2373                                 guint32 size = ins->inst_imm;
2374                                 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
2375
2376                                 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
2377
2378                                 if (ins->flags & MONO_INST_INIT) {
2379                                         LLVMValueRef args [4];
2380
2381                                         args [0] = v;
2382                                         args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
2383                                         args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE);
2384                                         args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
2385                                         LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memset.i32"), args, 4, "");
2386                                 }
2387
2388                                 values [ins->dreg] = v;
2389                                 break;
2390                         }
2391                         case OP_LOCALLOC: {
2392                                 LLVMValueRef v, size;
2393
2394                                 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, lhs, LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
2395
2396                                 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
2397
2398                                 if (ins->flags & MONO_INST_INIT) {
2399                                         LLVMValueRef args [4];
2400
2401                                         args [0] = v;
2402                                         args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
2403                                         args [2] = size;
2404                                         args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
2405                                         LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memset.i32"), args, 4, "");
2406                                 }
2407                                 values [ins->dreg] = v;
2408                                 break;
2409                         }
2410
2411                         case OP_LOADI1_MEMBASE:
2412                         case OP_LOADU1_MEMBASE:
2413                         case OP_LOADI2_MEMBASE:
2414                         case OP_LOADU2_MEMBASE:
2415                         case OP_LOADI4_MEMBASE:
2416                         case OP_LOADU4_MEMBASE:
2417                         case OP_LOADI8_MEMBASE:
2418                         case OP_LOADR4_MEMBASE:
2419                         case OP_LOADR8_MEMBASE:
2420                         case OP_LOAD_MEMBASE:
2421                         case OP_LOADI8_MEM:
2422                         case OP_LOADU1_MEM:
2423                         case OP_LOADU2_MEM:
2424                         case OP_LOADI4_MEM:
2425                         case OP_LOADU4_MEM:
2426                         case OP_LOAD_MEM: {
2427                                 int size = 8;
2428                                 LLVMValueRef index, addr;
2429                                 LLVMTypeRef t;
2430                                 gboolean sext = FALSE, zext = FALSE;
2431                                 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
2432
2433                                 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
2434
2435                                 if (sext || zext)
2436                                         dname = (char*)"";
2437
2438                                 /*
2439                                  * We emit volatile loads for loads which can fault, because otherwise
2440                                  * LLVM will generate invalid code when encountering a load from a
2441                                  * NULL address.
2442                                  */
2443                                 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
2444                                         addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
2445                                 } else if (ins->inst_offset == 0) {
2446                                         addr = values [ins->inst_basereg];
2447                                 } else if (ins->inst_offset % size != 0) {
2448                                         /* Unaligned load */
2449                                         index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
2450                                         addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
2451                                 } else {
2452                                         index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
2453                                         addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (t, 0)), &index, 1, "");
2454                                 }
2455
2456                                 addr = convert (ctx, addr, LLVMPointerType (t, 0));
2457
2458                                 values [ins->dreg] = mono_llvm_build_load (builder, addr, dname, is_volatile);
2459
2460                                 if (sext)
2461                                         values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
2462                                 else if (zext)
2463                                         values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
2464                                 else if (ins->opcode == OP_LOADR4_MEMBASE)
2465                                         values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
2466                                 break;
2467                         }
2468
2469                         case OP_STOREI1_MEMBASE_REG:
2470                         case OP_STOREI2_MEMBASE_REG:
2471                         case OP_STOREI4_MEMBASE_REG:
2472                         case OP_STOREI8_MEMBASE_REG:
2473                         case OP_STORER4_MEMBASE_REG:
2474                         case OP_STORER8_MEMBASE_REG:
2475                         case OP_STORE_MEMBASE_REG: {
2476                                 int size = 8;
2477                                 LLVMValueRef index, addr;
2478                                 LLVMTypeRef t;
2479                                 gboolean sext = FALSE, zext = FALSE;
2480
2481                                 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
2482
2483                                 if (ins->inst_offset % size != 0) {
2484                                         /* Unaligned store */
2485                                         index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
2486                                         addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
2487                                 } else {
2488                                         index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
2489                                         addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
2490                                 }
2491                                 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)));
2492                                 break;
2493                         }
2494
2495                         case OP_STOREI1_MEMBASE_IMM:
2496                         case OP_STOREI2_MEMBASE_IMM:
2497                         case OP_STOREI4_MEMBASE_IMM:
2498                         case OP_STOREI8_MEMBASE_IMM:
2499                         case OP_STORE_MEMBASE_IMM: {
2500                                 int size = 8;
2501                                 LLVMValueRef index, addr;
2502                                 LLVMTypeRef t;
2503                                 gboolean sext = FALSE, zext = FALSE;
2504
2505                                 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
2506
2507                                 if (ins->inst_offset % size != 0) {
2508                                         /* Unaligned store */
2509                                         index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
2510                                         addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
2511                                 } else {
2512                                         index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
2513                                         addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
2514                                 }
2515                                 LLVMBuildStore (builder, convert (ctx, LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE), t), addr);
2516                                 break;
2517                         }
2518
2519                         case OP_CHECK_THIS:
2520                                 mono_llvm_build_load (builder, convert (ctx, values [ins->sreg1], LLVMPointerType (IntPtrType (), 0)), "", TRUE);
2521                                 break;
2522                         case OP_OUTARG_VTRETADDR:
2523                                 break;
2524                         case OP_VOIDCALL:
2525                         case OP_CALL:
2526                         case OP_LCALL:
2527                         case OP_FCALL:
2528                         case OP_VCALL:
2529                         case OP_VOIDCALL_MEMBASE:
2530                         case OP_CALL_MEMBASE:
2531                         case OP_LCALL_MEMBASE:
2532                         case OP_FCALL_MEMBASE:
2533                         case OP_VCALL_MEMBASE:
2534                         case OP_VOIDCALL_REG:
2535                         case OP_CALL_REG:
2536                         case OP_LCALL_REG:
2537                         case OP_FCALL_REG:
2538                         case OP_VCALL_REG: {
2539                                 MonoCallInst *call = (MonoCallInst*)ins;
2540                                 MonoMethodSignature *sig = call->signature;
2541                                 LLVMValueRef callee, lcall;
2542                                 LLVMValueRef *args;
2543                                 LLVMCallInfo *cinfo;
2544                                 GSList *l;
2545                                 int i, pindex;
2546                                 gboolean vretaddr;
2547                                 LLVMTypeRef llvm_sig;
2548                                 gpointer target;
2549                                 int *pindexes;
2550                                 gboolean virtual, calli;
2551
2552                                 cinfo = call->cinfo;
2553
2554                                 vretaddr = cinfo && cinfo->ret.storage == LLVMArgVtypeRetAddr;
2555
2556                                 llvm_sig = sig_to_llvm_sig (ctx, sig, cinfo);
2557                                 CHECK_FAILURE (ctx);
2558
2559                                 virtual = (ins->opcode == OP_VOIDCALL_MEMBASE || ins->opcode == OP_CALL_MEMBASE || ins->opcode == OP_VCALL_MEMBASE || ins->opcode == OP_LCALL_MEMBASE || ins->opcode == OP_FCALL_MEMBASE);
2560                                 calli = (ins->opcode == OP_VOIDCALL_REG || ins->opcode == OP_CALL_REG || ins->opcode == OP_VCALL_REG || ins->opcode == OP_LCALL_REG || ins->opcode == OP_FCALL_REG);
2561
2562                                 pindexes = mono_mempool_alloc0 (cfg->mempool, (sig->param_count + 2) * sizeof (guint32));
2563
2564                                 /* FIXME: Avoid creating duplicate methods */
2565
2566                                 if (ins->flags & MONO_INST_HAS_METHOD) {
2567                                         if (virtual) {
2568                                                 callee = NULL;
2569                                         } else {
2570                                                 if (cfg->compile_aot) {
2571                                                         callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
2572                                                         if (!callee)
2573                                                                 LLVM_FAILURE (ctx, "can't encode patch");
2574                                                 } else {
2575                                                         callee = LLVMAddFunction (module, "", llvm_sig);
2576
2577                                                         target =
2578                                                                 mono_create_jit_trampoline_in_domain (mono_domain_get (),
2579                                                                                                                                           call->method);
2580                                                         LLVMAddGlobalMapping (ee, callee, target);
2581                                                 }
2582                                         }
2583                                 } else if (calli) {
2584                                 } else {
2585                                         MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr);
2586
2587                                         if (info) {
2588                                                 /*
2589                                                 MonoJumpInfo ji;
2590
2591                                                 memset (&ji, 0, sizeof (ji));
2592                                                 ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR;
2593                                                 ji.data.target = info->name;
2594
2595                                                 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
2596                                                 */
2597                                                 if (cfg->compile_aot) {
2598                                                         callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_INTERNAL_METHOD, (char*)info->name);
2599                                                         if (!callee)
2600                                                                 LLVM_FAILURE (ctx, "can't encode patch");
2601                                                 } else {
2602                                                         callee = LLVMAddFunction (module, "", llvm_sig);
2603                                                         target = (gpointer)mono_icall_get_wrapper (info);
2604                                                         LLVMAddGlobalMapping (ee, callee, target);
2605                                                 }
2606                                         } else {
2607                                                 if (cfg->compile_aot) {
2608                                                         callee = NULL;
2609                                                         if (cfg->abs_patches) {
2610                                                                 MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr);
2611                                                                 if (abs_ji) {
2612                                                                         callee = get_plt_entry (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
2613                                                                         if (!callee)
2614                                                                                 LLVM_FAILURE (ctx, "can't encode patch");
2615                                                                 }
2616                                                         }
2617                                                         if (!callee)
2618                                                                 LLVM_FAILURE (ctx, "aot");
2619                                                 } else {
2620                                                         callee = LLVMAddFunction (module, "", llvm_sig);
2621                                                         target = NULL;
2622                                                         if (cfg->abs_patches) {
2623                                                                 MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr);
2624                                                                 if (abs_ji) {
2625                                                                         /*
2626                                                                          * The monitor entry/exit trampolines might have
2627                                                                          * their own calling convention on some platforms.
2628                                                                          */
2629 #ifndef TARGET_AMD64
2630                                                                         if (abs_ji->type == MONO_PATCH_INFO_MONITOR_ENTER || abs_ji->type == MONO_PATCH_INFO_MONITOR_EXIT)
2631                                                                                 LLVM_FAILURE (ctx, "monitor enter/exit");
2632 #endif
2633                                                                         target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE);
2634                                                                         LLVMAddGlobalMapping (ee, callee, target);
2635                                                                 }
2636                                                         }
2637                                                         if (!target)
2638                                                                 LLVMAddGlobalMapping (ee, callee, (gpointer)call->fptr);
2639                                                 }
2640                                         }
2641                                 }
2642
2643                                 if (virtual) {
2644                                         int size = sizeof (gpointer);
2645                                         LLVMValueRef index;
2646
2647                                         g_assert (ins->inst_offset % size == 0);
2648                                         index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
2649
2650                                         // FIXME: mono_arch_get_vcall_slot () can't decode the code
2651                                         // generated by LLVM
2652                                         //LLVM_FAILURE (ctx, "virtual call");
2653
2654                                         if (call->method && call->method->klass->flags & TYPE_ATTRIBUTE_INTERFACE) {
2655 #ifdef MONO_ARCH_HAVE_LLVM_IMT_TRAMPOLINE
2656                                                 if (cfg->compile_aot) {
2657                                                         MonoJumpInfoImtTramp *imt_tramp = g_new0 (MonoJumpInfoImtTramp, 1);
2658                                                         imt_tramp->method = call->method;
2659                                                         imt_tramp->vt_offset = call->inst.inst_offset;
2660
2661                                                         callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_LLVM_IMT_TRAMPOLINE, imt_tramp);
2662                                                 } else {
2663                                                         callee = LLVMAddFunction (module, "", llvm_sig);
2664                                                         target = mono_create_llvm_imt_trampoline (cfg->domain, call->method, call->inst.inst_offset);
2665                                                         LLVMAddGlobalMapping (ee, callee, target);
2666                                                 }
2667 #else
2668                                                 /* No support for passing the IMT argument */
2669                                                 LLVM_FAILURE (ctx, "imt");
2670 #endif
2671                                         } else {
2672                                                 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
2673                                         }
2674                                 } else if (calli) {
2675                                         callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
2676                                 } else {
2677                                         if (ins->flags & MONO_INST_HAS_METHOD) {
2678                                         }
2679                                 }
2680
2681                                 /*
2682                                  * Collect and convert arguments
2683                                  */
2684
2685                                 args = alloca (sizeof (LLVMValueRef) * ((sig->param_count * 2) + sig->hasthis + vretaddr));
2686                                 l = call->out_ireg_args;
2687                                 pindex = 0;
2688                                 if (vretaddr) {
2689                                         if (!addresses [call->inst.dreg])
2690                                                 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
2691                                         args [pindex ++] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
2692                                 }
2693
2694                                 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
2695                                         guint32 regpair;
2696                                         int reg;
2697                                         LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i] : NULL;
2698
2699                                         regpair = (guint32)(gssize)(l->data);
2700                                         reg = regpair & 0xffffff;
2701                                         args [pindex] = values [reg];
2702                                         if (ainfo->storage == LLVMArgVtypeInReg) {
2703                                                 int j;
2704                                                 LLVMValueRef regs [2];
2705                                                 guint32 nregs;
2706
2707                                                 g_assert (ainfo);
2708
2709                                                 g_assert (addresses [reg]);
2710
2711                                                 emit_vtype_to_reg (ctx, builder, sig->params [i - sig->hasthis], addresses [reg], ainfo, regs, &nregs);
2712                                                 for (j = 0; j < nregs; ++j)
2713                                                         args [pindex ++] = regs [j];
2714
2715                                                 // FIXME: alignment
2716                                                 // FIXME: Get rid of the VMOVE
2717                                         } else if (ainfo->storage == LLVMArgVtypeByVal) {
2718                                                 g_assert (addresses [reg]);
2719                                                 args [pindex] = addresses [reg];
2720                                                 pindexes [i] = pindex;
2721                                                 pindex ++;
2722                                         } else {
2723                                                 g_assert (args [pindex]);
2724                                                 if (i == 0 && sig->hasthis)
2725                                                         args [pindex] = convert (ctx, args [pindex], IntPtrType ());
2726                                                 else
2727                                                         args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, sig->params [i - sig->hasthis]));
2728                                                 pindex ++;
2729                                         }
2730
2731                                         l = l->next;
2732                                 }
2733
2734                                 // FIXME: Align call sites
2735
2736                                 /*
2737                                  * Emit the call
2738                                  */
2739
2740                                 lcall = emit_call (ctx, bb, &builder, callee, args, pindex);
2741
2742                                 /* Add byval attributes if needed */
2743                                 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
2744                                         LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i] : NULL;
2745
2746                                         if (ainfo && ainfo->storage == LLVMArgVtypeByVal) {
2747                                                 LLVMAddInstrAttribute (lcall, pindexes [i] + 1, LLVMByValAttribute);
2748                                         }
2749                                 }
2750
2751                                 /*
2752                                  * Convert the result
2753                                  */
2754                                 if (cinfo && cinfo->ret.storage == LLVMArgVtypeInReg) {
2755                                         LLVMValueRef regs [2];
2756
2757                                         if (!addresses [ins->dreg])
2758                                                 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
2759
2760                                         regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
2761                                         if (cinfo->ret.pair_storage [1] != LLVMArgNone)
2762                                                 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
2763
2764                                         emit_reg_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
2765                                 } else if (sig->ret->type != MONO_TYPE_VOID && !vretaddr) {
2766                                         /* If the method returns an unsigned value, need to zext it */
2767
2768                                         values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
2769                                 }
2770                                 break;
2771                         }
2772                         case OP_AOTCONST: {
2773                                 guint32 got_offset;
2774                                 LLVMValueRef indexes [2];
2775                                 MonoJumpInfo *ji;
2776                                 LLVMValueRef got_entry_addr;
2777
2778                                 /*
2779                                  * FIXME: Can't allocate from the cfg mempool since that is freed if
2780                                  * the LLVM compile fails.
2781                                  */
2782                                 ji = g_new0 (MonoJumpInfo, 1);
2783                                 ji->type = (MonoJumpInfoType)ins->inst_i1;
2784                                 ji->data.target = ins->inst_p0;
2785
2786                                 ji = mono_aot_patch_info_dup (ji);
2787
2788                                 ji->next = cfg->patch_info;
2789                                 cfg->patch_info = ji;
2790
2791                                 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
2792                                 got_offset = mono_aot_get_got_offset (cfg->patch_info);
2793
2794                                 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2795                                 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
2796                                 got_entry_addr = LLVMBuildGEP (builder, ctx->lmodule->got_var, indexes, 2, "");
2797
2798                                 // FIXME: This doesn't work right now, because it must be
2799                                 // paired with an invariant.end, and even then, its only in effect
2800                                 // inside its basic block
2801 #if 0
2802                                 {
2803                                         LLVMValueRef args [3];
2804                                         LLVMValueRef ptr, val;
2805
2806                                         ptr = LLVMBuildBitCast (builder, got_entry_addr, LLVMPointerType (LLVMInt8Type (), 0), "ptr");
2807
2808                                         args [0] = LLVMConstInt (LLVMInt64Type (), sizeof (gpointer), FALSE);
2809                                         args [1] = ptr;
2810                                         val = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.invariant.start"), args, 2, "");
2811                                 }
2812 #endif
2813
2814                                 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, dname);
2815                                 break;
2816                         }
2817                         case OP_NOT_REACHED:
2818                                 LLVMBuildUnreachable (builder);
2819                                 has_terminator = TRUE;
2820                                 g_assert (bb->block_num < cfg->max_block_num);
2821                                 unreachable [bb->block_num] = TRUE;
2822                                 /* Might have instructions after this */
2823                                 while (ins->next) {
2824                                         MonoInst *next = ins->next;
2825                                         MONO_DELETE_INS (bb, next);
2826                                 }
2827                                 break;
2828                         case OP_LDADDR: {
2829                                 MonoInst *var = ins->inst_p0;
2830
2831                                 values [ins->dreg] = addresses [var->dreg];
2832                                 break;
2833                         }
2834                         case OP_SIN: {
2835                                 LLVMValueRef args [1];
2836
2837                                 args [0] = convert (ctx, lhs, LLVMDoubleType ());
2838                                 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sin.f64"), args, 1, dname);
2839                                 break;
2840                         }
2841                         case OP_COS: {
2842                                 LLVMValueRef args [1];
2843
2844                                 args [0] = convert (ctx, lhs, LLVMDoubleType ());
2845                                 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.cos.f64"), args, 1, dname);
2846                                 break;
2847                         }
2848                                 /* test_0_sqrt_nan fails with LLVM */
2849                                 /*
2850                         case OP_SQRT: {
2851                                 LLVMValueRef args [1];
2852
2853                                 args [0] = lhs;
2854                                 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sqrt.f64"), args, 1, dname);
2855                                 break;
2856                         }
2857                                 */
2858
2859                         case OP_ABS: {
2860                                 LLVMValueRef args [1];
2861
2862                                 args [0] = lhs;
2863                                 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "fabs"), args, 1, dname);
2864                                 break;
2865                         }
2866
2867                         case OP_IMIN:
2868                         case OP_LMIN: {
2869                                 LLVMValueRef v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
2870                                 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
2871                                 break;
2872                         }
2873                         case OP_IMAX:
2874                         case OP_LMAX: {
2875                                 LLVMValueRef v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
2876                                 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
2877                                 break;
2878                         }
2879                         case OP_IMIN_UN:
2880                         case OP_LMIN_UN: {
2881                                 LLVMValueRef v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
2882                                 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
2883                                 break;
2884                         }
2885                         case OP_IMAX_UN:
2886                         case OP_LMAX_UN: {
2887                                 LLVMValueRef v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
2888                                 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
2889                                 break;
2890                         }
2891                         case OP_ATOMIC_EXCHANGE_I4: {
2892                                 LLVMValueRef args [2];
2893
2894                                 g_assert (ins->inst_offset == 0);
2895
2896                                 args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt32Type (), 0));
2897                                 args [1] = rhs;
2898                                 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.swap.i32.p0i32"), args, 2, dname);
2899                                 break;
2900                         }
2901                         case OP_ATOMIC_EXCHANGE_I8: {
2902                                 LLVMValueRef args [2];
2903
2904                                 g_assert (ins->inst_offset == 0);
2905
2906                                 args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt64Type (), 0));
2907                                 args [1] = rhs;
2908                                 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.swap.i64.p0i64"), args, 2, dname);
2909                                 break;
2910                         }
2911                         case OP_ATOMIC_ADD_NEW_I4: {
2912                                 LLVMValueRef args [2];
2913
2914                                 g_assert (ins->inst_offset == 0);
2915
2916                                 args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt32Type (), 0));
2917                                 args [1] = rhs;
2918                                 values [ins->dreg] = LLVMBuildAdd (builder, LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.load.add.i32.p0i32"), args, 2, ""), args [1], dname);
2919                                 break;
2920                         }
2921                         case OP_ATOMIC_ADD_NEW_I8: {
2922                                 LLVMValueRef args [2];
2923
2924                                 g_assert (ins->inst_offset == 0);
2925
2926                                 args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt64Type (), 0));
2927                                 args [1] = convert (ctx, rhs, LLVMInt64Type ());
2928                                 values [ins->dreg] = LLVMBuildAdd (builder, LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.load.add.i64.p0i64"), args, 2, ""), args [1], dname);
2929                                 break;
2930                         }
2931                         case OP_ATOMIC_CAS_I4:
2932                         case OP_ATOMIC_CAS_I8: {
2933                                 LLVMValueRef args [3];
2934                                 LLVMTypeRef t;
2935                                 const char *intrins;
2936
2937                                 if (ins->opcode == OP_ATOMIC_CAS_I4) {
2938                                         t = LLVMInt32Type ();
2939                                         intrins = "llvm.atomic.cmp.swap.i32.p0i32";
2940                                 } else {
2941                                         t = LLVMInt64Type ();
2942                                         intrins = "llvm.atomic.cmp.swap.i64.p0i64";
2943                                 }
2944
2945                                 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
2946                                 /* comparand */
2947                                 args [1] = convert (ctx, values [ins->sreg3], t);
2948                                 /* new value */
2949                                 args [2] = convert (ctx, values [ins->sreg2], t);
2950                                 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, intrins), args, 3, dname);
2951                                 break;
2952                         }
2953                         case OP_MEMORY_BARRIER: {
2954                                 LLVMValueRef args [5];
2955
2956                                 for (i = 0; i < 5; ++i)
2957                                         args [i] = LLVMConstInt (LLVMInt1Type (), TRUE, TRUE);
2958
2959                                 LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memory.barrier"), args, 5, "");
2960                                 break;
2961                         }
2962                         case OP_RELAXED_NOP: {
2963 #if defined(TARGET_AMD64) || defined(TARGET_X86)
2964                                 /* No way to get LLVM to emit this */
2965                                 LLVM_FAILURE (ctx, "relaxed_nop");
2966 #else
2967                                 break;
2968 #endif
2969                         }
2970                         case OP_TLS_GET: {
2971 #if defined(TARGET_AMD64) || defined(TARGET_X86)
2972 #ifdef TARGET_AMD64
2973                                 // 257 == FS segment register
2974                                 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
2975 #else
2976                                 // 256 == GS segment register
2977                                 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
2978 #endif
2979
2980                                 // FIXME: XEN
2981                                 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
2982 #else
2983                                 LLVM_FAILURE (ctx, "opcode tls-get");
2984 #endif
2985
2986                                 break;
2987                         }
2988
2989                         /*
2990                          * Overflow opcodes.
2991                          */
2992                         case OP_IADD_OVF:
2993                         case OP_IADD_OVF_UN:
2994                         case OP_ISUB_OVF:
2995                         case OP_ISUB_OVF_UN:
2996                         case OP_IMUL_OVF:
2997                         case OP_IMUL_OVF_UN:
2998 #if SIZEOF_VOID_P == 8
2999                         case OP_LADD_OVF:
3000                         case OP_LADD_OVF_UN:
3001                         case OP_LSUB_OVF:
3002                         case OP_LSUB_OVF_UN:
3003                         case OP_LMUL_OVF:
3004                         case OP_LMUL_OVF_UN:
3005 #endif
3006                         {
3007                                 LLVMValueRef args [2], val, ovf, func;
3008
3009                                 emit_cond_throw_pos (ctx);
3010
3011                                 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
3012                                 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
3013                                 func = LLVMGetNamedFunction (module, ovf_op_to_intrins (ins->opcode));
3014                                 g_assert (func);
3015                                 val = LLVMBuildCall (builder, func, args, 2, "");
3016                                 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
3017                                 ovf = LLVMBuildExtractValue (builder, val, 1, "");
3018                                 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
3019                                 builder = ctx->builder;
3020                                 break;
3021                         }
3022
3023                         /*
3024                          * Valuetypes.
3025                          *   We currently model them using arrays. Promotion to local vregs is
3026                          * disabled for them in mono_handle_global_vregs () in the LLVM case,
3027                          * so we always have an entry in cfg->varinfo for them.
3028                          * FIXME: Is this needed ?
3029                          */
3030                         case OP_VZERO: {
3031                                 MonoClass *klass = ins->klass;
3032                                 LLVMValueRef args [4];
3033
3034                                 if (!klass) {
3035                                         // FIXME:
3036                                         LLVM_FAILURE (ctx, "!klass");
3037                                         break;
3038                                 }
3039
3040                                 if (!addresses [ins->dreg])
3041                                         addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
3042                                 args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
3043                                 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
3044                                 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
3045                                 // FIXME: Alignment
3046                                 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3047                                 LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memset.i32"), args, 4, "");
3048                                 break;
3049                         }
3050
3051                         case OP_STOREV_MEMBASE:
3052                         case OP_LOADV_MEMBASE:
3053                         case OP_VMOVE: {
3054                                 MonoClass *klass = ins->klass;
3055                                 LLVMValueRef src, dst, args [4];
3056                                 gboolean done = FALSE;
3057
3058                                 if (!klass) {
3059                                         // FIXME:
3060                                         LLVM_FAILURE (ctx, "!klass");
3061                                         break;
3062                                 }
3063
3064                                 switch (ins->opcode) {
3065                                 case OP_STOREV_MEMBASE:
3066                                         if (!addresses [ins->sreg1]) {
3067                                                 /* SIMD */
3068                                                 g_assert (values [ins->sreg1]);
3069                                                 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, &klass->byval_arg), 0));
3070                                                 LLVMBuildStore (builder, values [ins->sreg1], dst);
3071                                                 done = TRUE;
3072                                         } else {
3073                                                 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
3074                                                 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
3075                                         }
3076                                         break;
3077                                 case OP_LOADV_MEMBASE:
3078                                         if (!addresses [ins->dreg])
3079                                                 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
3080                                         src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
3081                                         dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
3082                                         break;
3083                                 case OP_VMOVE:
3084                                         if (!addresses [ins->sreg1])
3085                                                 addresses [ins->sreg1] = build_alloca (ctx, &klass->byval_arg);
3086                                         if (!addresses [ins->dreg])
3087                                                 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
3088                                         src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
3089                                         dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
3090                                         break;
3091                                 default:
3092                                         g_assert_not_reached ();
3093                                 }
3094
3095                                 if (done)
3096                                         break;
3097
3098                                 args [0] = dst;
3099                                 args [1] = src;
3100                                 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
3101                                 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3102                                 // FIXME: Alignment
3103                                 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3104                                 LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memcpy.i32"), args, 4, "");
3105                                 break;
3106                         }
3107                         case OP_LLVM_OUTARG_VT:
3108                                 if (!addresses [ins->sreg1]) {
3109                                         addresses [ins->sreg1] = build_alloca (ctx, &ins->klass->byval_arg);
3110                                         g_assert (values [ins->sreg1]);
3111                                         LLVMBuildStore (builder, values [ins->sreg1], addresses [ins->sreg1]);
3112                                 }
3113                                 addresses [ins->dreg] = addresses [ins->sreg1];
3114                                 break;
3115
3116                         /*
3117                          * SIMD
3118                          */
3119 #if defined(TARGET_X86) || defined(TARGET_AMD64)
3120                         case OP_XZERO: {
3121                                 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, &ins->klass->byval_arg));
3122                                 break;
3123                         }
3124                         case OP_LOADX_MEMBASE: {
3125                                 LLVMTypeRef t = type_to_llvm_type (ctx, &ins->klass->byval_arg);
3126                                 LLVMValueRef src;
3127
3128                                 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
3129                                 values [ins->dreg] = LLVMBuildLoad (builder, src, "");
3130                                 break;
3131                         }
3132                         case OP_ADDPD:
3133                         case OP_ADDPS:
3134                         case OP_PADDB:
3135                         case OP_PADDW:
3136                         case OP_PADDD:
3137                         case OP_PADDQ:
3138                                 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
3139                                 break;
3140                         case OP_SUBPD:
3141                         case OP_SUBPS:
3142                         case OP_PSUBB:
3143                         case OP_PSUBW:
3144                         case OP_PSUBD:
3145                         case OP_PSUBQ:
3146                                 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
3147                                 break;
3148                         case OP_MULPD:
3149                         case OP_MULPS:
3150                                 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
3151                                 break;
3152                         case OP_DIVPD:
3153                         case OP_DIVPS:
3154                                 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
3155                                 break;
3156                         case OP_PAND:
3157                                 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
3158                                 break;
3159                         case OP_POR:
3160                                 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
3161                                 break;
3162                         case OP_PXOR:
3163                                 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
3164                                 break;
3165                         case OP_ANDPS:
3166                         case OP_ANDNPS:
3167                         case OP_ORPS:
3168                         case OP_XORPS:
3169                         case OP_ANDPD:
3170                         case OP_ANDNPD:
3171                         case OP_ORPD:
3172                         case OP_XORPD: {
3173                                 LLVMTypeRef t, rt;
3174                                 LLVMValueRef v;
3175
3176                                 switch (ins->opcode) {
3177                                 case OP_ANDPS:
3178                                 case OP_ANDNPS:
3179                                 case OP_ORPS:
3180                                 case OP_XORPS:
3181                                         t = LLVMVectorType (LLVMInt32Type (), 4);
3182                                         rt = LLVMVectorType (LLVMFloatType (), 4);
3183                                         break;
3184                                 case OP_ANDPD:
3185                                 case OP_ANDNPD:
3186                                 case OP_ORPD:
3187                                 case OP_XORPD:
3188                                         t = LLVMVectorType (LLVMInt64Type (), 2);
3189                                         rt = LLVMVectorType (LLVMDoubleType (), 2);
3190                                         break;
3191                                 default:
3192                                         t = LLVMInt32Type ();
3193                                         rt = LLVMInt32Type ();
3194                                         g_assert_not_reached ();
3195                                 }
3196
3197                                 lhs = LLVMBuildBitCast (builder, lhs, t, "");
3198                                 rhs = LLVMBuildBitCast (builder, rhs, t, "");
3199                                 switch (ins->opcode) {
3200                                 case OP_ANDPS:
3201                                 case OP_ANDPD:
3202                                         v = LLVMBuildAnd (builder, lhs, rhs, "");
3203                                         break;
3204                                 case OP_ORPS:
3205                                 case OP_ORPD:
3206                                         v = LLVMBuildOr (builder, lhs, rhs, "");
3207                                         break;
3208                                 case OP_XORPS:
3209                                 case OP_XORPD:
3210                                         v = LLVMBuildXor (builder, lhs, rhs, "");
3211                                         break;
3212                                 case OP_ANDNPS:
3213                                 case OP_ANDNPD:
3214                                         v = LLVMBuildAnd (builder, lhs, LLVMBuildNot (builder, rhs, ""), "");
3215                                         break;
3216                                 }
3217                                 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
3218                                 break;
3219                         }
3220                         case OP_MINPD:
3221                         case OP_MINPS:
3222                         case OP_MAXPD:
3223                         case OP_MAXPS:
3224                         case OP_PMIND_UN:
3225                         case OP_PMINW_UN:
3226                         case OP_PMINB_UN:
3227                         case OP_PMAXD_UN:
3228                         case OP_PMAXW_UN:
3229                         case OP_PMAXB_UN: {
3230                                 LLVMValueRef args [2];
3231
3232                                 args [0] = lhs;
3233                                 args [1] = rhs;
3234
3235                                 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 2, dname);
3236                                 break;
3237                         }
3238                         case OP_EXTRACT_R8:
3239                         case OP_EXTRACT_I8:
3240                         case OP_EXTRACT_I4:
3241                         case OP_EXTRACT_I2:
3242                         case OP_EXTRACT_U2:
3243                         case OP_EXTRACT_I1:
3244                         case OP_EXTRACT_U1: {
3245                                 LLVMTypeRef t;
3246
3247                                 switch (ins->opcode) {
3248                                 case OP_EXTRACT_R8:
3249                                         t = LLVMVectorType (LLVMDoubleType (), 2);
3250                                         break;
3251                                 case OP_EXTRACT_I8:
3252                                         t = LLVMVectorType (LLVMInt64Type (), 2);
3253                                         break;
3254                                 case OP_EXTRACT_I4:
3255                                         t = LLVMVectorType (LLVMInt32Type (), 4);
3256                                         break;
3257                                 case OP_EXTRACT_I2:
3258                                 case OP_EXTRACT_U2:
3259                                         t = LLVMVectorType (LLVMInt16Type (), 8);
3260                                         break;
3261                                 case OP_EXTRACT_I1:
3262                                 case OP_EXTRACT_U1:
3263                                         t = LLVMVectorType (LLVMInt8Type (), 16);
3264                                         break;
3265                                 default:
3266                                         t = LLVMInt32Type ();
3267                                         g_assert_not_reached ();
3268                                 }
3269
3270                                 lhs = LLVMBuildBitCast (builder, lhs, t, "");
3271                                 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
3272                                 break;
3273                         }
3274 #endif
3275
3276                         case OP_DUMMY_USE:
3277                                 break;
3278
3279                         /*
3280                          * EXCEPTION HANDLING
3281                          */
3282                         case OP_IMPLICIT_EXCEPTION:
3283                                 /* This marks a place where an implicit exception can happen */
3284                                 if (bb->region != -1)
3285                                         LLVM_FAILURE (ctx, "implicit-exception");
3286                                 break;
3287                         case OP_THROW: {
3288                                 MonoMethodSignature *throw_sig;
3289                                 LLVMValueRef callee, arg;
3290
3291                                 if (!ctx->lmodule->throw) {
3292                                         throw_sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
3293                                         throw_sig->ret = &mono_defaults.void_class->byval_arg;
3294                                         throw_sig->params [0] = &mono_defaults.object_class->byval_arg;
3295                                         if (cfg->compile_aot) {
3296                                                 callee = get_plt_entry (ctx, sig_to_llvm_sig (ctx, throw_sig, NULL), MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_exception");
3297                                         } else {
3298                                                 callee = LLVMAddFunction (module, "mono_arch_throw_exception", sig_to_llvm_sig (ctx, throw_sig, NULL));
3299
3300 #ifdef TARGET_X86
3301                                                 /*
3302                                                  * LLVM doesn't push the exception argument, so we need a different
3303                                                  * trampoline.
3304                                                  */
3305                                                 LLVMAddGlobalMapping (ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_llvm_throw_exception"));
3306 #else
3307                                                 LLVMAddGlobalMapping (ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_exception"));
3308 #endif
3309                                         }
3310
3311                                         mono_memory_barrier ();
3312                                         ctx->lmodule->throw = callee;
3313                                 }
3314                                 arg = convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, &mono_defaults.object_class->byval_arg));
3315                                 emit_call (ctx, bb, &builder, ctx->lmodule->throw, &arg, 1);
3316                                 break;
3317                         }
3318                         case OP_CALL_HANDLER: {
3319                                 /*
3320                                  * We don't 'call' handlers, but instead simply branch to them.
3321                                  * The code generated by ENDFINALLY will branch back to us.
3322                                  */
3323                                 LLVMBasicBlockRef finally_bb, noex_bb;
3324                                 GSList *bb_list;
3325
3326                                 finally_bb = get_bb (ctx, ins->inst_target_bb);
3327
3328                                 bb_list = bblocks [ins->inst_target_bb->block_num].call_handler_return_bbs;
3329
3330                                 /*
3331                                  * Set the indicator variable for the finally clause.
3332                                  */
3333                                 lhs = bblocks [ins->inst_target_bb->block_num].finally_ind;
3334                                 g_assert (lhs);
3335                                 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
3336
3337                                 /* Branch to the finally clause */
3338                                 LLVMBuildBr (builder, finally_bb);
3339
3340                                 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
3341                                 // FIXME: Use a mempool
3342                                 bblocks [ins->inst_target_bb->block_num].call_handler_return_bbs = g_slist_append (bblocks [ins->inst_target_bb->block_num].call_handler_return_bbs, noex_bb);
3343
3344                                 builder = ctx->builder = create_builder (ctx);
3345                                 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
3346
3347                                 bblocks [bb->block_num].end_bblock = noex_bb;
3348                                 break;
3349                         }
3350                         case OP_START_HANDLER: {
3351                                 break;
3352                         }
3353                         case OP_ENDFINALLY: {
3354                                 LLVMBasicBlockRef resume_bb;
3355                                 MonoBasicBlock *handler_bb;
3356                                 LLVMValueRef val, switch_ins;
3357                                 GSList *bb_list;
3358
3359                                 handler_bb = g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
3360                                 g_assert (handler_bb);
3361                                 lhs = bblocks [handler_bb->block_num].finally_ind;
3362                                 g_assert (lhs);
3363
3364                                 bb_list = bblocks [handler_bb->block_num].call_handler_return_bbs;
3365
3366                                 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
3367
3368                                 /* Load the finally variable */
3369                                 val = LLVMBuildLoad (builder, lhs, "");
3370
3371                                 /* Reset the variable */
3372                                 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
3373
3374                                 /* Branch to either resume_bb, or to the bblocks in bb_list */
3375                                 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
3376                                 /*
3377                                  * The other targets are added at the end to handle OP_CALL_HANDLER
3378                                  * opcodes processed later.
3379                                  */
3380                                 bblocks [handler_bb->block_num].endfinally_switch = switch_ins;
3381                                 /*
3382                                 for (i = 0; i < g_slist_length (bb_list); ++i)
3383                                         LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), g_slist_nth (bb_list, i)->data);
3384                                 */
3385
3386                                 builder = ctx->builder = create_builder (ctx);
3387                                 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
3388
3389                                 LLVMBuildCall (builder, LLVMGetNamedFunction (module, "mono_resume_unwind"), NULL, 0, "");
3390                                 LLVMBuildUnreachable (builder);
3391                                 has_terminator = TRUE;
3392                                 break;
3393                         }
3394                         default: {
3395                                 char reason [128];
3396
3397                                 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
3398                                 LLVM_FAILURE (ctx, reason);
3399                                 break;
3400                         }
3401                         }
3402
3403                         /* Convert the value to the type required by phi nodes */
3404                         if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && vreg_types [ins->dreg]) {
3405                                 if (!values [ins->dreg])
3406                                         /* vtypes */
3407                                         values [ins->dreg] = addresses [ins->dreg];
3408                                 else
3409                                         values [ins->dreg] = convert (ctx, values [ins->dreg], vreg_types [ins->dreg]);
3410                         }
3411
3412                         /* Add stores for volatile variables */
3413                         if (spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
3414                                 emit_volatile_store (ctx, ins->dreg);
3415                 }
3416
3417                 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0))
3418                         LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
3419
3420                 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID)
3421                         LLVMBuildRetVoid (builder);
3422
3423                 if (bb == cfg->bb_entry)
3424                         ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
3425         }
3426
3427         /* Add incoming phi values */
3428         for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3429                 GSList *l, *ins_list;
3430
3431                 ins_list = bblocks [bb->block_num].phi_nodes;
3432
3433                 for (l = ins_list; l; l = l->next) {
3434                         PhiNode *node = l->data;
3435                         MonoInst *phi = node->phi;
3436                         int sreg1 = node->sreg;
3437                         LLVMBasicBlockRef in_bb;
3438
3439                         if (sreg1 == -1)
3440                                 continue;
3441
3442                         in_bb = get_end_bb (ctx, node->in_bb);
3443
3444                         if (unreachable [node->in_bb->block_num])
3445                                 continue;
3446
3447                         g_assert (values [sreg1]);
3448
3449                         g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
3450                         LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
3451                 }
3452         }
3453
3454         /* Create the SWITCH statements for ENDFINALLY instructions */
3455         for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3456                 if (bblocks [bb->block_num].endfinally_switch) {
3457                         LLVMValueRef switch_ins = bblocks [bb->block_num].endfinally_switch;
3458                         GSList *bb_list = bblocks [bb->block_num].call_handler_return_bbs;
3459
3460                         for (i = 0; i < g_slist_length (bb_list); ++i)
3461                                 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), g_slist_nth (bb_list, i)->data);
3462                 }
3463         }
3464
3465         if (cfg->verbose_level > 1)
3466                 mono_llvm_dump_value (method);
3467
3468         mark_as_used (module, method);
3469
3470         if (cfg->compile_aot) {
3471                 /* Don't generate native code, keep the LLVM IR */
3472
3473                 /* Can't delete the method if it has an alias, so only add it if successful */
3474                 if (debug_name) {
3475                         debug_alias = LLVMAddAlias (module, LLVMTypeOf (method), method, debug_name);
3476                         LLVMSetLinkage (debug_alias, LLVMInternalLinkage);
3477                         LLVMSetVisibility (debug_alias, LLVMHiddenVisibility);
3478                 }
3479
3480                 if (cfg->compile_aot && cfg->verbose_level)
3481                         printf ("%s emitted as %s\n", mono_method_full_name (cfg->method, TRUE), method_name);
3482
3483                 //LLVMVerifyFunction(method, 0);
3484         } else {
3485                 mono_llvm_optimize_method (method);
3486
3487                 if (cfg->verbose_level > 1)
3488                         mono_llvm_dump_value (method);
3489
3490                 cfg->native_code = LLVMGetPointerToGlobal (ee, method);
3491
3492                 /* Set by emit_cb */
3493                 g_assert (cfg->code_len);
3494
3495                 /* FIXME: Free the LLVM IL for the function */
3496         }
3497
3498         goto CLEANUP;
3499
3500  FAILURE:
3501
3502         if (method) {
3503                 /* Need to add unused phi nodes as they can be referenced by other values */
3504                 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (method, "PHI_BB");
3505                 LLVMBuilderRef builder;
3506
3507                 builder = create_builder (ctx);
3508                 LLVMPositionBuilderAtEnd (builder, phi_bb);
3509
3510                 for (i = 0; i < phi_values->len; ++i) {
3511                         LLVMValueRef v = g_ptr_array_index (phi_values, i);
3512                         if (LLVMGetInstructionParent (v) == NULL)
3513                                 LLVMInsertIntoBuilder (builder, v);
3514                 }
3515
3516                 LLVMDeleteFunction (method);
3517         }
3518
3519  CLEANUP:
3520         g_free (values);
3521         g_free (addresses);
3522         g_free (vreg_types);
3523         g_free (vreg_cli_types);
3524         g_free (pindexes);
3525         g_free (debug_name);
3526         g_ptr_array_free (phi_values, TRUE);
3527         g_free (ctx->bblocks);
3528         g_hash_table_destroy (ctx->region_to_handler);
3529         g_free (method_name);
3530         g_ptr_array_free (bblock_list, TRUE);
3531
3532         for (l = ctx->builders; l; l = l->next) {
3533                 LLVMBuilderRef builder = l->data;
3534                 LLVMDisposeBuilder (builder);
3535         }
3536
3537         g_free (ctx);
3538
3539         TlsSetValue (current_cfg_tls_id, NULL);
3540
3541         mono_loader_unlock ();
3542 }
3543
3544 /*
3545  * mono_llvm_emit_call:
3546  *
3547  *   Same as mono_arch_emit_call () for LLVM.
3548  */
3549 void
3550 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
3551 {
3552         MonoInst *in;
3553         MonoMethodSignature *sig;
3554         int i, n, stack_size;
3555         LLVMArgInfo *ainfo;
3556
3557         stack_size = 0;
3558
3559         sig = call->signature;
3560         n = sig->param_count + sig->hasthis;
3561
3562         call->cinfo = mono_arch_get_llvm_call_info (cfg, sig);
3563
3564         if (cfg->disable_llvm)
3565                 return;
3566
3567         if (sig->call_convention == MONO_CALL_VARARG) {
3568                 cfg->exception_message = g_strdup ("varargs");
3569                 cfg->disable_llvm = TRUE;
3570         }
3571
3572         for (i = 0; i < n; ++i) {
3573                 MonoInst *ins;
3574
3575                 ainfo = call->cinfo->args + i;
3576
3577                 in = call->args [i];
3578
3579                 /* Simply remember the arguments */
3580                 switch (ainfo->storage) {
3581                 case LLVMArgInIReg:
3582                         MONO_INST_NEW (cfg, ins, OP_MOVE);
3583                         ins->dreg = mono_alloc_ireg (cfg);
3584                         ins->sreg1 = in->dreg;
3585                         break;
3586                 case LLVMArgInFPReg:
3587                         MONO_INST_NEW (cfg, ins, OP_FMOVE);
3588                         ins->dreg = mono_alloc_freg (cfg);
3589                         ins->sreg1 = in->dreg;
3590                         break;
3591                 case LLVMArgVtypeByVal:
3592                 case LLVMArgVtypeInReg:
3593                         MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
3594                         ins->dreg = mono_alloc_ireg (cfg);
3595                         ins->sreg1 = in->dreg;
3596                         ins->klass = mono_class_from_mono_type (sig->params [i - sig->hasthis]);
3597                         break;
3598                 default:
3599                         call->cinfo = mono_arch_get_llvm_call_info (cfg, sig);
3600                         cfg->exception_message = g_strdup ("ainfo->storage");
3601                         cfg->disable_llvm = TRUE;
3602                         return;
3603                 }
3604
3605                 if (!cfg->disable_llvm) {
3606                         MONO_ADD_INS (cfg->cbb, ins);
3607                         mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
3608                 }
3609         }
3610 }
3611
3612 static unsigned char*
3613 alloc_cb (LLVMValueRef function, int size)
3614 {
3615         MonoCompile *cfg;
3616
3617         cfg = TlsGetValue (current_cfg_tls_id);
3618
3619         if (cfg) {
3620                 // FIXME: dynamic
3621                 return mono_domain_code_reserve (cfg->domain, size);
3622         } else {
3623                 return mono_domain_code_reserve (mono_domain_get (), size);
3624         }
3625 }
3626
3627 static void
3628 emitted_cb (LLVMValueRef function, void *start, void *end)
3629 {
3630         MonoCompile *cfg;
3631
3632         cfg = TlsGetValue (current_cfg_tls_id);
3633         g_assert (cfg);
3634         cfg->code_len = (guint8*)end - (guint8*)start;
3635 }
3636
3637 static void
3638 exception_cb (void *data)
3639 {
3640         MonoCompile *cfg;
3641         MonoJitExceptionInfo *ei;
3642         guint32 ei_len, i;
3643         gpointer *type_info;
3644
3645         cfg = TlsGetValue (current_cfg_tls_id);
3646         g_assert (cfg);
3647
3648         /*
3649          * data points to a DWARF FDE structure, convert it to our unwind format and
3650          * save it.
3651          * An alternative would be to save it directly, and modify our unwinder to work
3652          * with it.
3653          */
3654         cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info);
3655
3656         cfg->llvm_ex_info = mono_mempool_alloc0 (cfg->mempool, ei_len * sizeof (MonoJitExceptionInfo));
3657         cfg->llvm_ex_info_len = ei_len;
3658         memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
3659         /* Fill the rest of the information from the type info */
3660         for (i = 0; i < ei_len; ++i) {
3661                 gint32 clause_index = *(gint32*)type_info [i];
3662                 MonoExceptionClause *clause = &mono_method_get_header (cfg->method)->clauses [clause_index];
3663
3664                 cfg->llvm_ex_info [i].flags = clause->flags;
3665                 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
3666         }
3667
3668         g_free (ei);
3669 }
3670
3671 static void
3672 add_intrinsics (LLVMModuleRef module)
3673 {
3674         /* Emit declarations of instrinsics */
3675         {
3676                 LLVMTypeRef memset_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type () };
3677
3678                 LLVMAddFunction (module, "llvm.memset.i32", LLVMFunctionType (LLVMVoidType (), memset_params, 4, FALSE));
3679         }
3680
3681         {
3682                 LLVMTypeRef memcpy_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type () };
3683
3684                 LLVMAddFunction (module, "llvm.memcpy.i32", LLVMFunctionType (LLVMVoidType (), memcpy_params, 4, FALSE));
3685         }
3686
3687         {
3688                 LLVMTypeRef params [] = { LLVMDoubleType () };
3689
3690                 LLVMAddFunction (module, "llvm.sin.f64", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
3691                 LLVMAddFunction (module, "llvm.cos.f64", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
3692                 LLVMAddFunction (module, "llvm.sqrt.f64", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
3693
3694                 /* This isn't an intrinsic, instead llvm seems to special case it by name */
3695                 LLVMAddFunction (module, "fabs", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
3696         }
3697
3698         {
3699                 LLVMTypeRef membar_params [] = { LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type () };
3700
3701                 LLVMAddFunction (module, "llvm.atomic.swap.i32.p0i32", LLVMFunctionType2 (LLVMInt32Type (), LLVMPointerType (LLVMInt32Type (), 0), LLVMInt32Type (), FALSE));
3702                 LLVMAddFunction (module, "llvm.atomic.swap.i64.p0i64", LLVMFunctionType2 (LLVMInt64Type (), LLVMPointerType (LLVMInt64Type (), 0), LLVMInt64Type (), FALSE));
3703                 LLVMAddFunction (module, "llvm.atomic.load.add.i32.p0i32", LLVMFunctionType2 (LLVMInt32Type (), LLVMPointerType (LLVMInt32Type (), 0), LLVMInt32Type (), FALSE));
3704                 LLVMAddFunction (module, "llvm.atomic.load.add.i64.p0i64", LLVMFunctionType2 (LLVMInt64Type (), LLVMPointerType (LLVMInt64Type (), 0), LLVMInt64Type (), FALSE));
3705                 LLVMAddFunction (module, "llvm.atomic.cmp.swap.i32.p0i32", LLVMFunctionType3 (LLVMInt32Type (), LLVMPointerType (LLVMInt32Type (), 0), LLVMInt32Type (), LLVMInt32Type (), FALSE));
3706                 LLVMAddFunction (module, "llvm.atomic.cmp.swap.i64.p0i64", LLVMFunctionType3 (LLVMInt64Type (), LLVMPointerType (LLVMInt64Type (), 0), LLVMInt64Type (), LLVMInt64Type (), FALSE));
3707                 LLVMAddFunction (module, "llvm.memory.barrier", LLVMFunctionType (LLVMVoidType (), membar_params, 5, FALSE));
3708         }
3709
3710         {
3711                 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
3712                 LLVMTypeRef ovf_params_i32 [] = { LLVMInt32Type (), LLVMInt32Type () };
3713
3714                 LLVMAddFunction (module, "llvm.sadd.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
3715                 LLVMAddFunction (module, "llvm.uadd.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
3716                 LLVMAddFunction (module, "llvm.ssub.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
3717                 LLVMAddFunction (module, "llvm.usub.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
3718                 LLVMAddFunction (module, "llvm.smul.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
3719                 LLVMAddFunction (module, "llvm.umul.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
3720         }
3721
3722         {
3723                 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
3724                 LLVMTypeRef ovf_params_i64 [] = { LLVMInt64Type (), LLVMInt64Type () };
3725
3726                 LLVMAddFunction (module, "llvm.sadd.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
3727                 LLVMAddFunction (module, "llvm.uadd.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
3728                 LLVMAddFunction (module, "llvm.ssub.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
3729                 LLVMAddFunction (module, "llvm.usub.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
3730                 LLVMAddFunction (module, "llvm.smul.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
3731                 LLVMAddFunction (module, "llvm.umul.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
3732         }
3733
3734         {
3735                 LLVMTypeRef struct_ptr = LLVMPointerType (LLVMStructType (NULL, 0, FALSE), 0);
3736                 LLVMTypeRef invariant_start_params [] = { LLVMInt64Type (), LLVMPointerType (LLVMInt8Type (), 0) };
3737                 LLVMTypeRef invariant_end_params [] = { struct_ptr, LLVMInt64Type (), LLVMPointerType (LLVMInt8Type (), 0) };
3738
3739                 LLVMAddFunction (module, "llvm.invariant.start", LLVMFunctionType (struct_ptr, invariant_start_params, 2, FALSE));
3740
3741                 LLVMAddFunction (module, "llvm.invariant.end", LLVMFunctionType (LLVMVoidType (), invariant_end_params, 3, FALSE));
3742         }
3743
3744         /* EH intrinsics */
3745         {
3746                 LLVMTypeRef arg_types [2];
3747
3748                 arg_types [0] = LLVMPointerType (LLVMInt8Type (), 0);
3749                 arg_types [1] = LLVMPointerType (LLVMInt8Type (), 0);
3750                 LLVMAddFunction (module, "llvm.eh.selector", LLVMFunctionType (LLVMInt32Type (), arg_types, 2, TRUE));
3751
3752                 LLVMAddFunction (module, "llvm.eh.exception", LLVMFunctionType (LLVMPointerType (LLVMInt8Type (), 0), NULL, 0, FALSE));
3753
3754                 LLVMAddFunction (module, "mono_personality", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3755
3756                 LLVMAddFunction (module, "mono_resume_unwind", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3757         }
3758
3759         /* SSE intrinsics */
3760         {
3761                 LLVMTypeRef vector_type, arg_types [2];
3762
3763                 vector_type = LLVMVectorType (LLVMInt32Type (), 4);
3764                 arg_types [0] = vector_type;
3765                 arg_types [1] = vector_type;
3766                 LLVMAddFunction (module, "llvm.x86.sse41.pminud", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3767                 LLVMAddFunction (module, "llvm.x86.sse41.pmaxud", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3768
3769                 vector_type = LLVMVectorType (LLVMInt16Type (), 8);
3770                 arg_types [0] = vector_type;
3771                 arg_types [1] = vector_type;
3772                 LLVMAddFunction (module, "llvm.x86.sse41.pminuw", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3773                 LLVMAddFunction (module, "llvm.x86.sse41.pmaxuw", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3774
3775                 vector_type = LLVMVectorType (LLVMInt8Type (), 16);
3776                 arg_types [0] = vector_type;
3777                 arg_types [1] = vector_type;
3778                 LLVMAddFunction (module, "llvm.x86.sse41.pminub", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3779                 LLVMAddFunction (module, "llvm.x86.sse41.pmaxub", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3780
3781                 vector_type = LLVMVectorType (LLVMDoubleType (), 2);
3782                 arg_types [0] = vector_type;
3783                 arg_types [1] = vector_type;
3784                 LLVMAddFunction (module, "llvm.x86.sse2.min.pd", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3785                 LLVMAddFunction (module, "llvm.x86.sse2.max.pd", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3786
3787                 vector_type = LLVMVectorType (LLVMFloatType (), 4);
3788                 arg_types [0] = vector_type;
3789                 arg_types [1] = vector_type;
3790                 LLVMAddFunction (module, "llvm.x86.sse2.min.ps", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3791                 LLVMAddFunction (module, "llvm.x86.sse2.max.ps", LLVMFunctionType (vector_type, arg_types, 2, FALSE));
3792         }
3793 }
3794
3795 void
3796 mono_llvm_init (void)
3797 {
3798         current_cfg_tls_id = TlsAlloc ();
3799 }
3800
3801 static void
3802 init_jit_module (void)
3803 {
3804         if (jit_module_inited)
3805                 return;
3806
3807         mono_loader_lock ();
3808
3809         if (jit_module_inited) {
3810                 mono_loader_unlock ();
3811                 return;
3812         }
3813
3814         jit_module.module = LLVMModuleCreateWithName ("mono");
3815
3816         ee = mono_llvm_create_ee (LLVMCreateModuleProviderForExistingModule (jit_module.module), alloc_cb, emitted_cb, exception_cb);
3817
3818         add_intrinsics (jit_module.module);
3819
3820         jit_module.llvm_types = g_hash_table_new (NULL, NULL);
3821
3822         LLVMAddGlobalMapping (ee, LLVMGetNamedFunction (jit_module.module, "mono_resume_unwind"), mono_resume_unwind);
3823
3824         jit_module_inited = TRUE;
3825
3826         mono_loader_unlock ();
3827 }
3828
3829 void
3830 mono_llvm_cleanup (void)
3831 {
3832         if (ee)
3833                 mono_llvm_dispose_ee (ee);
3834
3835         if (jit_module.llvm_types)
3836                 g_hash_table_destroy (jit_module.llvm_types);
3837 }
3838
3839 void
3840 mono_llvm_create_aot_module (const char *got_symbol)
3841 {
3842         /* Delete previous module */
3843         if (aot_module.plt_entries)
3844                 g_hash_table_destroy (aot_module.plt_entries);
3845
3846         memset (&aot_module, 0, sizeof (aot_module));
3847
3848         aot_module.module = LLVMModuleCreateWithName ("aot");
3849         aot_module.got_symbol = got_symbol;
3850
3851         add_intrinsics (aot_module.module);
3852
3853         /* Add GOT */
3854         /*
3855          * We couldn't compute the type of the LLVM global representing the got because
3856          * its size is only known after all the methods have been emitted. So create
3857          * a dummy variable, and replace all uses it with the real got variable when
3858          * its size is known in mono_llvm_emit_aot_module ().
3859          */
3860         {
3861                 LLVMTypeRef got_type = LLVMArrayType (IntPtrType (), 0);
3862
3863                 aot_module.got_var = LLVMAddGlobal (aot_module.module, got_type, "mono_dummy_got");
3864                 LLVMSetInitializer (aot_module.got_var, LLVMConstNull (got_type));
3865         }
3866
3867         /* Add a dummy personality function */
3868         {
3869                 LLVMBasicBlockRef lbb;
3870                 LLVMBuilderRef lbuilder;
3871                 LLVMValueRef personality;
3872
3873                 personality = LLVMAddFunction (aot_module.module, "mono_aot_personality", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
3874                 LLVMSetLinkage (personality, LLVMPrivateLinkage);
3875                 lbb = LLVMAppendBasicBlock (personality, "BB0");
3876                 lbuilder = LLVMCreateBuilder ();
3877                 LLVMPositionBuilderAtEnd (lbuilder, lbb);
3878                 LLVMBuildRetVoid (lbuilder);
3879         }
3880
3881         aot_module.llvm_types = g_hash_table_new (NULL, NULL);
3882         aot_module.plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
3883 }
3884
3885 /*
3886  * Emit the aot module into the LLVM bitcode file FILENAME.
3887  */
3888 void
3889 mono_llvm_emit_aot_module (const char *filename, int got_size)
3890 {
3891         LLVMTypeRef got_type;
3892         LLVMValueRef real_got;
3893
3894         /*
3895          * Create the real got variable and replace all uses of the dummy variable with
3896          * the real one.
3897          */
3898         got_type = LLVMArrayType (IntPtrType (), got_size);
3899         real_got = LLVMAddGlobal (aot_module.module, got_type, aot_module.got_symbol);
3900         LLVMSetInitializer (real_got, LLVMConstNull (got_type));
3901         LLVMSetLinkage (real_got, LLVMInternalLinkage);
3902
3903         mono_llvm_replace_uses_of (aot_module.got_var, real_got);
3904
3905         mark_as_used (aot_module.module, real_got);
3906
3907         /* Delete the dummy got so it doesn't become a global */
3908         LLVMDeleteGlobal (aot_module.got_var);
3909
3910 #if 0
3911         {
3912                 char *verifier_err;
3913
3914                 if (LLVMVerifyModule (aot_module.module, LLVMReturnStatusAction, &verifier_err)) {
3915                         g_assert_not_reached ();
3916                 }
3917         }
3918 #endif
3919
3920         LLVMWriteBitcodeToFile (aot_module.module, filename);
3921 }
3922
3923 /*
3924   DESIGN:
3925   - Emit LLVM IR from the mono IR using the LLVM C API.
3926   - The original arch specific code remains, so we can fall back to it if we run
3927     into something we can't handle.
3928   FIXME:
3929   - llvm's PrettyStackTrace class seems to register a signal handler which screws
3930     up our GC. Also, it calls sigaction () a _lot_ of times instead of just once.
3931 */
3932
3933 /*
3934   A partial list of issues:
3935   - Handling of opcodes which can throw exceptions.
3936
3937       In the mono JIT, these are implemented using code like this:
3938           method:
3939       <compare>
3940           throw_pos:
3941           b<cond> ex_label
3942           <rest of code>
3943       ex_label:
3944           push throw_pos - method
3945           call <exception trampoline>
3946
3947           The problematic part is push throw_pos - method, which cannot be represented
3948       in the LLVM IR, since it does not support label values.
3949           -> this can be implemented in AOT mode using inline asm + labels, but cannot
3950           be implemented in JIT mode ?
3951           -> a possible but slower implementation would use the normal exception
3952       throwing code but it would need to control the placement of the throw code
3953       (it needs to be exactly after the compare+branch).
3954           -> perhaps add a PC offset intrinsics ?
3955
3956   - efficient implementation of .ovf opcodes.
3957
3958           These are currently implemented as:
3959           <ins which sets the condition codes>
3960           b<cond> ex_label
3961
3962           Some overflow opcodes are now supported by LLVM SVN.
3963
3964   - exception handling, unwinding.
3965     - SSA is disabled for methods with exception handlers
3966         - How to obtain unwind info for LLVM compiled methods ?
3967           -> this is now solved by converting the unwind info generated by LLVM
3968              into our format.
3969         - LLVM uses the c++ exception handling framework, while we use our home grown
3970       code, and couldn't use the c++ one:
3971       - its not supported under VC++, other exotic platforms.
3972           - it might be impossible to support filter clauses with it.
3973
3974   - trampolines.
3975
3976     The trampolines need a predictable call sequence, since they need to disasm
3977     the calling code to obtain register numbers / offsets.
3978
3979     LLVM currently generates this code in non-JIT mode:
3980            mov    -0x98(%rax),%eax
3981            callq  *%rax
3982     Here, the vtable pointer is lost.
3983     -> solution: use one vtable trampoline per class.
3984
3985   - passing/receiving the IMT pointer/RGCTX.
3986     -> solution: pass them as normal arguments ?
3987
3988   - argument passing.
3989
3990           LLVM does not allow the specification of argument registers etc. This means
3991       that all calls are made according to the platform ABI.
3992
3993   - passing/receiving vtypes.
3994
3995       Vtypes passed/received in registers are handled by the front end by using
3996           a signature with scalar arguments, and loading the parts of the vtype into those
3997           arguments.
3998
3999           Vtypes passed on the stack are handled using the 'byval' attribute.
4000
4001   - ldaddr.
4002
4003     Supported though alloca, we need to emit the load/store code.
4004
4005   - types.
4006
4007     The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
4008     typed registers, so we have to keep track of the precise LLVM type of each vreg.
4009     This is made easier because the IR is already in SSA form.
4010     An additional problem is that our IR is not consistent with types, i.e. i32/ia64
4011         types are frequently used incorrectly.
4012 */
4013
4014 /*
4015   AOT SUPPORT:
4016   Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then
4017   append the AOT data structures to that file. For methods which cannot be
4018   handled by LLVM, the normal JIT compiled versions are used.
4019 */
4020
4021 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
4022  *   - each bblock should end with a branch
4023  *   - setting the return value, making cfg->ret non-volatile
4024  * - merge some changes back to HEAD, to reduce the differences.
4025  * - avoid some transformations in the JIT which make it harder for us to generate
4026  *   code.
4027  * - fix memory leaks.
4028  * - use pointer types to help optimizations.
4029  */