src/vm/jit/inline/inline.c

   1 /* src/vm/jit/inline/inline.c - method inlining
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Edwin Steiner
  28
  29    Changes:
  30
  31    $Id: inline.c 6262 2006-12-31 17:57:22Z edwin $
  32
  33 */
  34
  35 #include "config.h"
  36
  37 #include "vm/types.h"
  38
  39 #include <assert.h>
  40 #include <stdio.h>
  41 #include <string.h>
  42 #include <limits.h>
  43
  44 #include "mm/memory.h"
  45 #include "toolbox/logging.h"
  46 #include "vm/builtin.h"
  47 #include "vm/global.h"
  48 #include "vm/options.h"
  49 #include "vm/statistics.h"
  50 #include "vm/jit/jit.h"
  51 #include "vm/jit/parse.h"
  52 #include "vm/jit/inline/inline.h"
  53 #include "vm/jit/loop/loop.h"
  54
  55 #include "vm/class.h"
  56 #include "vm/initialize.h"
  57 #include "vm/method.h"
  58 #include "vm/jit/jit.h"
  59 #include "vm/jit/show.h"
  60
  61 #include "vm/jit/reg.h"
  62 #include "vm/jit/stack.h"
  63
  64 #include "vm/jit/verify/typecheck.h"
  65
  66 #if defined(ENABLE_THREADS)
  67 # include "threads/native/threads.h"
  68 #endif
  69
  70
  71 /* debugging ******************************************************************/
  72
  73 #if !defined(NDEBUG)
  74 #define INLINE_VERBOSE
  75 bool inline_debug_log = 0;
  76 bool inline_debug_log_names = 0;
  77 int inline_debug_start_counter = 0;
  78 int inline_debug_max_size = INT_MAX;
  79 int inline_debug_min_size = 0;
  80 int inline_debug_end_counter = INT_MAX;
  81 int inline_count_methods = 0;
  82 #define DOLOG(code) do{ if (inline_debug_log) { code; } }while(0)
  83 #else
  84 #define DOLOG(code)
  85 #endif
  86
  87
  88 /* types **********************************************************************/
  89
  90 typedef struct inline_node inline_node;
  91 typedef struct inline_target_ref inline_target_ref;
  92 typedef struct inline_context inline_context;
  93 typedef struct inline_block_map inline_block_map;
  94
  95 struct inline_node {
  96         inline_context *ctx;
  97
  98         jitdata *jd;
  99         methodinfo *m;
 100         inline_node *children;
 101         inline_node *next;                             /* next node at this depth */
 102         inline_node *prev;                             /* prev node at this depth */
 103         int depth;                                  /* inlining depth, 0 for root */
 104
 105         /* info about the call site (if depth > 0)*/
 106         inline_node *parent;                /* node of the caller (NULL for root) */
 107         basicblock *callerblock;         /* original block containing the INVOKE* */
 108         instruction *callerins;               /* the original INVOKE* instruction */
 109         s4 callerpc;
 110         s4 *n_passthroughvars;
 111         int n_passthroughcount;
 112         int n_selfpassthroughcount;  /* # of pass-through vars of the call itself */
 113         exception_entry **o_handlers;
 114         int n_handlercount;                 /* # of handlers protecting this call */
 115         int n_resultlocal;
 116         int synclocal;                    /* variable used for synchr., or UNUSED */
 117
 118         bool blockbefore;                  /* block boundary before inlined body? */
 119         bool blockafter;                   /* block boundary after inlined body?  */
 120
 121         /* info about the callee */
 122         int localsoffset;
 123         int prolog_instructioncount;         /* # of ICMDs in the inlining prolog */
 124         int epilog_instructioncount;         /* # of ICMDs in the inlining epilog */
 125         int extra_instructioncount;
 126         int instructioncount;
 127         bool synchronize;                /* do we have to synchronize enter/exit? */
 128         basicblock *handler_monitorexit;     /* handler for synchronized inlinees */
 129         s4 *varmap;
 130
 131         /* cumulative values */
 132         int cumul_instructioncount;  /* ICMDs in this node and its children       */
 133         int cumul_basicblockcount;   /* BBs started by this node and its children */
 134         int cumul_blockmapcount;
 135         int cumul_maxlocals;
 136         int cumul_exceptiontablelength;
 137
 138         /* output */
 139         instruction *inlined_iinstr;
 140         instruction *inlined_iinstr_cursor;
 141         basicblock *inlined_basicblocks;
 142         basicblock *inlined_basicblocks_cursor;
 143
 144         /* register data */
 145         registerdata *regdata;
 146
 147         /* temporary */
 148         inline_target_ref *refs;
 149         instruction *inline_start_instruction;
 150         s4 *javalocals;
 151
 152         /* XXX debug */
 153         char *indent;
 154         int debugnr;
 155 };
 156
 157 struct inline_target_ref {
 158         inline_target_ref *next;
 159         union {
 160                 basicblock **block;
 161                 s4 *nr;
 162         } ref;
 163         basicblock *target;
 164         bool isnumber;
 165 };
 166
 167 struct inline_block_map {
 168         inline_node *iln;
 169         basicblock *o_block;
 170         basicblock *n_block;
 171 };
 172
 173 struct inline_context {
 174         inline_node *master;
 175
 176         jitdata *resultjd;
 177
 178         int next_block_number;
 179         inline_block_map *blockmap;
 180         int blockmap_index;
 181
 182         int maxinoutdepth;
 183
 184         bool calls_others;
 185
 186         int next_debugnr; /* XXX debug */
 187 };
 188
 189
 190 /* prototypes *****************************************************************/
 191
 192 static bool inline_inline_intern(methodinfo *m, inline_node *iln);
 193 static void inline_post_process(jitdata *jd);
 194
 195
 196 /* debug helpers **************************************************************/
 197
 198 #if !defined(NDEBUG)
 199 #include "inline_debug.inc"
 200
 201 void inline_print_stats()
 202 {
 203         printf("inlined callers: %d\n", inline_count_methods);
 204 }
 205 #endif
 206
 207
 208 /* compilation of callees *****************************************************/
 209
 210 static bool inline_jit_compile_intern(jitdata *jd)
 211 {
 212         methodinfo *m;
 213
 214         /* XXX should share code with jit.c */
 215
 216         assert(jd);
 217
 218         /* XXX initialize the static function's class */
 219
 220         m = jd->m;
 221
 222         /* call the compiler passes ***********************************************/
 223
 224         /* call parse pass */
 225
 226         DOLOG( log_message_class("Parsing ", m->class) );
 227         if (!parse(jd)) {
 228                 return false;
 229         }
 230
 231         /* call stack analysis pass */
 232
 233         if (!stack_analyse(jd)) {
 234                 return false;
 235         }
 236
 237         return true;
 238 }
 239
 240
 241 static bool inline_jit_compile(inline_node *iln)
 242 {
 243         bool                r;
 244         methodinfo         *m;
 245         jitdata            *jd;
 246
 247         /* XXX should share code with jit.c */
 248
 249         assert(iln);
 250         m = iln->m;
 251         assert(m);
 252
 253 #if defined(ENABLE_THREADS)
 254         /* enter a monitor on the method */
 255         lock_monitor_enter((java_objectheader *) m);
 256 #endif
 257
 258         /* allocate jitdata structure and fill it */
 259
 260         jd = jit_jitdata_new(m);
 261         iln->jd = jd;
 262
 263         jd->flags = 0; /* XXX */
 264
 265         /* initialize the register allocator */
 266
 267         reg_setup(jd);
 268
 269         /* setup the codegendata memory */
 270
 271         /* XXX do a pseudo setup */
 272         jd->cd = DNEW(codegendata);
 273         MZERO(jd->cd, codegendata, 1);
 274         jd->cd->maxstack = m->maxstack;
 275         jd->cd->method = m;
 276         /* XXX uses too much dump memory codegen_setup(jd); */
 277
 278         /* now call internal compile function */
 279
 280         r = inline_jit_compile_intern(jd);
 281
 282         if (r) {
 283                 iln->regdata = jd->rd;
 284         }
 285
 286         /* free some memory */
 287 #if 0
 288
 289 #if defined(ENABLE_JIT)
 290 # if defined(ENABLE_INTRP)
 291         if (!opt_intrp)
 292 # endif
 293                 codegen_free(jd);
 294 #endif
 295
 296 #endif
 297
 298 #if defined(ENABLE_THREADS)
 299         /* leave the monitor */
 300         lock_monitor_exit((java_objectheader *) m );
 301 #endif
 302
 303         return r;
 304 }
 305
 306
 307 /* inlining tree handling *****************************************************/
 308
 309 static void insert_inline_node(inline_node *parent, inline_node *child)
 310 {
 311         inline_node *first;
 312         inline_node *succ;
 313
 314         assert(parent && child);
 315
 316         child->parent = parent;
 317
 318         child->debugnr = parent->ctx->next_debugnr++; /* XXX debug */
 319
 320         first = parent->children;
 321         if (!first) {
 322                 /* insert as only node */
 323                 parent->children = child;
 324                 child->next = child;
 325                 child->prev = child;
 326                 return;
 327         }
 328
 329         /* {there is at least one child already there} */
 330
 331         /* XXX is this search necessary, or could we always add at the end? */
 332
 333         succ = first;
 334         while (succ->callerpc < child->callerpc) {
 335                 succ = succ->next;
 336                 if (succ == first) {
 337                         /* insert as last node */
 338                         child->prev = first->prev;
 339                         child->next = first;
 340                         child->prev->next = child;
 341                         child->next->prev = child;
 342                         return;
 343                 }
 344         }
 345
 346         assert(succ->callerpc > child->callerpc);
 347
 348         /* insert before succ */
 349
 350         child->prev = succ->prev;
 351         child->next = succ;
 352         child->prev->next = child;
 353         child->next->prev = child;
 354 }
 355
 356
 357 /* variable handling **********************************************************/
 358
 359 static s4 inline_new_variable(jitdata *jd, s4 type, s4 flags)
 360 {
 361         s4 index;
 362         s4 newcount;
 363
 364         index = jd->vartop++;
 365         if (index >= jd->varcount) {
 366                 newcount = jd->vartop * 2; /* XXX */
 367                 jd->var = DMREALLOC(jd->var, varinfo, jd->varcount, newcount);
 368                 MZERO(jd->var + jd->varcount, varinfo, (newcount - jd->varcount));
 369                 jd->varcount = newcount;
 370         }
 371
 372         jd->var[index].type = type;
 373         jd->var[index].flags = flags;
 374
 375         return index;
 376 }
 377
 378
 379 static s4 inline_new_variable_clone(jitdata *jd, jitdata *origjd, s4 origidx)
 380 {
 381         varinfo *v;
 382         s4       newidx;
 383
 384         v = &(origjd->var[origidx]);
 385
 386         newidx = inline_new_variable(jd, v->type, v->flags);
 387
 388         jd->var[newidx].vv = v->vv;
 389
 390         return newidx;
 391 }
 392
 393
 394 static s4 inline_new_temp_variable(jitdata *jd, s4 type)
 395 {
 396         return inline_new_variable(jd, type, 0);
 397 }
 398
 399
 400 static s4 inline_translate_variable(jitdata *jd, jitdata *origjd, s4 *varmap, s4 index)
 401 {
 402         s4 idx;
 403
 404         idx = varmap[index];
 405
 406         if (idx < 0) {
 407                 idx = inline_new_variable_clone(jd, origjd, index);
 408                 varmap[index] = idx;
 409         }
 410
 411         return idx;
 412 }
 413
 414
 415 static s4 *create_variable_map(inline_node *callee)
 416 {
 417         s4 *varmap;
 418         s4 i, t;
 419         s4 idx;
 420         s4 n_idx;
 421         s4 avail;
 422         varinfo *v;
 423
 424         /* create the variable mapping */
 425
 426         varmap = DMNEW(s4, callee->jd->varcount);
 427         for (i=0; i<callee->jd->varcount; ++i)
 428                 varmap[i] = -1;
 429
 430         /* translate local variables */
 431
 432         for (i=0; i<callee->m->maxlocals; ++i) {
 433                 for (t=0; t<5; ++t) {
 434                         idx = callee->jd->local_map[5*i + t];
 435                         if (idx == UNUSED)
 436                                 continue;
 437
 438                         v = &(callee->jd->var[idx]);
 439                         assert(v->type == t || v->type == TYPE_VOID); /* XXX stack leaves VOID */
 440                         v->type = t; /* XXX restore if it is TYPE_VOID */
 441
 442                         avail = callee->ctx->resultjd->local_map[5*(callee->localsoffset + i) + t];
 443
 444                         if (avail == UNUSED) {
 445                                 avail = inline_new_variable_clone(callee->ctx->resultjd, callee->jd, idx);
 446                                 callee->ctx->resultjd->local_map[5*(callee->localsoffset + i) + t] = avail;
 447                         }
 448
 449                         varmap[idx] = avail;
 450                 }
 451         }
 452
 453         /* for synchronized instance methods we need an extra local */
 454
 455         if (callee->synchronize && !(callee->m->flags & ACC_STATIC)) {
 456                 n_idx = callee->localsoffset - 1;
 457                 assert(n_idx >= 0);
 458                 assert(callee->parent);
 459                 assert(n_idx == callee->parent->localsoffset + callee->parent->m->maxlocals);
 460
 461                 avail = callee->ctx->resultjd->local_map[5*n_idx + TYPE_ADR];
 462
 463                 if (avail == UNUSED) {
 464                         avail = inline_new_variable(callee->ctx->resultjd, TYPE_ADR, 0);
 465                         callee->ctx->resultjd->local_map[5*n_idx + TYPE_ADR] = avail;
 466                 }
 467
 468                 callee->synclocal = avail;
 469         }
 470         else {
 471                 callee->synclocal = UNUSED;
 472         }
 473
 474         return varmap;
 475 }
 476
 477
 478 /* basic block translation ****************************************************/
 479
 480 #define INLINE_RETURN_REFERENCE(callee)  \
 481         ( (basicblock *) (ptrint) (0x333 + (callee)->depth) )
 482
 483 #define RETADDRNR_FROM_BLOCK(bptr)  (UNUSED - 1 - (bptr)->nr)
 484
 485
 486 static void inline_add_block_reference(inline_node *iln, basicblock **blockp)
 487 {
 488         inline_target_ref *ref;
 489
 490         ref = DNEW(inline_target_ref);
 491         ref->ref.block = blockp;
 492         ref->isnumber = false;
 493         ref->next = iln->refs;
 494         iln->refs = ref;
 495 }
 496
 497
 498 static void inline_add_blocknr_reference(inline_node *iln, s4 *nrp)
 499 {
 500         inline_target_ref *ref;
 501
 502         ref = DNEW(inline_target_ref);
 503         ref->ref.nr = nrp;
 504         ref->isnumber = true;
 505         ref->next = iln->refs;
 506         iln->refs = ref;
 507 }
 508
 509
 510 static void inline_block_translation(inline_node *iln, basicblock *o_bptr, basicblock *n_bptr)
 511 {
 512         inline_context *ctx;
 513
 514         ctx = iln->ctx;
 515         assert(ctx->blockmap_index < ctx->master->cumul_blockmapcount);
 516
 517         ctx->blockmap[ctx->blockmap_index].iln = iln;
 518         ctx->blockmap[ctx->blockmap_index].o_block = o_bptr;
 519         ctx->blockmap[ctx->blockmap_index].n_block = n_bptr;
 520
 521         ctx->blockmap_index++;
 522 }
 523
 524
 525 static basicblock * inline_map_block(inline_node *iln,
 526                                                                          basicblock *o_block,
 527                                                                          inline_node *targetiln)
 528 {
 529         inline_block_map *bm;
 530         inline_block_map *bmend;
 531
 532         assert(iln);
 533         assert(targetiln);
 534
 535         if (!o_block)
 536                 return NULL;
 537
 538         bm = iln->ctx->blockmap;
 539         bmend = bm + iln->ctx->blockmap_index;
 540
 541         while (bm < bmend) {
 542                 assert(bm->iln && bm->o_block && bm->n_block);
 543                 if (bm->o_block == o_block && bm->iln == targetiln)
 544                         return bm->n_block;
 545                 bm++;
 546         }
 547
 548         assert(false);
 549         return NULL; /* not reached */
 550 }
 551
 552
 553 static void inline_resolve_block_refs(inline_target_ref **refs,
 554                                                                           basicblock *o_bptr,
 555                                                                           basicblock *n_bptr,
 556                                                                           bool returnref)
 557 {
 558         inline_target_ref *ref;
 559         inline_target_ref *prev;
 560
 561         prev = NULL;
 562         for (ref = *refs; ref != NULL; ref = ref->next) {
 563                 if (ref->isnumber && !returnref) {
 564                         if (*(ref->ref.nr) == RETADDRNR_FROM_BLOCK(o_bptr)) {
 565                                 *(ref->ref.nr) = RETADDRNR_FROM_BLOCK(n_bptr);
 566                                 goto remove_ref;
 567                         }
 568                 }
 569                 else {
 570                         if (*(ref->ref.block) == o_bptr) {
 571                                 *(ref->ref.block) = n_bptr;
 572                                 goto remove_ref;
 573                         }
 574                 }
 575
 576                 /* skip this ref */
 577
 578                 prev = ref;
 579                 continue;
 580
 581 remove_ref:
 582                 /* remove this ref */
 583
 584                 if (prev) {
 585                         prev->next = ref->next;
 586                 }
 587                 else {
 588                         *refs = ref->next;
 589                 }
 590         }
 591 }
 592
 593
 594 /* basic block creation *******************************************************/
 595
 596 static basicblock * create_block(inline_node *container,
 597                                                                  inline_node *iln,
 598                                                                  inline_node *inner,
 599                                                                  int indepth)
 600 {
 601         basicblock  *n_bptr;
 602         inline_node *outer;
 603         s4           i;
 604         s4           depth;
 605         s4           varidx;
 606         s4           newvaridx;
 607
 608         assert(container);
 609         assert(iln);
 610         assert(inner);
 611         assert(indepth >= 0);
 612
 613         n_bptr = container->inlined_basicblocks_cursor++;
 614         assert(n_bptr);
 615         assert((n_bptr - container->inlined_basicblocks) < container->cumul_basicblockcount);
 616
 617         BASICBLOCK_INIT(n_bptr, iln->m);
 618
 619         n_bptr->iinstr = container->inlined_iinstr_cursor;
 620         n_bptr->next = n_bptr + 1;
 621         n_bptr->nr = container->ctx->next_block_number++;
 622         n_bptr->indepth = indepth;
 623         n_bptr->flags = BBFINISHED; /* XXX */
 624
 625         /* set the inlineinfo of the new block */
 626
 627         if (iln->inline_start_instruction)
 628                 n_bptr->inlineinfo = iln->inline_start_instruction->sx.s23.s3.inlineinfo;
 629
 630         if (indepth > container->ctx->maxinoutdepth)
 631                 container->ctx->maxinoutdepth = indepth;
 632
 633         if (indepth) {
 634                 n_bptr->invars = DMNEW(s4, indepth);
 635
 636
 637                 for (i=0; i<indepth; ++i)
 638                         n_bptr->invars[i] = -1; /* XXX debug */
 639
 640                 /* pass-through variables enter the block */
 641
 642                 outer = inner->parent;
 643                 while (outer != NULL) {
 644                         depth = outer->n_passthroughcount;
 645
 646                         assert(depth + inner->n_selfpassthroughcount <= indepth);
 647
 648                         for (i=0; i<inner->n_selfpassthroughcount; ++i) {
 649                                 varidx = inner->n_passthroughvars[i];
 650                                 newvaridx =
 651                                         inline_new_variable_clone(container->ctx->resultjd,
 652                                                                                           outer->jd,
 653                                                                                           varidx);
 654                                 n_bptr->invars[depth + i] = newvaridx;
 655                                 outer->varmap[varidx] = newvaridx;
 656                         }
 657                         inner = outer;
 658                         outer = outer->parent;
 659                 }
 660         }
 661         else {
 662                 n_bptr->invars = NULL;
 663         }
 664
 665         /* XXX for the verifier. should not be here */
 666
 667         {
 668                 varinfo *dv;
 669
 670                 dv = DMNEW(varinfo, iln->ctx->resultjd->localcount + VERIFIER_EXTRA_LOCALS);
 671                 MZERO(dv, varinfo,  iln->ctx->resultjd->localcount + VERIFIER_EXTRA_LOCALS);
 672                 n_bptr->inlocals = dv;
 673         }
 674
 675         return n_bptr;
 676 }
 677
 678
 679 static s4 *translate_javalocals(inline_node *iln, s4 *javalocals)
 680 {
 681         s4 *jl;
 682         s4 i, j;
 683
 684         jl = DMNEW(s4, iln->jd->maxlocals);
 685
 686         for (i=0; i<iln->jd->maxlocals; ++i) {
 687                 j = javalocals[i];
 688                 if (j > UNUSED)
 689                         j = inline_translate_variable(iln->ctx->resultjd, iln->jd, iln->varmap, j);
 690                 jl[i] = j;
 691
 692 #if 0
 693                 if (j < UNUSED) {
 694                         /* an encoded returnAddress value - must be relocated */
 695                         inline_add_blocknr_reference(iln, &(jl[i]));
 696                 }
 697 #endif
 698         }
 699
 700         return jl;
 701 }
 702
 703
 704 static basicblock * create_body_block(inline_node *iln,
 705                                                                           basicblock *o_bptr, s4 *varmap)
 706 {
 707         basicblock *n_bptr;
 708         s4 i;
 709
 710         n_bptr = create_block(iln, iln, iln,
 711                                                   o_bptr->indepth + iln->n_passthroughcount);
 712
 713         n_bptr->type = o_bptr->type;
 714         n_bptr->flags = o_bptr->flags;
 715         n_bptr->bitflags = o_bptr->bitflags;
 716
 717         /* resolve references to this block */
 718
 719         inline_resolve_block_refs(&(iln->refs), o_bptr, n_bptr, false);
 720
 721         /* translate the invars of the original block */
 722
 723         for (i=0; i<o_bptr->indepth; ++i) {
 724                 n_bptr->invars[iln->n_passthroughcount + i] =
 725                         inline_translate_variable(iln->ctx->resultjd, iln->jd,
 726                                 varmap,
 727                                 o_bptr->invars[i]);
 728         }
 729
 730         /* translate javalocals info (not for dead code) */
 731
 732         if (n_bptr->flags >= BBREACHED)
 733                 n_bptr->javalocals = translate_javalocals(iln, o_bptr->javalocals);
 734
 735         return n_bptr;
 736 }
 737
 738
 739 static basicblock * create_epilog_block(inline_node *caller, inline_node *callee, s4 *varmap)
 740 {
 741         basicblock *n_bptr;
 742         s4 retcount;
 743         s4 idx;
 744
 745         /* number of return variables */
 746
 747         retcount = (callee->n_resultlocal == -1
 748                                 && callee->m->parseddesc->returntype.type != TYPE_VOID) ? 1 : 0;
 749
 750         /* start the epilog block */
 751
 752         n_bptr = create_block(caller, caller, callee,
 753                                                   callee->n_passthroughcount + retcount);
 754
 755         /* resolve references to the return block */
 756
 757         inline_resolve_block_refs(&(callee->refs),
 758                                                           INLINE_RETURN_REFERENCE(callee),
 759                                                           n_bptr,
 760                                                           true);
 761
 762         /* return variable */
 763
 764         if (retcount) {
 765                 idx = inline_new_variable(caller->ctx->resultjd,
 766                            callee->m->parseddesc->returntype.type, 0 /* XXX */);
 767                 n_bptr->invars[callee->n_passthroughcount] = idx;
 768                 varmap[callee->callerins->dst.varindex] = idx;
 769         }
 770
 771         /* set javalocals */
 772
 773         n_bptr->javalocals = DMNEW(s4, caller->jd->maxlocals);
 774         MCOPY(n_bptr->javalocals, caller->javalocals, s4, caller->jd->maxlocals);
 775
 776         /* set block flags & type */
 777
 778         n_bptr->flags = /* XXX original block flags */ BBFINISHED;
 779         n_bptr->type = BBTYPE_STD;
 780
 781         return n_bptr;
 782 }
 783
 784
 785 static void close_block(inline_node *iln, inline_node *inner, basicblock *n_bptr, s4 outdepth)
 786 {
 787         inline_node *outer;
 788         s4           i;
 789         s4           depth;
 790         s4           varidx;
 791
 792         n_bptr->outdepth = outdepth;
 793         n_bptr->outvars = DMNEW(s4, outdepth);
 794
 795         for (i=0; i<outdepth; ++i)
 796                 n_bptr->outvars[i] = 0; /* XXX debug */
 797
 798         if (outdepth > iln->ctx->maxinoutdepth)
 799                 iln->ctx->maxinoutdepth = outdepth;
 800
 801         /* pass-through variables leave the block */
 802
 803         outer = inner->parent;
 804         while (outer != NULL) {
 805                 depth = outer->n_passthroughcount;
 806
 807                 assert(depth + inner->n_selfpassthroughcount <= outdepth);
 808
 809                 for (i=0; i<inner->n_selfpassthroughcount; ++i) {
 810                         varidx = inner->n_passthroughvars[i];
 811                         n_bptr->outvars[depth + i] =
 812                                 inline_translate_variable(iln->ctx->resultjd,
 813                                                                                   outer->jd,
 814                                                                                   outer->varmap,
 815                                                                                   varidx);
 816                 }
 817                 inner = outer;
 818                 outer = outer->parent;
 819         }
 820 }
 821
 822
 823 static void close_prolog_block(inline_node *iln,
 824                                                            basicblock *n_bptr,
 825                                                            inline_node *nextcall)
 826 {
 827         /* XXX add original outvars! */
 828         close_block(iln, nextcall, n_bptr, nextcall->n_passthroughcount);
 829
 830         /* pass-through variables */
 831
 832         DOLOG( printf("closed prolog block:\n");
 833                    show_basicblock(iln->ctx->resultjd, n_bptr, SHOW_STACK); );
 834 }
 835
 836
 837 static void close_body_block(inline_node *iln,
 838                                                          basicblock *n_bptr,
 839                                                          basicblock *o_bptr,
 840                                                          s4 *varmap,
 841                                                          s4 retcount,
 842                                                          s4 retidx)
 843 {
 844         s4 i;
 845
 846         close_block(iln, iln, n_bptr, iln->n_passthroughcount + o_bptr->outdepth + retcount);
 847
 848         /* translate the outvars of the original block */
 849
 850         /* XXX reuse code */
 851         for (i=0; i<o_bptr->outdepth; ++i) {
 852                 n_bptr->outvars[iln->n_passthroughcount + i] =
 853                         inline_translate_variable(iln->ctx->resultjd, iln->jd, varmap,
 854                                         o_bptr->outvars[i]);
 855         }
 856
 857         /* set the return variable, if any */
 858
 859         if (retcount) {
 860                 assert(retidx >= 0);
 861                 n_bptr->outvars[iln->n_passthroughcount + o_bptr->outdepth] = retidx;
 862         }
 863 }
 864
 865
 866 /* inlined code generation ****************************************************/
 867
 868 static instruction * inline_instruction(inline_node *iln,
 869                                                                                 s4 opcode,
 870                                                                                 instruction *o_iptr)
 871 {
 872         instruction *n_iptr;
 873
 874         n_iptr = (iln->inlined_iinstr_cursor++);
 875         assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
 876
 877         n_iptr->opc = opcode;
 878         n_iptr->flags.bits = o_iptr->flags.bits & INS_FLAG_ID_MASK;
 879         n_iptr->line = o_iptr->line;
 880
 881         return n_iptr;
 882 }
 883
 884 static void inline_generate_sync_builtin(inline_node *iln,
 885                                                                                  inline_node *callee,
 886                                                                                  instruction *o_iptr,
 887                                                                                  s4 instancevar,
 888                                                                                  functionptr func)
 889 {
 890         int          syncvar;
 891         instruction *n_ins;
 892
 893         if (callee->m->flags & ACC_STATIC) {
 894                 /* ACONST */
 895                 syncvar = inline_new_temp_variable(iln->ctx->resultjd, TYPE_ADR);
 896
 897                 n_ins = inline_instruction(iln, ICMD_ACONST, o_iptr);
 898                 n_ins->sx.val.c.cls = callee->m->class;
 899                 n_ins->dst.varindex = syncvar;
 900                 n_ins->flags.bits |= INS_FLAG_CLASS;
 901         }
 902         else {
 903                 syncvar = instancevar;
 904         }
 905
 906         assert(syncvar != UNUSED);
 907
 908         /* MONITORENTER / MONITOREXIT */
 909
 910         n_ins = inline_instruction(iln, ICMD_BUILTIN, o_iptr);
 911         n_ins->sx.s23.s3.bte = builtintable_get_internal(func);
 912         n_ins->s1.argcount = 1; /* XXX add through-vars */
 913         n_ins->sx.s23.s2.args = DMNEW(s4, 1);
 914         n_ins->sx.s23.s2.args[0] = syncvar;
 915 }
 916
 917 static s4 emit_inlining_prolog(inline_node *iln,
 918                                                            inline_node *callee,
 919                                                            instruction *o_iptr,
 920                                                            s4 *varmap)
 921 {
 922         methodinfo *calleem;
 923         methoddesc *md;
 924         int i;
 925         int localindex;
 926         int type;
 927         bool isstatic;
 928         instruction *n_ins;
 929         insinfo_inline *insinfo;
 930         s4 argvar;
 931
 932         assert(iln && callee && o_iptr);
 933
 934         calleem = callee->m;
 935         md = calleem->parseddesc;
 936         isstatic = (calleem->flags & ACC_STATIC);
 937
 938         /* INLINE_START instruction */
 939
 940         n_ins = inline_instruction(iln, ICMD_INLINE_START, o_iptr);
 941
 942         insinfo = DNEW(insinfo_inline);
 943         insinfo->method = callee->m;
 944         insinfo->outer = iln->m;
 945         insinfo->synclocal = callee->synclocal;
 946         insinfo->synchronize = callee->synchronize;
 947         insinfo->javalocals_start = NULL;
 948         insinfo->javalocals_end = NULL;
 949
 950         /* info about stack vars live at the INLINE_START */
 951
 952         insinfo->throughcount = callee->n_passthroughcount;
 953         insinfo->paramcount = md->paramcount;
 954         insinfo->stackvarscount = o_iptr->s1.argcount;
 955         insinfo->stackvars = DMNEW(s4, insinfo->stackvarscount);
 956         for (i=0; i<insinfo->stackvarscount; ++i)
 957                 insinfo->stackvars[i] = iln->varmap[o_iptr->sx.s23.s2.args[i]];
 958
 959         /* info about the surrounding inlining */
 960
 961         if (iln->inline_start_instruction)
 962                 insinfo->parent = iln->inline_start_instruction->sx.s23.s3.inlineinfo;
 963         else
 964                 insinfo->parent = NULL;
 965
 966         /* finish the INLINE_START instruction */
 967
 968         n_ins->sx.s23.s3.inlineinfo = insinfo;
 969         callee->inline_start_instruction = n_ins;
 970
 971         DOLOG( printf("%sprolog: ", iln->indent);
 972                    show_icmd(iln->ctx->resultjd, n_ins, false, SHOW_STACK); printf("\n"); );
 973
 974         /* handle parameters for the inlined callee */
 975
 976         localindex = callee->localsoffset + md->paramslots;
 977
 978         for (i=md->paramcount-1; i>=0; --i) {
 979                 assert(iln);
 980
 981                 type = md->paramtypes[i].type;
 982
 983                 localindex -= IS_2_WORD_TYPE(type) ? 2 : 1;
 984                 assert(callee->regdata);
 985
 986                 /* translate the argument variable */
 987
 988                 argvar = varmap[o_iptr->sx.s23.s2.args[i]];
 989                 assert(argvar != UNUSED);
 990
 991                 /* remove preallocation from the argument variable */
 992
 993                 iln->ctx->resultjd->var[argvar].flags &= ~(PREALLOC | INMEMORY);
 994
 995                 /* check the instance slot against NULL */
 996                 /* we don't need that for <init> methods, as the verifier  */
 997                 /* ensures that they are only called for an uninit. object */
 998                 /* (which may not be NULL).                                */
 999
1000                 if (!isstatic && i == 0 && calleem->name != utf_init) {
1001                         assert(type == TYPE_ADR);
1002                         n_ins = inline_instruction(iln, ICMD_CHECKNULL, o_iptr);
1003                         n_ins->s1.varindex = argvar;
1004                         n_ins->dst.varindex = n_ins->s1.varindex;
1005                 }
1006
1007                 /* store argument into local variable of inlined callee */
1008
1009                 if (callee->jd->local_map[5*(localindex - callee->localsoffset) + type] != UNUSED)
1010                 {
1011                         /* this value is used in the callee */
1012
1013                         if (i == 0 && callee->synclocal != UNUSED) {
1014                                 /* we also need it for synchronization, so copy it */
1015                                 assert(type == TYPE_ADR);
1016                                 n_ins = inline_instruction(iln, ICMD_COPY, o_iptr);
1017                         }
1018                         else {
1019                                 n_ins = inline_instruction(iln, ICMD_ISTORE + type, o_iptr);
1020                                 n_ins->sx.s23.s3.javaindex = UNUSED;
1021                         }
1022                         n_ins->s1.varindex = argvar;
1023                         n_ins->dst.varindex = iln->ctx->resultjd->local_map[5*localindex + type];
1024                         assert(n_ins->dst.varindex != UNUSED);
1025                 }
1026                 else if (i == 0 && callee->synclocal != UNUSED) {
1027                         /* the value is not used inside the callee, but we need it for */
1028                         /* synchronization                                             */
1029                         /* XXX In this case it actually makes no sense to create a     */
1030                         /*     separate synchronization variable.                      */
1031
1032                         n_ins = inline_instruction(iln, ICMD_NOP, o_iptr);
1033                 }
1034                 else {
1035                         /* this value is not used, pop it */
1036
1037                         n_ins = inline_instruction(iln, ICMD_POP, o_iptr);
1038                         n_ins->s1.varindex = argvar;
1039                 }
1040
1041                 DOLOG( printf("%sprolog: ", iln->indent);
1042                            show_icmd(iln->ctx->resultjd, n_ins, false, SHOW_STACK); printf("\n"); );
1043         }
1044
1045         /* COPY for synchronized instance methods */
1046
1047         if (callee->synclocal != UNUSED) {
1048                 n_ins = inline_instruction(iln, ICMD_COPY, o_iptr);
1049                 n_ins->s1.varindex = varmap[o_iptr->sx.s23.s2.args[0]];
1050                 n_ins->dst.varindex = callee->synclocal;
1051
1052                 assert(n_ins->s1.varindex != UNUSED);
1053         }
1054
1055         if (callee->synchronize) {
1056                 inline_generate_sync_builtin(iln, callee, o_iptr,
1057                                                                          (isstatic) ? UNUSED : varmap[o_iptr->sx.s23.s2.args[0]],
1058                                                                          LOCK_monitor_enter);
1059         }
1060
1061         /* INLINE_BODY instruction */
1062
1063         n_ins = inline_instruction(iln, ICMD_INLINE_BODY, callee->jd->basicblocks[0].iinstr);
1064         n_ins->sx.s23.s3.inlineinfo = insinfo;
1065
1066         return 0; /* XXX */
1067 }
1068
1069
1070 static void emit_inlining_epilog(inline_node *iln, inline_node *callee, instruction *o_iptr)
1071 {
1072         instruction *n_ins;
1073         s4          *jl;
1074
1075         assert(iln && callee && o_iptr);
1076         assert(callee->inline_start_instruction);
1077
1078         if (callee->synchronize) {
1079                 inline_generate_sync_builtin(iln, callee, o_iptr,
1080                                                                          callee->synclocal,
1081                                                                          LOCK_monitor_exit);
1082         }
1083
1084         /* INLINE_END instruction */
1085
1086         n_ins = inline_instruction(iln, ICMD_INLINE_END, o_iptr);
1087         n_ins->sx.s23.s3.inlineinfo = callee->inline_start_instruction->sx.s23.s3.inlineinfo;
1088
1089         /* set the javalocals */
1090
1091         jl = DMNEW(s4, iln->jd->maxlocals);
1092         MCOPY(jl, iln->javalocals, s4, iln->jd->maxlocals);
1093         n_ins->sx.s23.s3.inlineinfo->javalocals_end = jl;
1094
1095         DOLOG( printf("%sepilog: ", iln->indent);
1096                    show_icmd(iln->ctx->resultjd, n_ins, false, SHOW_STACK); printf("\n"); );
1097 }
1098
1099
1100 #define TRANSLATE_VAROP(vo)  \
1101         n_iptr->vo.varindex = inline_translate_variable(jd, origjd, varmap, n_iptr->vo.varindex)
1102
1103
1104 static void inline_clone_instruction(inline_node *iln,
1105                                                                          jitdata *jd,
1106                                                                          jitdata *origjd,
1107                                                                          s4 *varmap,
1108                                                                          instruction *o_iptr,
1109                                                                          instruction *n_iptr)
1110 {
1111         icmdtable_entry_t *icmdt;
1112         builtintable_entry *bte;
1113         methoddesc *md;
1114         s4 i, j;
1115         branch_target_t *table;
1116         lookup_target_t *lookup;
1117         inline_node *scope;
1118
1119         *n_iptr = *o_iptr;
1120
1121         icmdt = &(icmd_table[o_iptr->opc]);
1122
1123         switch (icmdt->dataflow) {
1124                 case DF_0_TO_0:
1125                         break;
1126
1127                 case DF_3_TO_0:
1128                         TRANSLATE_VAROP(sx.s23.s3);
1129                 case DF_2_TO_0:
1130                         TRANSLATE_VAROP(sx.s23.s2);
1131                 case DF_1_TO_0:
1132                         TRANSLATE_VAROP(s1);
1133                         break;
1134
1135                 case DF_2_TO_1:
1136                         TRANSLATE_VAROP(sx.s23.s2);
1137                 case DF_1_TO_1:
1138                 case DF_COPY:
1139                 case DF_MOVE:
1140                         TRANSLATE_VAROP(s1);
1141                 case DF_0_TO_1:
1142                         TRANSLATE_VAROP(dst);
1143                         break;
1144
1145                 case DF_N_TO_1:
1146                         n_iptr->sx.s23.s2.args = DMNEW(s4, n_iptr->s1.argcount);
1147                         for (i=0; i<n_iptr->s1.argcount; ++i) {
1148                                 n_iptr->sx.s23.s2.args[i] =
1149                                         inline_translate_variable(jd, origjd, varmap,
1150                                                         o_iptr->sx.s23.s2.args[i]);
1151                         }
1152                         TRANSLATE_VAROP(dst);
1153                         break;
1154
1155                 case DF_INVOKE:
1156                         INSTRUCTION_GET_METHODDESC(n_iptr, md);
1157 clone_call:
1158                         n_iptr->s1.argcount += iln->n_passthroughcount;
1159                         n_iptr->sx.s23.s2.args = DMNEW(s4, n_iptr->s1.argcount);
1160                         for (i=0; i<o_iptr->s1.argcount; ++i) {
1161                                 n_iptr->sx.s23.s2.args[i] =
1162                                         inline_translate_variable(jd, origjd, varmap,
1163                                                         o_iptr->sx.s23.s2.args[i]);
1164                         }
1165                         for (scope = iln; scope != NULL; scope = scope->parent) {
1166                                 for (j = 0; j < scope->n_selfpassthroughcount; ++j) {
1167                                         n_iptr->sx.s23.s2.args[i++] =
1168                                                 scope->parent->varmap[scope->n_passthroughvars[j]];
1169                                 }
1170                         }
1171                         if (md->returntype.type != TYPE_VOID)
1172                                 TRANSLATE_VAROP(dst);
1173                         break;
1174
1175                 case DF_BUILTIN:
1176                         bte = n_iptr->sx.s23.s3.bte;
1177                         md = bte->md;
1178                         goto clone_call;
1179
1180                 default:
1181                         assert(0);
1182         }
1183
1184         switch (icmdt->controlflow) {
1185                 case CF_RET:
1186                         TRANSLATE_VAROP(s1); /* XXX should be handled by data-flow */
1187                         /* FALLTHROUGH */
1188                 case CF_IF:
1189                 case CF_GOTO:
1190                         inline_add_block_reference(iln, &(n_iptr->dst.block));
1191                         break;
1192
1193                 case CF_JSR:
1194                         inline_add_block_reference(iln, &(n_iptr->sx.s23.s3.jsrtarget.block));
1195                         break;
1196
1197                 case CF_TABLE:
1198                         i = n_iptr->sx.s23.s3.tablehigh - n_iptr->sx.s23.s2.tablelow + 1 + 1 /* default */;
1199
1200                         table = DMNEW(branch_target_t, i);
1201                         MCOPY(table, o_iptr->dst.table, branch_target_t, i);
1202                         n_iptr->dst.table = table;
1203
1204                         while (--i >= 0) {
1205                                 inline_add_block_reference(iln, &(table->block));
1206                                 table++;
1207                         }
1208                         break;
1209
1210                 case CF_LOOKUP:
1211                         inline_add_block_reference(iln, &(n_iptr->sx.s23.s3.lookupdefault.block));
1212
1213                         i = n_iptr->sx.s23.s2.lookupcount;
1214                         lookup = DMNEW(lookup_target_t, i);
1215                         MCOPY(lookup, o_iptr->dst.lookup, lookup_target_t, i);
1216                         n_iptr->dst.lookup = lookup;
1217
1218                         while (--i >= 0) {
1219                                 inline_add_block_reference(iln, &(lookup->target.block));
1220                                 lookup++;
1221                         }
1222                         break;
1223         }
1224
1225         /* XXX move this to dataflow section? */
1226
1227         switch (n_iptr->opc) {
1228                 case ICMD_ASTORE:
1229 #if 0
1230                         if (n_iptr->flags.bits & INS_FLAG_RETADDR)
1231                                 inline_add_blocknr_reference(iln, &(n_iptr->sx.s23.s2.retaddrnr));
1232 #endif
1233                         /* FALLTHROUGH! */
1234                 case ICMD_ISTORE:
1235                 case ICMD_LSTORE:
1236                 case ICMD_FSTORE:
1237                 case ICMD_DSTORE:
1238                         stack_javalocals_store(n_iptr, iln->javalocals);
1239                         break;
1240         }
1241 }
1242
1243
1244 static void rewrite_method(inline_node *iln)
1245 {
1246         basicblock *o_bptr;
1247         s4 len;
1248         instruction *o_iptr;
1249         instruction *n_iptr;
1250         inline_node *nextcall;
1251         basicblock *n_bptr;
1252         inline_block_map *bm;
1253         int i;
1254         int icount;
1255         jitdata *resultjd;
1256         jitdata *origjd;
1257         char indent[100]; /* XXX debug */
1258         s4 retcount;
1259         s4 retidx;
1260
1261         assert(iln);
1262
1263         resultjd = iln->ctx->resultjd;
1264         origjd = iln->jd;
1265
1266         n_bptr = NULL;
1267         nextcall = iln->children;
1268
1269         /* XXX debug */
1270         for (i=0; i<iln->depth; ++i)
1271                 indent[i] = '\t';
1272         indent[i] = 0;
1273         iln->indent = indent;
1274
1275         DOLOG( printf("%srewriting: ", indent); method_println(iln->m);
1276                    printf("%s(passthrough: %d+%d)\n",
1277                                 indent, iln->n_passthroughcount - iln->n_selfpassthroughcount,
1278                                 iln->n_passthroughcount); );
1279
1280         /* set memory cursors */
1281
1282         iln->inlined_iinstr_cursor = iln->inlined_iinstr;
1283         iln->inlined_basicblocks_cursor = iln->inlined_basicblocks;
1284
1285         /* allocate temporary buffers */
1286
1287         iln->javalocals = DMNEW(s4, iln->jd->maxlocals);
1288
1289         /* loop over basic blocks */
1290
1291         o_bptr = iln->jd->basicblocks;
1292         for (; o_bptr; o_bptr = o_bptr->next) {
1293
1294                 if (o_bptr->flags < BBREACHED) {
1295
1296                         /* ignore the dummy end block */
1297
1298                         if (o_bptr->icount == 0 && o_bptr->next == NULL) {
1299                                 /* enter the following block as translation, for exception handler ranges */
1300                                 inline_block_translation(iln, o_bptr, iln->inlined_basicblocks_cursor);
1301                                 continue;
1302                         }
1303
1304                         DOLOG(
1305                         printf("%s* skipping old L%03d (flags=%d, type=%d, oid=%d) of ",
1306                                         indent,
1307                                         o_bptr->nr, o_bptr->flags, o_bptr->type,
1308                                         o_bptr->indepth);
1309                         method_println(iln->m);
1310                         );
1311
1312                         n_bptr = create_body_block(iln, o_bptr, iln->varmap);
1313
1314                         /* enter it in the blockmap */
1315
1316                         inline_block_translation(iln, o_bptr, n_bptr);
1317
1318                         close_body_block(iln, n_bptr, o_bptr, iln->varmap, 0, -1);
1319                         continue;
1320                 }
1321
1322                 len = o_bptr->icount;
1323                 o_iptr = o_bptr->iinstr;
1324
1325                 DOLOG(
1326                 printf("%s* rewriting old L%03d (flags=%d, type=%d, oid=%d) of ",
1327                                 indent,
1328                                 o_bptr->nr, o_bptr->flags, o_bptr->type,
1329                                 o_bptr->indepth);
1330                 method_println(iln->m);
1331                 show_basicblock(iln->jd, o_bptr, SHOW_STACK);
1332                 );
1333
1334                 if (iln->blockbefore || o_bptr != iln->jd->basicblocks) {
1335                         /* create an inlined clone of this block */
1336
1337                         n_bptr = create_body_block(iln, o_bptr, iln->varmap);
1338                         icount = 0;
1339
1340                         /* enter it in the blockmap */
1341
1342                         inline_block_translation(iln, o_bptr, n_bptr);
1343
1344                         /* initialize the javalocals */
1345
1346                         MCOPY(iln->javalocals, n_bptr->javalocals, s4, iln->jd->maxlocals);
1347                 }
1348                 else {
1349                         s4 *jl;
1350
1351                         /* continue caller block */
1352
1353                         n_bptr = iln->inlined_basicblocks_cursor - 1;
1354                         icount = n_bptr->icount;
1355
1356                         /* translate the javalocals */
1357
1358                         jl = translate_javalocals(iln, o_bptr->javalocals);
1359                         iln->inline_start_instruction->sx.s23.s3.inlineinfo->javalocals_start = jl;
1360
1361                         MCOPY(iln->javalocals, jl, s4, iln->jd->maxlocals);
1362                 }
1363
1364                 /* iterate over the ICMDs of this block */
1365
1366                 retcount = 0;
1367                 retidx = UNUSED;
1368
1369                 while (--len >= 0) {
1370
1371                         DOLOG( fputs(indent, stdout); show_icmd(iln->jd, o_iptr, false,  SHOW_STACK);
1372                                    printf("\n") );
1373
1374                         /* handle calls that will be inlined */
1375
1376                         if (nextcall && o_iptr == nextcall->callerins) {
1377
1378                                 /* write the inlining prolog */
1379
1380                                 (void) emit_inlining_prolog(iln, nextcall, o_iptr, iln->varmap);
1381                                 icount += nextcall->prolog_instructioncount;
1382
1383                                 /* end current block, or glue blocks together */
1384
1385                                 n_bptr->icount = icount;
1386
1387                                 if (nextcall->blockbefore) {
1388                                         close_prolog_block(iln, n_bptr, nextcall);
1389                                 }
1390                                 else {
1391                                         /* XXX */
1392                                 }
1393
1394                                 /* check if the result is a local variable */
1395
1396                                 if (nextcall->m->parseddesc->returntype.type != TYPE_VOID
1397                                                 && o_iptr->dst.varindex < iln->jd->localcount)
1398                                 {
1399                                         nextcall->n_resultlocal = iln->varmap[o_iptr->dst.varindex];
1400                                 }
1401                                 else
1402                                         nextcall->n_resultlocal = -1;
1403
1404                                 /* set memory pointers in the callee */
1405
1406                                 nextcall->inlined_iinstr = iln->inlined_iinstr_cursor;
1407                                 nextcall->inlined_basicblocks = iln->inlined_basicblocks_cursor;
1408
1409                                 /* recurse */
1410
1411                                 DOLOG( printf("%sentering inline ", indent);
1412                                            show_icmd(origjd, o_iptr, false, SHOW_STACK); printf("\n") );
1413
1414                                 rewrite_method(nextcall);
1415
1416                                 DOLOG( printf("%sleaving inline ", indent);
1417                                            show_icmd(origjd, o_iptr, false, SHOW_STACK); printf("\n") );
1418
1419                                 /* update memory cursors */
1420
1421                                 assert(nextcall->inlined_iinstr_cursor
1422                                                 <= iln->inlined_iinstr_cursor + nextcall->cumul_instructioncount);
1423                                 assert(nextcall->inlined_basicblocks_cursor
1424                                                 == iln->inlined_basicblocks_cursor + nextcall->cumul_basicblockcount);
1425                                 iln->inlined_iinstr_cursor = nextcall->inlined_iinstr_cursor;
1426                                 iln->inlined_basicblocks_cursor = nextcall->inlined_basicblocks_cursor;
1427
1428                                 /* start new block, or glue blocks together */
1429
1430                                 if (nextcall->blockafter) {
1431                                         n_bptr = create_epilog_block(iln, nextcall, iln->varmap);
1432                                         icount = 0;
1433                                 }
1434                                 else {
1435                                         n_bptr = iln->inlined_basicblocks_cursor - 1;
1436                                         icount = n_bptr->icount;
1437                                         /* XXX */
1438                                 }
1439
1440                                 /* emit inlining epilog */
1441
1442                                 emit_inlining_epilog(iln, nextcall, o_iptr);
1443                                 icount += nextcall->epilog_instructioncount;
1444
1445                                 /* proceed to next call */
1446
1447                                 nextcall = nextcall->next;
1448                         }
1449                         else {
1450                                 n_iptr = (iln->inlined_iinstr_cursor++);
1451                                 assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1452
1453                                 switch (o_iptr->opc) {
1454                                         case ICMD_RETURN:
1455                                                 if (iln->depth == 0)
1456                                                         goto default_clone;
1457                                                 goto return_tail;
1458
1459                                         case ICMD_IRETURN:
1460                                         case ICMD_ARETURN:
1461                                         case ICMD_LRETURN:
1462                                         case ICMD_FRETURN:
1463                                         case ICMD_DRETURN:
1464                                                 if (iln->depth == 0)
1465                                                         goto default_clone;
1466                                                 retcount = 1;
1467                                                 retidx = iln->varmap[o_iptr->s1.varindex];
1468                                                 if (iln->n_resultlocal != -1) {
1469                                                         /* store result in a local variable */
1470
1471                                                         DOLOG( printf("USING RESULTLOCAL %d\n", iln->n_resultlocal); );
1472                                                         /* This relies on the same sequence of types for */
1473                                                         /* ?STORE and ?RETURN opcodes.                   */
1474                                                         n_iptr->opc = ICMD_ISTORE + (o_iptr->opc - ICMD_IRETURN);
1475                                                         n_iptr->s1.varindex = retidx;
1476                                                         n_iptr->dst.varindex = iln->n_resultlocal;
1477                                                         n_iptr->sx.s23.s3.javaindex = UNUSED; /* XXX set real javaindex? */
1478
1479                                                         retcount = 0;
1480                                                         retidx = UNUSED;
1481
1482                                                         n_iptr = (iln->inlined_iinstr_cursor++);
1483                                                         assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1484                                                         icount++;
1485                                                 }
1486                                                 else if ((retidx < resultjd->localcount && iln->blockafter)
1487                                                                 || !iln->blockafter) /* XXX do we really always need the MOVE? */
1488                                                 {
1489                                                         /* local must not become outvar, insert a MOVE */
1490
1491                                                         n_iptr->opc = ICMD_MOVE;
1492                                                         n_iptr->s1.varindex = retidx;
1493                                                         retidx = inline_new_temp_variable(resultjd,
1494                                                                                                                           resultjd->var[retidx].type);
1495                                                         n_iptr->dst.varindex = retidx;
1496
1497                                                         n_iptr = (iln->inlined_iinstr_cursor++);
1498                                                         assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1499                                                         icount++;
1500                                                 }
1501 return_tail:
1502                                                 if (iln->blockafter) {
1503                                                         n_iptr->opc = ICMD_GOTO;
1504                                                         n_iptr->dst.block = INLINE_RETURN_REFERENCE(iln);
1505                                                         inline_add_block_reference(iln, &(n_iptr->dst.block));
1506                                                 }
1507                                                 else {
1508                                                         n_iptr->opc = ICMD_NOP;
1509                                                 }
1510                                                 break;
1511 #if 0
1512                                                 if (o_bptr->next == NULL || (o_bptr->next->icount==0 && o_bptr->next->next == NULL)) {
1513                                                         n_iptr->opc = ICMD_NOP;
1514                                                         break;
1515                                                 }
1516                                                 goto default_clone;
1517                                                 break;
1518 #endif
1519
1520                                         default:
1521 default_clone:
1522                                                 inline_clone_instruction(iln, resultjd, iln->jd, iln->varmap, o_iptr, n_iptr);
1523                                 }
1524
1525                                 DOLOG( fputs(indent, stdout); show_icmd(resultjd, n_iptr, false, SHOW_STACK);
1526                                            printf("\n"););
1527
1528                                 icount++;
1529                         }
1530
1531                         o_iptr++;
1532                 }
1533
1534                 /* end of basic block */
1535
1536                 if (iln->blockafter || (o_bptr->next && o_bptr->next->next)) {
1537                         close_body_block(iln, n_bptr, o_bptr, iln->varmap, retcount, retidx);
1538                         n_bptr->icount = icount;
1539
1540                         DOLOG( printf("closed body block:\n");
1541                                    show_basicblock(resultjd, n_bptr, SHOW_STACK); );
1542                 }
1543                 else {
1544                         n_bptr->icount = icount;
1545                         assert(iln->parent);
1546                         if (retidx != UNUSED)
1547                                 iln->parent->varmap[iln->callerins->dst.varindex] = retidx;
1548                 }
1549         }
1550
1551         bm = iln->ctx->blockmap;
1552         for (i=0; i<iln->ctx->blockmap_index; ++i, ++bm) {
1553                 assert(bm->iln && bm->o_block && bm->n_block);
1554                 if (bm->iln == iln)
1555                         inline_resolve_block_refs(&(iln->refs), bm->o_block, bm->n_block, false);
1556         }
1557
1558 #if !defined(NDEBUG)
1559         if (iln->refs) {
1560                 inline_target_ref *ref;
1561                 ref = iln->refs;
1562                 while (ref) {
1563                         if (!iln->depth || ref->isnumber || *(ref->ref.block) != INLINE_RETURN_REFERENCE(iln)) {
1564                                 DOLOG( printf("XXX REMAINING REF at depth %d: %p\n", iln->depth,
1565                                            (void*)*(ref->ref.block)) );
1566                                 assert(false);
1567                         }
1568                         ref = ref->next;
1569                 }
1570         }
1571 #endif
1572 }
1573
1574
1575 static exception_entry * inline_exception_tables(inline_node *iln,
1576                                                                                                  exception_entry *n_extable,
1577                                                                                                  exception_entry **prevextable)
1578 {
1579         inline_node *child;
1580         inline_node *scope;
1581         exception_entry *et;
1582
1583         assert(iln);
1584         assert(n_extable);
1585         assert(prevextable);
1586
1587         child = iln->children;
1588         if (child) {
1589                 do {
1590                         n_extable = inline_exception_tables(child, n_extable, prevextable);
1591                         child = child->next;
1592                 } while (child != iln->children);
1593         }
1594
1595         et = iln->jd->exceptiontable;
1596         for (; et != NULL; et = et->down) {
1597                 assert(et);
1598                 MZERO(n_extable, exception_entry, 1);
1599                 n_extable->start     = inline_map_block(iln, et->start  , iln);
1600                 n_extable->end       = inline_map_block(iln, et->end    , iln);
1601                 n_extable->handler   = inline_map_block(iln, et->handler, iln);
1602                 n_extable->catchtype = et->catchtype;
1603
1604                 if (*prevextable) {
1605                         (*prevextable)->down = n_extable;
1606                 }
1607                 *prevextable = n_extable;
1608
1609                 n_extable++;
1610         }
1611
1612         if (iln->handler_monitorexit) {
1613                 exception_entry **activehandlers;
1614
1615                 MZERO(n_extable, exception_entry, 1);
1616                 n_extable->start   = iln->inlined_basicblocks;
1617                 n_extable->end     = iln->inlined_basicblocks_cursor;
1618                 n_extable->handler = iln->handler_monitorexit;
1619                 n_extable->catchtype.any = NULL; /* finally */
1620
1621                 if (*prevextable) {
1622                         (*prevextable)->down = n_extable;
1623                 }
1624                 *prevextable = n_extable;
1625
1626                 n_extable++;
1627
1628                 /* We have to protect the created handler with the same handlers */
1629                 /* that protect the method body itself.                          */
1630
1631                 for (scope = iln; scope->parent != NULL; scope = scope->parent) {
1632
1633                         activehandlers = scope->o_handlers;
1634                         assert(activehandlers);
1635
1636                         while (*activehandlers) {
1637
1638                                 assert(scope->parent);
1639
1640                                 MZERO(n_extable, exception_entry, 1);
1641                                 n_extable->start     = iln->handler_monitorexit;
1642                                 n_extable->end       = iln->handler_monitorexit + 1; /* XXX ok in this case? */
1643                                 n_extable->handler   = inline_map_block(scope->parent,
1644                                                                                                                 (*activehandlers)->handler,
1645                                                                                                                 scope->parent);
1646                                 n_extable->catchtype = (*activehandlers)->catchtype;
1647
1648                                 if (*prevextable) {
1649                                         (*prevextable)->down = n_extable;
1650                                 }
1651                                 *prevextable = n_extable;
1652
1653                                 n_extable++;
1654                                 activehandlers++;
1655                         }
1656                 }
1657         }
1658
1659         return n_extable;
1660 }
1661
1662
1663 static void inline_locals(inline_node *iln)
1664 {
1665         inline_node *child;
1666
1667         assert(iln);
1668
1669         iln->varmap = create_variable_map(iln);
1670
1671         child = iln->children;
1672         if (child) {
1673                 do {
1674                         inline_locals(child);
1675                         child = child->next;
1676                 } while (child != iln->children);
1677         }
1678
1679         if (iln->regdata->memuse > iln->ctx->resultjd->rd->memuse)
1680                 iln->ctx->resultjd->rd->memuse = iln->regdata->memuse;
1681         if (iln->regdata->argintreguse > iln->ctx->resultjd->rd->argintreguse)
1682                 iln->ctx->resultjd->rd->argintreguse = iln->regdata->argintreguse;
1683         if (iln->regdata->argfltreguse > iln->ctx->resultjd->rd->argfltreguse)
1684                 iln->ctx->resultjd->rd->argfltreguse = iln->regdata->argfltreguse;
1685 }
1686
1687
1688 static void inline_interface_variables(inline_node *iln)
1689 {
1690         basicblock *bptr;
1691         jitdata *resultjd;
1692         s4 i;
1693         varinfo *v;
1694
1695         resultjd = iln->ctx->resultjd;
1696
1697         resultjd->interface_map = DMNEW(interface_info, 5*iln->ctx->maxinoutdepth);
1698         for (i=0; i<5*iln->ctx->maxinoutdepth; ++i)
1699                 resultjd->interface_map[i].flags = UNUSED;
1700
1701         for (bptr = resultjd->basicblocks; bptr != NULL; bptr = bptr->next) {
1702                 assert(bptr->indepth  <= iln->ctx->maxinoutdepth);
1703                 assert(bptr->outdepth <= iln->ctx->maxinoutdepth);
1704
1705                 for (i=0; i<bptr->indepth; ++i) {
1706                         v = &(resultjd->var[bptr->invars[i]]);
1707                         v->flags |= INOUT;
1708                         if (v->type == TYPE_RET)
1709                                 v->flags |= PREALLOC;
1710                         else
1711                                 v->flags &= ~PREALLOC;
1712                         v->flags &= ~INMEMORY;
1713                         assert(bptr->invars[i] >= resultjd->localcount);
1714
1715                         if (resultjd->interface_map[5*i + v->type].flags == UNUSED) {
1716                                 resultjd->interface_map[5*i + v->type].flags = v->flags;
1717                         }
1718                         else {
1719                                 resultjd->interface_map[5*i + v->type].flags |= v->flags;
1720                         }
1721                 }
1722
1723                 for (i=0; i<bptr->outdepth; ++i) {
1724                         v = &(resultjd->var[bptr->outvars[i]]);
1725                         v->flags |= INOUT;
1726                         if (v->type == TYPE_RET)
1727                                 v->flags |= PREALLOC;
1728                         else
1729                                 v->flags &= ~PREALLOC;
1730                         v->flags &= ~INMEMORY;
1731                         assert(bptr->outvars[i] >= resultjd->localcount);
1732
1733                         if (resultjd->interface_map[5*i + v->type].flags == UNUSED) {
1734                                 resultjd->interface_map[5*i + v->type].flags = v->flags;
1735                         }
1736                         else {
1737                                 resultjd->interface_map[5*i + v->type].flags |= v->flags;
1738                         }
1739                 }
1740         }
1741 }
1742
1743
1744 static void inline_write_exception_handlers(inline_node *master, inline_node *iln)
1745 {
1746         basicblock *n_bptr;
1747         instruction *n_ins;
1748         inline_node *child;
1749         builtintable_entry *bte;
1750         s4 exvar;
1751         s4 syncvar;
1752         s4 i;
1753
1754         child = iln->children;
1755         if (child) {
1756                 do {
1757                         inline_write_exception_handlers(master, child);
1758                         child = child->next;
1759                 } while (child != iln->children);
1760         }
1761
1762         if (iln->synchronize) {
1763                 /* create the monitorexit handler */
1764                 n_bptr = create_block(master, iln, iln,
1765                                                           iln->n_passthroughcount + 1);
1766                 n_bptr->type = BBTYPE_EXH;
1767                 n_bptr->flags = BBFINISHED;
1768
1769                 exvar = inline_new_variable(master->ctx->resultjd, TYPE_ADR, 0);
1770                 n_bptr->invars[iln->n_passthroughcount] = exvar;
1771
1772                 iln->handler_monitorexit = n_bptr;
1773
1774                 /* ACONST / ALOAD */
1775
1776                 n_ins = master->inlined_iinstr_cursor++;
1777                 if (iln->m->flags & ACC_STATIC) {
1778                         n_ins->opc = ICMD_ACONST;
1779                         n_ins->sx.val.c.cls = iln->m->class;
1780                         n_ins->flags.bits = INS_FLAG_CLASS;
1781                 }
1782                 else {
1783                         n_ins->opc = ICMD_ALOAD;
1784                         n_ins->s1.varindex = iln->synclocal;
1785                         assert(n_ins->s1.varindex != UNUSED);
1786                 }
1787                 /* XXX could be PREALLOCed for  builtin call */
1788                 syncvar = inline_new_variable(master->ctx->resultjd, TYPE_ADR, 0);
1789                 n_ins->dst.varindex = syncvar;
1790                 n_ins->line = 0;
1791
1792                 /* MONITOREXIT */
1793
1794                 bte = builtintable_get_internal(LOCK_monitor_exit);
1795
1796                 n_ins = master->inlined_iinstr_cursor++;
1797                 n_ins->opc = ICMD_BUILTIN;
1798                 n_ins->s1.argcount = 1 + iln->n_passthroughcount + 1;
1799                 n_ins->sx.s23.s2.args = DMNEW(s4, n_ins->s1.argcount);
1800                 n_ins->sx.s23.s2.args[0] = syncvar;
1801                 for (i=0; i < iln->n_passthroughcount + 1; ++i) {
1802                         n_ins->sx.s23.s2.args[1 + i] = n_bptr->invars[i];
1803                 }
1804                 n_ins->sx.s23.s3.bte = bte;
1805                 n_ins->line = 0;
1806
1807                 /* ATHROW */
1808
1809                 n_ins = master->inlined_iinstr_cursor++;
1810                 n_ins->opc = ICMD_ATHROW;
1811                 n_ins->flags.bits = 0;
1812                 n_ins->s1.varindex = exvar;
1813                 n_ins->line = 0;
1814
1815                 /* close basic block */
1816
1817                 close_block(iln, iln, n_bptr, iln->n_passthroughcount);
1818                 n_bptr->icount = 3;
1819         }
1820 }
1821
1822
1823 /* second pass driver *********************************************************/
1824
1825 static bool test_inlining(inline_node *iln, jitdata *jd)
1826 {
1827         instruction *n_ins;
1828         basicblock *n_bb;
1829         basicblock *n_bptr;
1830         exception_entry *n_ext;
1831         exception_entry *prevext;
1832         codegendata *n_cd;
1833         jitdata *n_jd;
1834         s4 i;
1835
1836
1837 #if !defined(NDEBUG)
1838         static int debug_verify_inlined_code = 1; /* XXX */
1839         static int debug_compile_inlined_code_counter = 0;
1840 #endif
1841
1842         DOLOG( dump_inline_tree(iln, 0); );
1843
1844         assert(iln && jd);
1845
1846         n_ins = DMNEW(instruction, iln->cumul_instructioncount);
1847         MZERO(n_ins, instruction, iln->cumul_instructioncount);
1848         iln->inlined_iinstr = n_ins;
1849
1850         n_bb = DMNEW(basicblock, iln->cumul_basicblockcount);
1851         MZERO(n_bb, basicblock, iln->cumul_basicblockcount);
1852         iln->inlined_basicblocks = n_bb;
1853
1854         iln->ctx->blockmap = DMNEW(inline_block_map, iln->cumul_blockmapcount);
1855
1856         n_jd = jit_jitdata_new(iln->m);
1857         n_jd->flags = jd->flags;
1858         n_jd->code->optlevel = jd->code->optlevel;
1859         iln->ctx->resultjd = n_jd;
1860
1861         reg_setup(n_jd);
1862
1863         /* create the local_map */
1864
1865         n_jd->local_map = DMNEW(s4, 5*iln->cumul_maxlocals);
1866         for (i=0; i<5*iln->cumul_maxlocals; ++i)
1867                 n_jd->local_map[i] = UNUSED;
1868
1869         /* create / coalesce local variables */
1870
1871         n_jd->varcount = 0;
1872         n_jd->vartop = 0;
1873         n_jd->var = NULL;
1874
1875         inline_locals(iln);
1876
1877         n_jd->localcount = n_jd->vartop;
1878
1879         /* extra variables for verification (DEBUG) */
1880
1881 #if !defined(NDEBUG)
1882         if (debug_verify_inlined_code) {
1883                 n_jd->vartop   += VERIFIER_EXTRA_LOCALS + VERIFIER_EXTRA_VARS + 100 /* XXX m->maxstack */;
1884                 if (n_jd->vartop > n_jd->varcount) {
1885                         /* XXX why? */
1886                         n_jd->var = DMREALLOC(n_jd->var, varinfo, n_jd->varcount, n_jd->vartop);
1887                         n_jd->varcount = n_jd->vartop;
1888                 }
1889         }
1890 #endif
1891
1892         /* write inlined code */
1893
1894         rewrite_method(iln);
1895
1896         /* create exception handlers */
1897
1898         inline_write_exception_handlers(iln, iln);
1899
1900         /* write the dummy end block */
1901
1902         n_bptr = create_block(iln, iln, iln, 0);
1903         n_bptr->flags = BBUNDEF;
1904         n_bptr->type = BBTYPE_STD;
1905
1906         /* store created code in jitdata */
1907
1908         n_jd->basicblocks = iln->inlined_basicblocks;
1909         n_jd->basicblockindex = NULL;
1910         n_jd->instructioncount = iln->cumul_instructioncount;
1911         n_jd->instructions = iln->inlined_iinstr;
1912
1913         /* link the basic blocks (dummy end block is not counted) */
1914
1915         n_jd->basicblockcount = (iln->inlined_basicblocks_cursor - iln->inlined_basicblocks) - 1;
1916         for (i=0; i<n_jd->basicblockcount + 1; ++i)
1917                 n_jd->basicblocks[i].next = &(n_jd->basicblocks[i+1]);
1918         if (i)
1919                 n_jd->basicblocks[i-1].next = NULL;
1920
1921         /* check basicblock numbers */
1922
1923 #if !defined(NDEBUG)
1924         jit_check_basicblock_numbers(n_jd);
1925 #endif
1926
1927         /* create the exception table */
1928
1929         if (iln->cumul_exceptiontablelength) {
1930                 exception_entry *tableend;
1931
1932                 n_ext = DMNEW(exception_entry, iln->cumul_exceptiontablelength);
1933                 prevext = NULL;
1934                 tableend = inline_exception_tables(iln, n_ext, &prevext);
1935                 assert(tableend == n_ext + iln->cumul_exceptiontablelength);
1936                 if (prevext)
1937                         prevext->down = NULL;
1938
1939                 n_jd->exceptiontablelength = iln->cumul_exceptiontablelength;
1940                 n_jd->exceptiontable = n_ext;
1941         }
1942         else {
1943                 n_ext = NULL;
1944         }
1945
1946         /*******************************************************************************/
1947
1948         n_cd = n_jd->cd;
1949         memcpy(n_cd, jd->cd, sizeof(codegendata));
1950
1951         n_cd->method = NULL; /* XXX */
1952         n_jd->maxlocals = iln->cumul_maxlocals;
1953         n_jd->maxinterfaces = iln->ctx->maxinoutdepth;
1954
1955         inline_post_process(n_jd);
1956
1957         inline_interface_variables(iln);
1958
1959 #if defined(ENABLE_VERIFIER) && !defined(NDEBUG) && 0
1960         if (debug_verify_inlined_code) {
1961                 debug_verify_inlined_code = 0;
1962                 DOLOG( printf("VERIFYING INLINED RESULT...\n"); fflush(stdout); );
1963                 if (!typecheck(n_jd)) {
1964                         *exceptionptr = NULL;
1965                         DOLOG( printf("XXX INLINED RESULT DID NOT PASS VERIFIER XXX\n") );
1966                         return false;
1967                 }
1968                 else {
1969                         DOLOG( printf("VERIFICATION PASSED.\n") );
1970                 }
1971                 debug_verify_inlined_code = 1;
1972         }
1973 #endif /* defined(ENABLE_VERIFIER) */
1974
1975         /* we need bigger free memory stacks (XXX these should not be allocated in reg_setup) */
1976
1977         n_jd->rd->freemem = DMNEW(s4, iln->ctx->maxinoutdepth + 1000) /* XXX max vars/block */;
1978 #if defined(HAS_4BYTE_STACKSLOT)
1979         n_jd->rd->freemem_2 = DMNEW(s4, iln->ctx->maxinoutdepth + 1000) /* XXX max vars/block */;
1980 #endif
1981
1982 #if !defined(NDEBUG)
1983         if (n_jd->instructioncount >= inline_debug_min_size
1984                         && n_jd->instructioncount <= inline_debug_max_size)
1985         {
1986            if (debug_compile_inlined_code_counter >= inline_debug_start_counter
1987                            && debug_compile_inlined_code_counter <= inline_debug_end_counter)
1988 #endif /* NDEBUG */
1989            {
1990                         /* install the inlined result */
1991
1992                         *jd->code = *n_jd->code;
1993                         n_jd->code = jd->code;
1994                         *jd = *n_jd;
1995
1996 #if !defined(NDEBUG)
1997                         inline_count_methods++;
1998
1999                         /* inline_debug_log++; */
2000                         DOLOG(
2001                         printf("==== %d.INLINE ==================================================================\n", debug_compile_inlined_code_counter);
2002                         printf("\ninline tree:\n");
2003                         dump_inline_tree(iln, 0);
2004                         n_jd->flags |= JITDATA_FLAG_SHOWINTERMEDIATE | JITDATA_FLAG_SHOWDISASSEMBLE;
2005                         /* debug_dump_inlined_code(iln, n_method, n_cd, n_rd); */
2006                         printf("-------- DONE -----------------------------------------------------------\n");
2007                         fflush(stdout);
2008                         );
2009                         /* inline_debug_log--; */
2010 #endif
2011            }
2012
2013 #if !defined(NDEBUG)
2014                 debug_compile_inlined_code_counter++;
2015         }
2016 #endif
2017         return true;
2018 }
2019
2020
2021 /* first pass: build inlining tree ********************************************/
2022
2023 static bool inline_analyse_callee(inline_node *caller,
2024                                                                   methodinfo *callee,
2025                                                                   basicblock *callerblock,
2026                                                                   instruction *calleriptr,
2027                                                                   s4 callerpc,
2028                                                                   exception_entry **handlers,
2029                                                                   s4 nhandlers)
2030 {
2031         inline_node *cn;              /* the callee inline_node */
2032         s4           argi;
2033         bool         isstatic;
2034         s4           i, j;
2035         basicblock  *bptr;
2036
2037         /* create an inline tree node */
2038
2039         cn = DNEW(inline_node);
2040         MZERO(cn, inline_node, 1);
2041
2042         cn->ctx = caller->ctx;
2043         cn->m = callee;
2044         cn->synchronize = (callee->flags & ACC_SYNCHRONIZED);
2045         isstatic = (callee->flags & ACC_STATIC);
2046
2047         /* get the intermediate representation of the callee */
2048
2049         if (!inline_jit_compile(cn))
2050                 return false;
2051
2052         /* info about the call site */
2053
2054         cn->depth = caller->depth + 1;
2055         cn->callerblock = callerblock;
2056         cn->callerins = calleriptr;
2057         cn->callerpc = callerpc;
2058         cn->o_handlers = handlers;
2059         cn->n_handlercount = caller->n_handlercount + nhandlers;
2060
2061         /* determine if we need basic block boundaries before/after */
2062
2063         cn->blockbefore = false;
2064         cn->blockafter = false;
2065
2066         if (cn->jd->branchtoentry)
2067                 cn->blockbefore = true;
2068
2069         if (cn->jd->branchtoend)
2070                 cn->blockafter = true;
2071
2072         if (cn->jd->returncount > 1)
2073                 cn->blockafter = true;
2074
2075         /* XXX make safer and reusable (maybe store last real block) */
2076         for (bptr = cn->jd->basicblocks; bptr && bptr->next && bptr->next->next; bptr = bptr->next)
2077                 ;
2078
2079         if (cn->jd->returnblock != bptr)
2080                 cn->blockafter = true;
2081
2082         /* info about the callee */
2083
2084         cn->localsoffset = caller->localsoffset + caller->m->maxlocals;
2085         cn->prolog_instructioncount = callee->parseddesc->paramcount + 2;
2086         cn->epilog_instructioncount = 1; /* INLINE_END */
2087         cn->extra_instructioncount = 0;
2088
2089         /* we need a CHECKNULL for instance methods, except for <init> */
2090
2091         if (!isstatic && callee->name != utf_init)
2092                 cn->prolog_instructioncount += 1;
2093
2094         /* deal with synchronized callees */
2095
2096         if (cn->synchronize) {
2097                 methoddesc         *md;
2098                 builtintable_entry *bte;
2099
2100                 /* we need basic block boundaries because of the handler */
2101
2102                 cn->blockbefore = true;
2103                 cn->blockafter = true;
2104
2105                 /* for synchronized static methods                 */
2106                 /* we need an ACONST, MONITORENTER in the prolog   */
2107                 /* and ACONST, MONITOREXIT in the epilog           */
2108
2109                 /* for synchronized instance methods               */
2110                 /* we need an COPY, MONITORENTER in the prolog     */
2111                 /* and MONITOREXIT in the epilog                   */
2112
2113                 if (isstatic) {
2114                         cn->prolog_instructioncount += 2;
2115                         cn->epilog_instructioncount += 2;
2116                 }
2117                 else {
2118                         cn->prolog_instructioncount += 2;
2119                         cn->epilog_instructioncount += 1;
2120                         cn->localsoffset += 1;
2121                 }
2122
2123                 /* and exception handler */
2124                 /* ALOAD, builtin_monitorexit, ATHROW */
2125
2126                 cn->extra_instructioncount += 3;
2127
2128                 /* exception table entries */
2129
2130                 caller->cumul_exceptiontablelength += 1 + cn->n_handlercount;
2131
2132                 /* we must call the builtins */
2133
2134                 bte = builtintable_get_internal(LOCK_monitor_enter);
2135                 md = bte->md;
2136                 if (md->memuse > cn->regdata->memuse)
2137                         cn->regdata->memuse = md->memuse;
2138                 if (md->argintreguse > cn->regdata->argintreguse)
2139                         cn->regdata->argintreguse = md->argintreguse;
2140
2141                 bte = builtintable_get_internal(LOCK_monitor_exit);
2142                 md = bte->md;
2143                 if (md->memuse > cn->regdata->memuse)
2144                         cn->regdata->memuse = md->memuse;
2145                 if (md->argintreguse > cn->regdata->argintreguse)
2146                         cn->regdata->argintreguse = md->argintreguse;
2147
2148                 caller->ctx->calls_others = true;
2149         }
2150
2151         /* determine pass-through variables */
2152
2153         i = calleriptr->s1.argcount - callee->parseddesc->paramcount; /* max # of pass-though vars */
2154
2155         cn->n_passthroughvars = DMNEW(s4, i);
2156         j = 0;
2157         for (argi = calleriptr->s1.argcount - 1; argi >= callee->parseddesc->paramcount; --argi) {
2158                 s4 idx = calleriptr->sx.s23.s2.args[argi];
2159                 if (idx >= caller->jd->localcount) {
2160                         cn->n_passthroughvars[j] = idx;
2161                         j++;
2162                 }
2163                 else {
2164                         DOLOG( printf("PASSING THROUGH LOCAL VARIABLE %d\n", idx); );
2165                 }
2166         }
2167         assert(j <= i);
2168         cn->n_selfpassthroughcount = j;
2169         cn->n_passthroughcount = caller->n_passthroughcount + cn->n_selfpassthroughcount;
2170
2171         /* insert the node into the inline tree */
2172
2173         insert_inline_node(caller, cn);
2174
2175         /* analyse recursively */
2176
2177         if (!inline_inline_intern(callee, cn))
2178                 return false;
2179
2180         /* subtract one block if we continue the caller block */
2181
2182         if (!cn->blockbefore)
2183                 cn->cumul_basicblockcount -= 1;
2184
2185         /* add exception handler block for synchronized callees */
2186
2187         if (cn->synchronize) {
2188                 caller->ctx->master->cumul_basicblockcount++;
2189                 caller->ctx->master->cumul_blockmapcount++;
2190         }
2191
2192         /* cumulate counters */
2193
2194         caller->cumul_instructioncount += cn->prolog_instructioncount;
2195         caller->cumul_instructioncount += cn->epilog_instructioncount;
2196         caller->cumul_instructioncount += cn->extra_instructioncount;
2197         caller->cumul_instructioncount += cn->cumul_instructioncount - 1 /*invoke*/;
2198
2199         caller->cumul_basicblockcount += cn->cumul_basicblockcount;
2200         caller->cumul_blockmapcount += cn->cumul_blockmapcount;
2201         caller->cumul_exceptiontablelength += cn->cumul_exceptiontablelength;
2202         if (cn->cumul_maxlocals > caller->cumul_maxlocals)
2203                 caller->cumul_maxlocals = cn->cumul_maxlocals;
2204
2205         if (caller->cumul_basicblockcount > 10*caller->ctx->master->jd->basicblockcount) {
2206 #if 0
2207                 printf("STOPPING to avoid code explosion (%d blocks)\n",
2208                                 caller->cumul_basicblockcount);
2209 #endif
2210                 return false;
2211         }
2212
2213         /* XXX extra block after inlined call */
2214         if (cn->blockafter) {
2215                 caller->cumul_basicblockcount += 1;
2216                 caller->cumul_blockmapcount += 1;
2217         }
2218
2219         return true;
2220 }
2221
2222
2223 static bool inline_inline_intern(methodinfo *m, inline_node *iln)
2224 {
2225         basicblock *bptr;
2226         s4 len;
2227         instruction *iptr;
2228         int opcode;                                   /* invocation opcode */
2229         methodinfo *callee;
2230         inline_node *active;
2231         exception_entry **handlers;
2232         exception_entry *ex;
2233         s4 nhandlers;
2234         jitdata *mjd;
2235         bool speculative;
2236
2237         assert(iln);
2238
2239         mjd = iln->jd;
2240
2241         /* initialize cumulative counters */
2242
2243         iln->cumul_maxlocals = iln->localsoffset + m->maxlocals;
2244         iln->cumul_exceptiontablelength += mjd->exceptiontablelength;
2245
2246         /* iterate over basic blocks */
2247
2248         for (bptr = mjd->basicblocks; bptr; bptr = bptr->next) {
2249
2250                 /* ignore dummy end blocks (but count them for the blockmap) */
2251
2252                 iln->cumul_blockmapcount++;
2253                 if (bptr->icount > 0 || bptr->next != NULL)
2254                         iln->cumul_basicblockcount++;
2255
2256                 /* skip dead code */
2257
2258                 if (bptr->flags < BBREACHED)
2259                         continue;
2260
2261                 /* allocate the buffer of active exception handlers */
2262                 /* XXX this wastes some memory, but probably it does not matter */
2263
2264                 handlers = DMNEW(exception_entry*, mjd->exceptiontablelength + 1);
2265
2266                 /* determine the active exception handlers for this block     */
2267                 /* XXX maybe the handlers of a block should be part of our IR */
2268                 /* XXX this should share code with the type checkers          */
2269                 nhandlers = 0;
2270                 for (ex = mjd->exceptiontable; ex ; ex = ex->down) {
2271                         if ((ex->start->nr <= bptr->nr) && (ex->end->nr > bptr->nr)) {
2272                                 handlers[nhandlers++] = ex;
2273                         }
2274                 }
2275                 handlers[nhandlers] = NULL;
2276
2277                 len = bptr->icount;
2278                 iptr = bptr->iinstr;
2279
2280                 iln->instructioncount += len;
2281                 iln->cumul_instructioncount += len;
2282
2283                 for (; --len >= 0; ++iptr) {
2284
2285                         opcode = iptr->opc;
2286
2287                         switch (opcode) {
2288                                 /****************************************/
2289                                 /* INVOKATIONS                          */
2290
2291                                 case ICMD_INVOKEVIRTUAL:
2292                                 case ICMD_INVOKESPECIAL:
2293                                 case ICMD_INVOKESTATIC:
2294                                 case ICMD_INVOKEINTERFACE:
2295
2296                                         if (!INSTRUCTION_IS_UNRESOLVED(iptr)) {
2297                                                 callee = iptr->sx.s23.s3.fmiref->p.method;
2298
2299 #if 0
2300                                                 if (
2301                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Code_Table") == 0
2302                                                                 && (strcmp(callee->name->text, "of") == 0
2303                                                                  || strcmp(callee->name->text, "set") == 0))
2304                                                         ||
2305                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Compressor") == 0
2306                                                                 && (strcmp(callee->name->text, "output") == 0))
2307                                                         ||
2308                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Decompressor") == 0
2309                                                                 && (strcmp(callee->name->text, "getcode") == 0))
2310                                                         ||
2311                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Output_Buffer") == 0
2312                                                                 && (strcmp(callee->name->text, "putbyte") == 0))
2313                                                         ||
2314                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Input_Buffer") == 0
2315                                                                 && (strcmp(callee->name->text, "getbyte") == 0
2316                                                                  || strcmp(callee->name->text, "readbytes") == 0
2317                                                                  ))
2318                                                         )
2319                                                         goto force_inline;
2320
2321                                                 if (callee->jcodelength > 0)
2322                                                         goto dont_inline;
2323 #endif
2324
2325                                                 if (callee->flags & ACC_NATIVE)
2326                                                         goto dont_inline;
2327
2328                                                 if (iln->depth >= 3)
2329                                                         goto dont_inline;
2330
2331                                                 if ((callee->flags & (ACC_STATIC | ACC_FINAL | ACC_PRIVATE)
2332                                                                         || opcode == ICMD_INVOKESPECIAL)) {
2333                                                         speculative = false;
2334                                                         goto maybe_inline;
2335                                                 }
2336
2337                                                 /* XXX search single implementation for abstract monomorphics */
2338                                                 if ((callee->flags & (ACC_METHOD_MONOMORPHIC | ACC_METHOD_IMPLEMENTED
2339                                                                                                                                          | ACC_ABSTRACT))
2340                                                                           == (ACC_METHOD_MONOMORPHIC | ACC_METHOD_IMPLEMENTED)) {
2341 #if 0
2342                                                         static int debug_spec_inline = 94;
2343                                                         if (debug_spec_inline-- > 0) {
2344 #endif
2345                                                         if (1) {
2346                                                                 DOLOG( printf("SPECULATIVE INLINE: "); method_println(callee); );
2347                                                                 speculative = true;
2348
2349 #if 0
2350                                                                 if (debug_spec_inline == 0) {
2351                                                                         printf("LAST INLINE: "); method_println(callee);
2352                                                                         printf("         in: "); method_println(m);
2353                                                                         printf("     master: "); method_println(iln->ctx->master->m);
2354                                                                 }
2355 #endif
2356                                                                 goto maybe_inline;
2357                                                         }
2358                                                 }
2359
2360                                                 /* polymorphic call site */
2361                                                 goto dont_inline;
2362
2363 maybe_inline:
2364                                                 for (active = iln; active; active = active->parent) {
2365                                                         if (callee == active->m) {
2366                                                                 DOLOG( printf("RECURSIVE!\n") );
2367                                                                 goto dont_inline;
2368                                                         }
2369                                                 }
2370 #if 0
2371 force_inline:
2372 #endif
2373                                                 if (!inline_analyse_callee(iln, callee,
2374                                                                         bptr,
2375                                                                         iptr,
2376                                                                         iln->instructioncount - len - 1 /* XXX ugly */,
2377                                                                         handlers,
2378                                                                         nhandlers))
2379                                                         return false;
2380
2381                                                 if (speculative)
2382                                                         method_add_assumption_monomorphic(callee, iln->ctx->master->m);
2383                                         }
2384 dont_inline:
2385                                         break;
2386
2387                                 case ICMD_RETURN:
2388                                 case ICMD_IRETURN:
2389                                 case ICMD_ARETURN:
2390                                 case ICMD_LRETURN:
2391                                 case ICMD_FRETURN:
2392                                 case ICMD_DRETURN:
2393                                         /* extra ICMD_MOVE may be necessary */
2394                                         iln->cumul_instructioncount++;
2395                                         break;
2396                         }
2397                 }
2398
2399                 /* end of basic block */
2400         }
2401
2402         return true;
2403 }
2404
2405
2406 /* post processing ************************************************************/
2407
2408 #define POSTPROCESS_SRC(varindex)  live[varindex]--
2409 #define POSTPROCESS_DST(varindex)  live[varindex]++
2410
2411 #define POSTPROCESS_SRCOP(s)  POSTPROCESS_SRC(iptr->s.varindex)
2412 #define POSTPROCESS_DSTOP(d)  POSTPROCESS_DST(iptr->d.varindex)
2413
2414 #define MARKSAVED(varindex)  jd->var[varindex].flags |= SAVEDVAR
2415
2416 #define MARK_ALL_SAVED                                               \
2417     do {                                                             \
2418         for (i=0; i<jd->vartop; ++i)                                 \
2419             if (live[i])                                             \
2420                 MARKSAVED(i);                                        \
2421     } while (0)
2422
2423 static void inline_post_process(jitdata *jd)
2424 {
2425         basicblock *bptr;
2426         instruction *iptr;
2427         instruction *iend;
2428         s4 i;
2429         icmdtable_entry_t *icmdt;
2430         s4 *live;
2431         methoddesc *md;
2432         builtintable_entry *bte;
2433
2434         /* reset the SAVEDVAR flag of all variables */
2435
2436         for (i=0; i<jd->vartop; ++i)
2437                 jd->var[i].flags &= ~SAVEDVAR;
2438
2439         /* allocate the life counters */
2440
2441         live = DMNEW(s4, jd->vartop);
2442         MZERO(live, s4, jd->vartop);
2443
2444         /* iterate over all basic blocks */
2445
2446         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
2447                 if (bptr->flags < BBREACHED)
2448                         continue;
2449
2450                 /* make invars live */
2451
2452                 for (i=0; i<bptr->indepth; ++i)
2453                         POSTPROCESS_DST(bptr->invars[i]);
2454
2455                 iptr = bptr->iinstr;
2456                 iend = iptr + bptr->icount;
2457
2458                 for (; iptr < iend; ++iptr) {
2459
2460                         icmdt = &(icmd_table[iptr->opc]);
2461
2462                         switch (icmdt->dataflow) {
2463                                 case DF_3_TO_0:
2464                                         POSTPROCESS_SRCOP(sx.s23.s3);
2465                                 case DF_2_TO_0:
2466                                         POSTPROCESS_SRCOP(sx.s23.s2);
2467                                 case DF_1_TO_0:
2468                                         POSTPROCESS_SRCOP(s1);
2469                                 case DF_0_TO_0:
2470                                         if (icmdt->flags & ICMDTABLE_CALLS) {
2471                                                 jd->isleafmethod = false;
2472                                                 MARK_ALL_SAVED;
2473                                         }
2474                                         break;
2475
2476                                 case DF_2_TO_1:
2477                                         POSTPROCESS_SRCOP(sx.s23.s2);
2478                                 case DF_1_TO_1:
2479                                 case DF_MOVE:
2480                                         POSTPROCESS_SRCOP(s1);
2481                                 case DF_0_TO_1:
2482                                         if (icmdt->flags & ICMDTABLE_CALLS) {
2483                                                 jd->isleafmethod = false;
2484                                                 MARK_ALL_SAVED;
2485                                         }
2486                                 case DF_COPY:
2487                                         POSTPROCESS_DSTOP(dst);
2488                                         break;
2489
2490                                 case DF_N_TO_1:
2491                                         for (i=0; i<iptr->s1.argcount; ++i) {
2492                                                 POSTPROCESS_SRC(iptr->sx.s23.s2.args[i]);
2493                                         }
2494                                         if (icmdt->flags & ICMDTABLE_CALLS) {
2495                                                 jd->isleafmethod = false;
2496                                                 MARK_ALL_SAVED;
2497                                         }
2498                                         POSTPROCESS_DSTOP(dst);
2499                                         break;
2500
2501                                 case DF_INVOKE:
2502                                         INSTRUCTION_GET_METHODDESC(iptr, md);
2503                 post_process_call:
2504                                         jd->isleafmethod = false;
2505                                         for (i=0; i<md->paramcount; ++i) {
2506                                                 POSTPROCESS_SRC(iptr->sx.s23.s2.args[i]);
2507                                         }
2508                                         for (; i<iptr->s1.argcount; ++i) {
2509                                                 MARKSAVED(iptr->sx.s23.s2.args[i]);
2510                                         }
2511                                         if (md->returntype.type != TYPE_VOID)
2512                                                 POSTPROCESS_DSTOP(dst);
2513                                         break;
2514
2515                                 case DF_BUILTIN:
2516                                         bte = iptr->sx.s23.s3.bte;
2517                                         md = bte->md;
2518                                         goto post_process_call;
2519
2520                                 default:
2521                                         assert(0);
2522                         }
2523
2524                 } /* end instruction loop */
2525
2526                 /* consume outvars */
2527
2528                 for (i=0; i<bptr->outdepth; ++i)
2529                         POSTPROCESS_SRC(bptr->outvars[i]);
2530
2531 #if !defined(NDEBUG)
2532                 for (i=jd->localcount; i < jd->vartop; ++i)
2533                         assert(live[i] == 0);
2534 #endif
2535
2536         } /* end basic block loop */
2537 }
2538
2539
2540 /* main driver function *******************************************************/
2541
2542 bool inline_inline(jitdata *jd)
2543 {
2544         inline_node *iln;
2545         methodinfo *m;
2546
2547         m = jd->m;
2548
2549         DOLOG( printf("==== INLINE ==================================================================\n");
2550                    show_method(jd, SHOW_STACK); );
2551
2552         iln = DNEW(inline_node);
2553         MZERO(iln, inline_node, 1);
2554
2555         iln->ctx = DNEW(inline_context);
2556         MZERO(iln->ctx, inline_context, 1);
2557         iln->ctx->master = iln;
2558         iln->ctx->calls_others = false;
2559         iln->m = m;
2560         iln->jd = jd;
2561         iln->regdata = jd->rd;
2562         iln->ctx->next_debugnr = 1; /* XXX debug */
2563
2564         iln->blockbefore = true;
2565         iln->blockafter = true;
2566
2567         /* we cannot use m->instructioncount because it may be greater than
2568          * the actual number of instructions in the basic blocks. */
2569         iln->instructioncount = 0;
2570         iln->cumul_instructioncount = 0;
2571         iln->cumul_basicblockcount = 1 /* dummy end block */;
2572
2573         if (inline_inline_intern(m, iln)) {
2574
2575                 DOLOG( printf("==== TEST INLINE =============================================================\n"); );
2576
2577                 if (iln->children)
2578                         test_inlining(iln, jd);
2579         }
2580
2581         DOLOG( printf("-------- DONE -----------------------------------------------------------\n");
2582         fflush(stdout); );
2583
2584         return true;
2585 }
2586
2587 /*
2588  * These are local overrides for various environment variables in Emacs.
2589  * Please do not remove this and leave it at the end of the file, where
2590  * Emacs will automagically detect them.
2591  * ---------------------------------------------------------------------
2592  * Local variables:
2593  * mode: c
2594  * indent-tabs-mode: t
2595  * c-basic-offset: 4
2596  * tab-width: 4
2597  * End:
2598  * vim:noexpandtab:sw=4:ts=4:
2599  */