src/vm/jit/inline/inline.c

   1 /* src/vm/jit/inline/inline.c - method inlining
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Edwin Steiner
  28
  29    Changes:
  30
  31    $Id: inline.c 6003 2006-11-15 23:24:46Z edwin $
  32
  33 */
  34
  35 #include "config.h"
  36
  37 #include "vm/types.h"
  38
  39 #include <assert.h>
  40 #include <stdio.h>
  41 #include <string.h>
  42 #include <limits.h>
  43
  44 #include "mm/memory.h"
  45 #include "toolbox/logging.h"
  46 #include "vm/builtin.h"
  47 #include "vm/global.h"
  48 #include "vm/options.h"
  49 #include "vm/statistics.h"
  50 #include "vm/jit/jit.h"
  51 #include "vm/jit/parse.h"
  52 #include "vm/jit/inline/inline.h"
  53 #include "vm/jit/loop/loop.h"
  54
  55 #include "vm/class.h"
  56 #include "vm/initialize.h"
  57 #include "vm/method.h"
  58 #include "vm/jit/jit.h"
  59 #include "vm/jit/show.h"
  60
  61 #include "vm/jit/reg.h"
  62 #include "vm/jit/stack.h"
  63
  64 #include "vm/jit/verify/typecheck.h"
  65
  66 #if defined(ENABLE_THREADS)
  67 # include "threads/native/threads.h"
  68 #endif
  69
  70
  71 /* debugging ******************************************************************/
  72
  73 #if !defined(NDEBUG)
  74 #define INLINE_VERBOSE
  75 bool inline_debug_log = 0;
  76 bool inline_debug_log_names = 0;
  77 int inline_debug_start_counter = 0;
  78 int inline_debug_max_size = INT_MAX;
  79 int inline_debug_min_size = 0;
  80 int inline_debug_end_counter = INT_MAX;
  81 int inline_count_methods = 0;
  82 #define DOLOG(code) do{ if (inline_debug_log) { code; } }while(0)
  83 #else
  84 #define DOLOG(code)
  85 #endif
  86
  87
  88 /* types **********************************************************************/
  89
  90 typedef struct inline_node inline_node;
  91 typedef struct inline_target_ref inline_target_ref;
  92 typedef struct inline_context inline_context;
  93 typedef struct inline_block_map inline_block_map;
  94
  95 struct inline_node {
  96         inline_context *ctx;
  97
  98         jitdata *jd;
  99         methodinfo *m;
 100         inline_node *children;
 101         inline_node *next;                             /* next node at this depth */
 102         inline_node *prev;                             /* prev node at this depth */
 103         int depth;                                  /* inlining depth, 0 for root */
 104
 105         /* info about the call site (if depth > 0)*/
 106         inline_node *parent;                /* node of the caller (NULL for root) */
 107         basicblock *callerblock;         /* original block containing the INVOKE* */
 108         instruction *callerins;               /* the original INVOKE* instruction */
 109         s4 callerpc;
 110         s4 *n_passthroughvars;
 111         int n_passthroughcount;
 112         int n_selfpassthroughcount;  /* # of pass-through vars of the call itself */
 113         exception_entry **o_handlers;
 114         int n_handlercount;                 /* # of handlers protecting this call */
 115         int n_resultlocal;
 116         int synclocal;                    /* variable used for synchr., or UNUSED */
 117
 118         bool blockbefore;                  /* block boundary before inlined body? */
 119         bool blockafter;                   /* block boundary after inlined body?  */
 120
 121         /* info about the callee */
 122         int localsoffset;
 123         int prolog_instructioncount;         /* # of ICMDs in the inlining prolog */
 124         int epilog_instructioncount;         /* # of ICMDs in the inlining epilog */
 125         int extra_instructioncount;
 126         int instructioncount;
 127         bool synchronize;                /* do we have to synchronize enter/exit? */
 128         basicblock *handler_monitorexit;     /* handler for synchronized inlinees */
 129         s4 *varmap;
 130
 131         /* cumulative values */
 132         int cumul_instructioncount;  /* ICMDs in this node and its children       */
 133         int cumul_basicblockcount;   /* BBs started by this node and its children */
 134         int cumul_blockmapcount;
 135         int cumul_maxlocals;
 136         int cumul_exceptiontablelength;
 137
 138         /* output */
 139         instruction *inlined_iinstr;
 140         instruction *inlined_iinstr_cursor;
 141         basicblock *inlined_basicblocks;
 142         basicblock *inlined_basicblocks_cursor;
 143
 144         /* register data */
 145         registerdata *regdata;
 146
 147         /* temporary */
 148         inline_target_ref *refs;
 149         instruction *inline_start_instruction;
 150         s4 *javalocals;
 151
 152         /* XXX debug */
 153         char *indent;
 154         int debugnr;
 155 };
 156
 157 struct inline_target_ref {
 158         inline_target_ref *next;
 159         basicblock **ref;
 160         basicblock *target;
 161 };
 162
 163 struct inline_block_map {
 164         inline_node *iln;
 165         basicblock *o_block;
 166         basicblock *n_block;
 167 };
 168
 169 struct inline_context {
 170         inline_node *master;
 171
 172         jitdata *resultjd;
 173
 174         int next_block_number;
 175         inline_block_map *blockmap;
 176         int blockmap_index;
 177
 178         int maxinoutdepth;
 179
 180         bool calls_others;
 181
 182         int next_debugnr; /* XXX debug */
 183 };
 184
 185
 186 /* prototypes *****************************************************************/
 187
 188 static bool inline_inline_intern(methodinfo *m, inline_node *iln);
 189 static void inline_post_process(jitdata *jd);
 190
 191
 192 /* debug helpers **************************************************************/
 193
 194 #if !defined(NDEBUG)
 195 #include "inline_debug.inc"
 196
 197 void inline_print_stats()
 198 {
 199         printf("inlined callers: %d\n", inline_count_methods);
 200 }
 201 #endif
 202
 203
 204 /* compilation of callees *****************************************************/
 205
 206 static bool inline_jit_compile_intern(jitdata *jd)
 207 {
 208         methodinfo *m;
 209
 210         /* XXX should share code with jit.c */
 211
 212         assert(jd);
 213
 214         /* XXX initialize the static function's class */
 215
 216         m = jd->m;
 217
 218         /* call the compiler passes ***********************************************/
 219
 220         /* call parse pass */
 221
 222         DOLOG( log_message_class("Parsing ", m->class) );
 223         if (!parse(jd)) {
 224                 return false;
 225         }
 226
 227         /* call stack analysis pass */
 228
 229         if (!stack_analyse(jd)) {
 230                 return false;
 231         }
 232
 233         return true;
 234 }
 235
 236
 237 static bool inline_jit_compile(inline_node *iln)
 238 {
 239         bool                r;
 240         methodinfo         *m;
 241         jitdata            *jd;
 242
 243         /* XXX should share code with jit.c */
 244
 245         assert(iln);
 246         m = iln->m;
 247         assert(m);
 248
 249 #if defined(ENABLE_THREADS)
 250         /* enter a monitor on the method */
 251         lock_monitor_enter((java_objectheader *) m);
 252 #endif
 253
 254         /* allocate jitdata structure and fill it */
 255
 256         jd = jit_jitdata_new(m);
 257         iln->jd = jd;
 258
 259         jd->flags = 0; /* XXX */
 260
 261         /* initialize the register allocator */
 262
 263         reg_setup(jd);
 264
 265         /* setup the codegendata memory */
 266
 267         /* XXX do a pseudo setup */
 268         jd->cd = DNEW(codegendata);
 269         MZERO(jd->cd, codegendata, 1);
 270         jd->cd->maxstack = m->maxstack;
 271         jd->cd->method = m;
 272         /* XXX uses too much dump memory codegen_setup(jd); */
 273
 274         /* now call internal compile function */
 275
 276         r = inline_jit_compile_intern(jd);
 277
 278         if (r) {
 279                 iln->regdata = jd->rd;
 280         }
 281
 282         /* free some memory */
 283 #if 0
 284
 285 #if defined(ENABLE_JIT)
 286 # if defined(ENABLE_INTRP)
 287         if (!opt_intrp)
 288 # endif
 289                 codegen_free(jd);
 290 #endif
 291
 292 #endif
 293
 294 #if defined(ENABLE_THREADS)
 295         /* leave the monitor */
 296         lock_monitor_exit((java_objectheader *) m );
 297 #endif
 298
 299         return r;
 300 }
 301
 302
 303 /* inlining tree handling *****************************************************/
 304
 305 static void insert_inline_node(inline_node *parent, inline_node *child)
 306 {
 307         inline_node *first;
 308         inline_node *succ;
 309
 310         assert(parent && child);
 311
 312         child->parent = parent;
 313
 314         child->debugnr = parent->ctx->next_debugnr++; /* XXX debug */
 315
 316         first = parent->children;
 317         if (!first) {
 318                 /* insert as only node */
 319                 parent->children = child;
 320                 child->next = child;
 321                 child->prev = child;
 322                 return;
 323         }
 324
 325         /* {there is at least one child already there} */
 326
 327         /* XXX is this search necessary, or could we always add at the end? */
 328
 329         succ = first;
 330         while (succ->callerpc < child->callerpc) {
 331                 succ = succ->next;
 332                 if (succ == first) {
 333                         /* insert as last node */
 334                         child->prev = first->prev;
 335                         child->next = first;
 336                         child->prev->next = child;
 337                         child->next->prev = child;
 338                         return;
 339                 }
 340         }
 341
 342         assert(succ->callerpc > child->callerpc);
 343
 344         /* insert before succ */
 345
 346         child->prev = succ->prev;
 347         child->next = succ;
 348         child->prev->next = child;
 349         child->next->prev = child;
 350 }
 351
 352
 353 /* variable handling **********************************************************/
 354
 355 static s4 inline_new_variable(jitdata *jd, s4 type, s4 flags)
 356 {
 357         s4 index;
 358         s4 newcount;
 359
 360         index = jd->vartop++;
 361         if (index >= jd->varcount) {
 362                 newcount = jd->vartop * 2; /* XXX */
 363                 jd->var = DMREALLOC(jd->var, varinfo, jd->varcount, newcount);
 364                 MZERO(jd->var + jd->varcount, varinfo, (newcount - jd->varcount));
 365                 jd->varcount = newcount;
 366         }
 367
 368         jd->var[index].type = type;
 369         jd->var[index].flags = flags;
 370
 371         return index;
 372 }
 373
 374
 375 static s4 inline_new_variable_clone(jitdata *jd, jitdata *origjd, s4 origidx)
 376 {
 377         varinfo *v;
 378         s4       newidx;
 379
 380         v = &(origjd->var[origidx]);
 381
 382         newidx = inline_new_variable(jd, v->type, v->flags);
 383
 384         jd->var[newidx].vv = v->vv;
 385
 386         return newidx;
 387 }
 388
 389
 390 static s4 inline_new_temp_variable(jitdata *jd, s4 type)
 391 {
 392         return inline_new_variable(jd, type, 0);
 393 }
 394
 395
 396 static s4 inline_translate_variable(jitdata *jd, jitdata *origjd, s4 *varmap, s4 index)
 397 {
 398         s4 idx;
 399
 400         idx = varmap[index];
 401
 402         if (idx < 0) {
 403                 idx = inline_new_variable_clone(jd, origjd, index);
 404                 varmap[index] = idx;
 405         }
 406
 407         return idx;
 408 }
 409
 410
 411 static s4 *create_variable_map(inline_node *callee)
 412 {
 413         s4 *varmap;
 414         s4 i, t;
 415         s4 idx;
 416         s4 n_idx;
 417         s4 avail;
 418         varinfo *v;
 419
 420         /* create the variable mapping */
 421
 422         varmap = DMNEW(s4, callee->jd->varcount);
 423         for (i=0; i<callee->jd->varcount; ++i)
 424                 varmap[i] = -1;
 425
 426         /* translate local variables */
 427
 428         for (i=0; i<callee->m->maxlocals; ++i) {
 429                 for (t=0; t<5; ++t) {
 430                         idx = callee->jd->local_map[5*i + t];
 431                         if (idx == UNUSED)
 432                                 continue;
 433
 434                         v = &(callee->jd->var[idx]);
 435                         assert(v->type == t || v->type == TYPE_VOID); /* XXX stack leaves VOID */
 436                         v->type = t; /* XXX restore if it is TYPE_VOID */
 437
 438                         avail = callee->ctx->resultjd->local_map[5*(callee->localsoffset + i) + t];
 439
 440                         if (avail == UNUSED) {
 441                                 avail = inline_new_variable_clone(callee->ctx->resultjd, callee->jd, idx);
 442                                 callee->ctx->resultjd->local_map[5*(callee->localsoffset + i) + t] = avail;
 443                         }
 444
 445                         varmap[idx] = avail;
 446                 }
 447         }
 448
 449         /* for synchronized instance methods we need an extra local */
 450
 451         if (callee->synchronize && !(callee->m->flags & ACC_STATIC)) {
 452                 n_idx = callee->localsoffset - 1;
 453                 assert(n_idx >= 0);
 454                 assert(callee->parent);
 455                 assert(n_idx == callee->parent->localsoffset + callee->parent->m->maxlocals);
 456
 457                 avail = callee->ctx->resultjd->local_map[5*n_idx + TYPE_ADR];
 458
 459                 if (avail == UNUSED) {
 460                         avail = inline_new_variable(callee->ctx->resultjd, TYPE_ADR, 0);
 461                         callee->ctx->resultjd->local_map[5*n_idx + TYPE_ADR] = avail;
 462                 }
 463
 464                 callee->synclocal = avail;
 465         }
 466         else {
 467                 callee->synclocal = UNUSED;
 468         }
 469
 470         return varmap;
 471 }
 472
 473
 474 /* basic block translation ****************************************************/
 475
 476 #define INLINE_RETURN_REFERENCE(callee)  \
 477         ( (basicblock *) (ptrint) (0x333 + (callee)->depth) )
 478
 479
 480 static void inline_add_block_reference(inline_node *iln, basicblock **blockp)
 481 {
 482         inline_target_ref *ref;
 483
 484         ref = DNEW(inline_target_ref);
 485         ref->ref = blockp;
 486         ref->next = iln->refs;
 487         iln->refs = ref;
 488 }
 489
 490
 491 static void inline_block_translation(inline_node *iln, basicblock *o_bptr, basicblock *n_bptr)
 492 {
 493         inline_context *ctx;
 494
 495         ctx = iln->ctx;
 496         assert(ctx->blockmap_index < ctx->master->cumul_blockmapcount);
 497
 498         ctx->blockmap[ctx->blockmap_index].iln = iln;
 499         ctx->blockmap[ctx->blockmap_index].o_block = o_bptr;
 500         ctx->blockmap[ctx->blockmap_index].n_block = n_bptr;
 501
 502         ctx->blockmap_index++;
 503 }
 504
 505
 506 static basicblock * inline_map_block(inline_node *iln,
 507                                                                          basicblock *o_block,
 508                                                                          inline_node *targetiln)
 509 {
 510         inline_block_map *bm;
 511         inline_block_map *bmend;
 512
 513         assert(iln);
 514         assert(targetiln);
 515
 516         if (!o_block)
 517                 return NULL;
 518
 519         bm = iln->ctx->blockmap;
 520         bmend = bm + iln->ctx->blockmap_index;
 521
 522         while (bm < bmend) {
 523                 assert(bm->iln && bm->o_block && bm->n_block);
 524                 if (bm->o_block == o_block && bm->iln == targetiln)
 525                         return bm->n_block;
 526                 bm++;
 527         }
 528
 529         assert(false);
 530         return NULL; /* not reached */
 531 }
 532
 533
 534 static void inline_resolve_block_refs(inline_target_ref **refs,
 535                                                                           basicblock *o_bptr,
 536                                                                           basicblock *n_bptr)
 537 {
 538         inline_target_ref *ref;
 539         inline_target_ref *prev;
 540
 541         ref = *refs;
 542         prev = NULL;
 543         while (ref) {
 544                 if (*(ref->ref) == o_bptr) {
 545
 546 #if defined(INLINE_VERBOSE)
 547                         if (inline_debug_log) {
 548                                 if ((ptrint) o_bptr < (0x333+100)) { /* XXX */
 549                                         printf("resolving RETURN block reference %p -> new L%03d (%p)\n",
 550                                                         (void*)o_bptr, n_bptr->nr, (void*)n_bptr);
 551                                 }
 552                                 else {
 553                                         printf("resolving block reference old L%03d (%p) -> new L%03d (%p)\n",
 554                                                         o_bptr->nr, (void*)o_bptr, n_bptr->nr, (void*)n_bptr);
 555                                 }
 556                         }
 557 #endif
 558
 559                         *(ref->ref) = n_bptr;
 560                         if (prev) {
 561                                 prev->next = ref->next;
 562                         }
 563                         else {
 564                                 *refs = ref->next;
 565                         }
 566                 }
 567                 else {
 568                         prev = ref;
 569                 }
 570                 ref = ref->next;
 571         }
 572 }
 573
 574
 575 /* basic block creation *******************************************************/
 576
 577 static basicblock * create_block(inline_node *container,
 578                                                                  inline_node *iln,
 579                                                                  inline_node *inner,
 580                                                                  basicblock *o_bptr,
 581                                                                  inline_target_ref **refs,
 582                                                                  int indepth)
 583 {
 584         basicblock  *n_bptr;
 585         inline_node *outer;
 586         s4           i;
 587         s4           depth;
 588         s4           varidx;
 589         s4           newvaridx;
 590
 591         assert(container);
 592         assert(iln);
 593         assert(inner);
 594         assert(indepth >= 0);
 595
 596         n_bptr = container->inlined_basicblocks_cursor++;
 597         assert(n_bptr);
 598         assert((n_bptr - container->inlined_basicblocks) < container->cumul_basicblockcount);
 599
 600         BASICBLOCK_INIT(n_bptr, iln->m);
 601
 602         n_bptr->iinstr = container->inlined_iinstr_cursor;
 603         n_bptr->next = n_bptr + 1;
 604         n_bptr->nr = container->ctx->next_block_number++;
 605         n_bptr->indepth = indepth;
 606         n_bptr->flags = BBFINISHED; /* XXX */
 607
 608         /* set the inlineinfo of the new block */
 609
 610         if (iln->inline_start_instruction)
 611                 n_bptr->inlineinfo = iln->inline_start_instruction->sx.s23.s3.inlineinfo;
 612
 613         if (indepth > container->ctx->maxinoutdepth)
 614                 container->ctx->maxinoutdepth = indepth;
 615
 616         if (indepth) {
 617                 n_bptr->invars = DMNEW(s4, indepth);
 618
 619
 620                 for (i=0; i<indepth; ++i)
 621                         n_bptr->invars[i] = -1; /* XXX debug */
 622
 623                 /* pass-through variables enter the block */
 624
 625                 outer = inner->parent;
 626                 while (outer != NULL) {
 627                         depth = outer->n_passthroughcount;
 628
 629                         assert(depth + inner->n_selfpassthroughcount <= indepth);
 630
 631                         for (i=0; i<inner->n_selfpassthroughcount; ++i) {
 632                                 varidx = inner->n_passthroughvars[i];
 633                                 newvaridx =
 634                                         inline_new_variable_clone(container->ctx->resultjd,
 635                                                                                           outer->jd,
 636                                                                                           varidx);
 637                                 n_bptr->invars[depth + i] = newvaridx;
 638                                 outer->varmap[varidx] = newvaridx;
 639                         }
 640                         inner = outer;
 641                         outer = outer->parent;
 642                 }
 643         }
 644         else {
 645                 n_bptr->invars = NULL;
 646         }
 647
 648         /* XXX move this to callers with o_bptr != NULL? */
 649         if (o_bptr) {
 650                 assert(refs);
 651                 inline_resolve_block_refs(refs, o_bptr, n_bptr);
 652         }
 653
 654         /* XXX for the verifier. should not be here */
 655
 656         {
 657                 varinfo *dv;
 658
 659                 dv = DMNEW(varinfo, iln->ctx->resultjd->localcount + VERIFIER_EXTRA_LOCALS);
 660                 MZERO(dv, varinfo,  iln->ctx->resultjd->localcount + VERIFIER_EXTRA_LOCALS);
 661                 n_bptr->inlocals = dv;
 662         }
 663
 664         return n_bptr;
 665 }
 666
 667
 668 static s4 *translate_javalocals(inline_node *iln, s4 *javalocals)
 669 {
 670         s4 *jl;
 671         s4 i, j;
 672
 673         jl = DMNEW(s4, iln->jd->maxlocals);
 674
 675         for (i=0; i<iln->jd->maxlocals; ++i) {
 676                 j = javalocals[i];
 677                 if (j != UNUSED)
 678                         j = inline_translate_variable(iln->ctx->resultjd, iln->jd, iln->varmap, j);
 679                 jl[i] = j;
 680         }
 681
 682         return jl;
 683 }
 684
 685
 686 static basicblock * create_body_block(inline_node *iln,
 687                                                                           basicblock *o_bptr, s4 *varmap)
 688 {
 689         basicblock *n_bptr;
 690         s4 i;
 691
 692         n_bptr = create_block(iln, iln, iln, o_bptr, &(iln->refs),
 693                                                   o_bptr->indepth + iln->n_passthroughcount);
 694
 695         n_bptr->type = o_bptr->type;
 696         n_bptr->flags = o_bptr->flags;
 697
 698         /* translate the invars of the original block */
 699
 700         for (i=0; i<o_bptr->indepth; ++i) {
 701                 n_bptr->invars[iln->n_passthroughcount + i] =
 702                         inline_translate_variable(iln->ctx->resultjd, iln->jd,
 703                                 varmap,
 704                                 o_bptr->invars[i]);
 705         }
 706
 707         /* translate javalocals info (not for dead code) */
 708
 709         if (n_bptr->flags >= BBREACHED)
 710                 n_bptr->javalocals = translate_javalocals(iln, o_bptr->javalocals);
 711
 712         return n_bptr;
 713 }
 714
 715
 716 static basicblock * create_epilog_block(inline_node *caller, inline_node *callee, s4 *varmap)
 717 {
 718         basicblock *n_bptr;
 719         s4 retcount;
 720         s4 idx;
 721
 722         /* number of return variables */
 723
 724         retcount = (callee->n_resultlocal == -1
 725                                 && callee->m->parseddesc->returntype.type != TYPE_VOID) ? 1 : 0;
 726
 727         /* start the epilog block */
 728
 729         n_bptr = create_block(caller, caller, callee, INLINE_RETURN_REFERENCE(callee),
 730                         &(callee->refs), callee->n_passthroughcount + retcount);
 731
 732         /* return variable */
 733
 734         if (retcount) {
 735                 idx = inline_new_variable(caller->ctx->resultjd,
 736                            callee->m->parseddesc->returntype.type, 0 /* XXX */);
 737                 n_bptr->invars[callee->n_passthroughcount] = idx;
 738                 varmap[callee->callerins->dst.varindex] = idx;
 739         }
 740
 741         /* set javalocals */
 742
 743         n_bptr->javalocals = DMNEW(s4, caller->jd->maxlocals);
 744         MCOPY(n_bptr->javalocals, caller->javalocals, s4, caller->jd->maxlocals);
 745
 746         /* set block flags & type */
 747
 748         n_bptr->flags = /* XXX original block flags */ BBFINISHED;
 749         n_bptr->type = BBTYPE_STD;
 750
 751         return n_bptr;
 752 }
 753
 754
 755 static void close_block(inline_node *iln, inline_node *inner, basicblock *n_bptr, s4 outdepth)
 756 {
 757         inline_node *outer;
 758         s4           i;
 759         s4           depth;
 760         s4           varidx;
 761
 762         n_bptr->outdepth = outdepth;
 763         n_bptr->outvars = DMNEW(s4, outdepth);
 764
 765         for (i=0; i<outdepth; ++i)
 766                 n_bptr->outvars[i] = 0; /* XXX debug */
 767
 768         if (outdepth > iln->ctx->maxinoutdepth)
 769                 iln->ctx->maxinoutdepth = outdepth;
 770
 771         /* pass-through variables leave the block */
 772
 773         outer = inner->parent;
 774         while (outer != NULL) {
 775                 depth = outer->n_passthroughcount;
 776
 777                 assert(depth + inner->n_selfpassthroughcount <= outdepth);
 778
 779                 for (i=0; i<inner->n_selfpassthroughcount; ++i) {
 780                         varidx = inner->n_passthroughvars[i];
 781                         n_bptr->outvars[depth + i] =
 782                                 inline_translate_variable(iln->ctx->resultjd,
 783                                                                                   outer->jd,
 784                                                                                   outer->varmap,
 785                                                                                   varidx);
 786                 }
 787                 inner = outer;
 788                 outer = outer->parent;
 789         }
 790 }
 791
 792
 793 static void close_prolog_block(inline_node *iln,
 794                                                            basicblock *n_bptr,
 795                                                            inline_node *nextcall)
 796 {
 797         /* XXX add original outvars! */
 798         close_block(iln, nextcall, n_bptr, nextcall->n_passthroughcount);
 799
 800         /* pass-through variables */
 801
 802         DOLOG( printf("closed prolog block:\n");
 803                    show_basicblock(iln->ctx->resultjd, n_bptr, SHOW_STACK); );
 804 }
 805
 806
 807 static void close_body_block(inline_node *iln,
 808                                                          basicblock *n_bptr,
 809                                                          basicblock *o_bptr,
 810                                                          s4 *varmap,
 811                                                          s4 retcount,
 812                                                          s4 retidx)
 813 {
 814         s4 i;
 815
 816         close_block(iln, iln, n_bptr, iln->n_passthroughcount + o_bptr->outdepth + retcount);
 817
 818         /* translate the outvars of the original block */
 819
 820         /* XXX reuse code */
 821         for (i=0; i<o_bptr->outdepth; ++i) {
 822                 n_bptr->outvars[iln->n_passthroughcount + i] =
 823                         inline_translate_variable(iln->ctx->resultjd, iln->jd, varmap,
 824                                         o_bptr->outvars[i]);
 825         }
 826
 827         /* set the return variable, if any */
 828
 829         if (retcount) {
 830                 assert(retidx >= 0);
 831                 n_bptr->outvars[iln->n_passthroughcount + o_bptr->outdepth] = retidx;
 832         }
 833 }
 834
 835
 836 /* inlined code generation ****************************************************/
 837
 838 static instruction * inline_instruction(inline_node *iln,
 839                                                                                 s4 opcode,
 840                                                                                 instruction *o_iptr)
 841 {
 842         instruction *n_iptr;
 843
 844         n_iptr = (iln->inlined_iinstr_cursor++);
 845         assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
 846
 847         n_iptr->opc = opcode;
 848         n_iptr->flags.bits = o_iptr->flags.bits & INS_FLAG_ID_MASK;
 849         n_iptr->line = o_iptr->line;
 850
 851         return n_iptr;
 852 }
 853
 854 static void inline_generate_sync_builtin(inline_node *iln,
 855                                                                                  inline_node *callee,
 856                                                                                  instruction *o_iptr,
 857                                                                                  s4 instancevar,
 858                                                                                  functionptr func)
 859 {
 860         int          syncvar;
 861         instruction *n_ins;
 862
 863         if (callee->m->flags & ACC_STATIC) {
 864                 /* ACONST */
 865                 syncvar = inline_new_temp_variable(iln->ctx->resultjd, TYPE_ADR);
 866
 867                 n_ins = inline_instruction(iln, ICMD_ACONST, o_iptr);
 868                 n_ins->sx.val.c.cls = callee->m->class;
 869                 n_ins->dst.varindex = syncvar;
 870                 n_ins->flags.bits |= INS_FLAG_CLASS;
 871         }
 872         else {
 873                 syncvar = instancevar;
 874         }
 875
 876         assert(syncvar != UNUSED);
 877
 878         /* MONITORENTER / MONITOREXIT */
 879
 880         n_ins = inline_instruction(iln, ICMD_BUILTIN, o_iptr);
 881         n_ins->sx.s23.s3.bte = builtintable_get_internal(func);
 882         n_ins->s1.argcount = 1; /* XXX add through-vars */
 883         n_ins->sx.s23.s2.args = DMNEW(s4, 1);
 884         n_ins->sx.s23.s2.args[0] = syncvar;
 885 }
 886
 887 static s4 emit_inlining_prolog(inline_node *iln,
 888                                                            inline_node *callee,
 889                                                            instruction *o_iptr,
 890                                                            s4 *varmap)
 891 {
 892         methodinfo *calleem;
 893         methoddesc *md;
 894         int i;
 895         int localindex;
 896         int type;
 897         bool isstatic;
 898         instruction *n_ins;
 899         insinfo_inline *insinfo;
 900         s4 argvar;
 901
 902         assert(iln && callee && o_iptr);
 903
 904         calleem = callee->m;
 905         md = calleem->parseddesc;
 906         isstatic = (calleem->flags & ACC_STATIC);
 907
 908         /* INLINE_START instruction */
 909
 910         n_ins = inline_instruction(iln, ICMD_INLINE_START, o_iptr);
 911
 912         insinfo = DNEW(insinfo_inline);
 913         insinfo->method = callee->m;
 914         insinfo->outer = iln->m;
 915         insinfo->synclocal = callee->synclocal;
 916         insinfo->synchronize = callee->synchronize;
 917         insinfo->javalocals_start = NULL;
 918         insinfo->javalocals_end = NULL;
 919
 920         /* info about stack vars live at the INLINE_START */
 921
 922         insinfo->throughcount = callee->n_passthroughcount;
 923         insinfo->paramcount = md->paramcount;
 924         insinfo->stackvarscount = o_iptr->s1.argcount;
 925         insinfo->stackvars = DMNEW(s4, insinfo->stackvarscount);
 926         for (i=0; i<insinfo->stackvarscount; ++i)
 927                 insinfo->stackvars[i] = iln->varmap[o_iptr->sx.s23.s2.args[i]];
 928
 929         /* info about the surrounding inlining */
 930
 931         if (iln->inline_start_instruction)
 932                 insinfo->parent = iln->inline_start_instruction->sx.s23.s3.inlineinfo;
 933         else
 934                 insinfo->parent = NULL;
 935
 936         /* finish the INLINE_START instruction */
 937
 938         n_ins->sx.s23.s3.inlineinfo = insinfo;
 939         callee->inline_start_instruction = n_ins;
 940
 941         DOLOG( printf("%sprolog: ", iln->indent);
 942                    show_icmd(iln->ctx->resultjd, n_ins, false, SHOW_STACK); printf("\n"); );
 943
 944         /* handle parameters for the inlined callee */
 945
 946         localindex = callee->localsoffset + md->paramslots;
 947
 948         for (i=md->paramcount-1; i>=0; --i) {
 949                 assert(iln);
 950
 951                 type = md->paramtypes[i].type;
 952
 953                 localindex -= IS_2_WORD_TYPE(type) ? 2 : 1;
 954                 assert(callee->regdata);
 955
 956                 /* translate the argument variable */
 957
 958                 argvar = varmap[o_iptr->sx.s23.s2.args[i]];
 959                 assert(argvar != UNUSED);
 960
 961                 /* remove preallocation from the argument variable */
 962
 963                 iln->ctx->resultjd->var[argvar].flags &= ~(PREALLOC | INMEMORY);
 964
 965                 /* check the instance slot against NULL */
 966
 967                 if (!isstatic && i == 0) {
 968                         assert(type == TYPE_ADR);
 969                         n_ins = inline_instruction(iln, ICMD_CHECKNULL, o_iptr);
 970                         n_ins->s1.varindex = argvar;
 971                         n_ins->dst.varindex = n_ins->s1.varindex;
 972                 }
 973
 974                 /* store argument into local variable of inlined callee */
 975
 976                 if (callee->jd->local_map[5*(localindex - callee->localsoffset) + type] != UNUSED)
 977                 {
 978                         /* this value is used in the callee */
 979
 980                         if (i == 0 && callee->synclocal != UNUSED) {
 981                                 /* we also need it for synchronization, so copy it */
 982                                 assert(type == TYPE_ADR);
 983                                 n_ins = inline_instruction(iln, ICMD_COPY, o_iptr);
 984                         }
 985                         else {
 986                                 n_ins = inline_instruction(iln, ICMD_ISTORE + type, o_iptr);
 987                                 n_ins->sx.s23.s3.javaindex = UNUSED;
 988                         }
 989                         n_ins->s1.varindex = argvar;
 990                         n_ins->dst.varindex = iln->ctx->resultjd->local_map[5*localindex + type];
 991                         assert(n_ins->dst.varindex != UNUSED);
 992                 }
 993                 else if (i == 0 && callee->synclocal != UNUSED) {
 994                         /* the value is not used inside the callee, but we need it for */
 995                         /* synchronization                                             */
 996                         /* XXX In this case it actually makes no sense to create a     */
 997                         /*     separate synchronization variable.                      */
 998
 999                         n_ins = inline_instruction(iln, ICMD_NOP, o_iptr);
1000                 }
1001                 else {
1002                         /* this value is not used, pop it */
1003
1004                         n_ins = inline_instruction(iln, ICMD_POP, o_iptr);
1005                         n_ins->s1.varindex = argvar;
1006                 }
1007
1008                 DOLOG( printf("%sprolog: ", iln->indent);
1009                            show_icmd(iln->ctx->resultjd, n_ins, false, SHOW_STACK); printf("\n"); );
1010         }
1011
1012         /* COPY for synchronized instance methods */
1013
1014         if (callee->synclocal != UNUSED) {
1015                 n_ins = inline_instruction(iln, ICMD_COPY, o_iptr);
1016                 n_ins->s1.varindex = varmap[o_iptr->sx.s23.s2.args[0]];
1017                 n_ins->dst.varindex = callee->synclocal;
1018
1019                 assert(n_ins->s1.varindex != UNUSED);
1020         }
1021
1022         if (callee->synchronize) {
1023                 inline_generate_sync_builtin(iln, callee, o_iptr,
1024                                                                          (isstatic) ? UNUSED : varmap[o_iptr->sx.s23.s2.args[0]],
1025                                                                          LOCK_monitor_enter);
1026         }
1027
1028         return 0; /* XXX */
1029 }
1030
1031
1032 static void emit_inlining_epilog(inline_node *iln, inline_node *callee, instruction *o_iptr)
1033 {
1034         instruction *n_ins;
1035         s4          *jl;
1036
1037         assert(iln && callee && o_iptr);
1038         assert(callee->inline_start_instruction);
1039
1040         if (callee->synchronize) {
1041                 inline_generate_sync_builtin(iln, callee, o_iptr,
1042                                                                          callee->synclocal,
1043                                                                          LOCK_monitor_exit);
1044         }
1045
1046         /* INLINE_END instruction */
1047
1048         n_ins = inline_instruction(iln, ICMD_INLINE_END, o_iptr);
1049         n_ins->sx.s23.s3.inlineinfo = callee->inline_start_instruction->sx.s23.s3.inlineinfo;
1050
1051         /* set the javalocals */
1052
1053         jl = DMNEW(s4, iln->jd->maxlocals);
1054         MCOPY(jl, iln->javalocals, s4, iln->jd->maxlocals);
1055         n_ins->sx.s23.s3.inlineinfo->javalocals_end = jl;
1056
1057         DOLOG( printf("%sepilog: ", iln->indent);
1058                    show_icmd(iln->ctx->resultjd, n_ins, false, SHOW_STACK); printf("\n"); );
1059 }
1060
1061
1062 #define TRANSLATE_VAROP(vo)  \
1063         n_iptr->vo.varindex = inline_translate_variable(jd, origjd, varmap, n_iptr->vo.varindex)
1064
1065
1066 static void inline_clone_instruction(inline_node *iln,
1067                                                                          jitdata *jd,
1068                                                                          jitdata *origjd,
1069                                                                          s4 *varmap,
1070                                                                          instruction *o_iptr,
1071                                                                          instruction *n_iptr)
1072 {
1073         icmdtable_entry_t *icmdt;
1074         builtintable_entry *bte;
1075         methoddesc *md;
1076         s4 i, j;
1077         branch_target_t *table;
1078         lookup_target_t *lookup;
1079         inline_node *scope;
1080
1081         *n_iptr = *o_iptr;
1082
1083         icmdt = &(icmd_table[o_iptr->opc]);
1084
1085         switch (icmdt->dataflow) {
1086                 case DF_0_TO_0:
1087                         break;
1088
1089                 case DF_3_TO_0:
1090                         TRANSLATE_VAROP(sx.s23.s3);
1091                 case DF_2_TO_0:
1092                         TRANSLATE_VAROP(sx.s23.s2);
1093                 case DF_1_TO_0:
1094                         TRANSLATE_VAROP(s1);
1095                         break;
1096
1097                 case DF_2_TO_1:
1098                         TRANSLATE_VAROP(sx.s23.s2);
1099                 case DF_1_TO_1:
1100                 case DF_COPY:
1101                 case DF_MOVE:
1102                         TRANSLATE_VAROP(s1);
1103                 case DF_0_TO_1:
1104                         TRANSLATE_VAROP(dst);
1105                         break;
1106
1107                 case DF_N_TO_1:
1108                         n_iptr->sx.s23.s2.args = DMNEW(s4, n_iptr->s1.argcount);
1109                         for (i=0; i<n_iptr->s1.argcount; ++i) {
1110                                 n_iptr->sx.s23.s2.args[i] =
1111                                         inline_translate_variable(jd, origjd, varmap,
1112                                                         o_iptr->sx.s23.s2.args[i]);
1113                         }
1114                         TRANSLATE_VAROP(dst);
1115                         break;
1116
1117                 case DF_INVOKE:
1118                         INSTRUCTION_GET_METHODDESC(n_iptr, md);
1119 clone_call:
1120                         n_iptr->s1.argcount += iln->n_passthroughcount;
1121                         n_iptr->sx.s23.s2.args = DMNEW(s4, n_iptr->s1.argcount);
1122                         for (i=0; i<o_iptr->s1.argcount; ++i) {
1123                                 n_iptr->sx.s23.s2.args[i] =
1124                                         inline_translate_variable(jd, origjd, varmap,
1125                                                         o_iptr->sx.s23.s2.args[i]);
1126                         }
1127                         for (scope = iln; scope != NULL; scope = scope->parent) {
1128                                 for (j = 0; j < scope->n_selfpassthroughcount; ++j) {
1129                                         n_iptr->sx.s23.s2.args[i++] =
1130                                                 scope->parent->varmap[scope->n_passthroughvars[j]];
1131                                 }
1132                         }
1133                         if (md->returntype.type != TYPE_VOID)
1134                                 TRANSLATE_VAROP(dst);
1135                         break;
1136
1137                 case DF_BUILTIN:
1138                         bte = n_iptr->sx.s23.s3.bte;
1139                         md = bte->md;
1140                         goto clone_call;
1141
1142                 default:
1143                         assert(0);
1144         }
1145
1146         switch (icmdt->controlflow) {
1147                 case CF_RET:
1148                         TRANSLATE_VAROP(s1); /* XXX should be handled by data-flow */
1149                         /* FALLTHROUGH */
1150                 case CF_IF:
1151                 case CF_GOTO:
1152                         inline_add_block_reference(iln, &(n_iptr->dst.block));
1153                         break;
1154
1155                 case CF_JSR:
1156                         inline_add_block_reference(iln, &(n_iptr->sx.s23.s3.jsrtarget.block));
1157                         break;
1158
1159                 case CF_TABLE:
1160                         i = n_iptr->sx.s23.s3.tablehigh - n_iptr->sx.s23.s2.tablelow + 1 + 1 /* default */;
1161
1162                         table = DMNEW(branch_target_t, i);
1163                         MCOPY(table, o_iptr->dst.table, branch_target_t, i);
1164                         n_iptr->dst.table = table;
1165
1166                         while (--i >= 0) {
1167                                 inline_add_block_reference(iln, &(table->block));
1168                                 table++;
1169                         }
1170                         break;
1171
1172                 case CF_LOOKUP:
1173                         inline_add_block_reference(iln, &(n_iptr->sx.s23.s3.lookupdefault.block));
1174
1175                         i = n_iptr->sx.s23.s2.lookupcount;
1176                         lookup = DMNEW(lookup_target_t, i);
1177                         MCOPY(lookup, o_iptr->dst.lookup, lookup_target_t, i);
1178                         n_iptr->dst.lookup = lookup;
1179
1180                         while (--i >= 0) {
1181                                 inline_add_block_reference(iln, &(lookup->target.block));
1182                                 lookup++;
1183                         }
1184                         break;
1185         }
1186
1187         /* XXX move this to dataflow section? */
1188
1189         switch (n_iptr->opc) {
1190                 case ICMD_ISTORE:
1191                 case ICMD_LSTORE:
1192                 case ICMD_FSTORE:
1193                 case ICMD_DSTORE:
1194                 case ICMD_ASTORE:
1195                         /* XXX share code with stack.c */
1196                         j = n_iptr->dst.varindex;
1197                         i = n_iptr->sx.s23.s3.javaindex;
1198                         if (i != UNUSED) {
1199                                 if (n_iptr->flags.bits & INS_FLAG_RETADDR)
1200                                         iln->javalocals[i] = UNUSED;
1201                                 else
1202                                         iln->javalocals[i] = j;
1203                                 if (n_iptr->flags.bits & INS_FLAG_KILL_PREV)
1204                                         iln->javalocals[i-1] = UNUSED;
1205                                 if (n_iptr->flags.bits & INS_FLAG_KILL_NEXT)
1206                                         iln->javalocals[i+1] = UNUSED;
1207                         }
1208                         break;
1209         }
1210 }
1211
1212
1213 static void rewrite_method(inline_node *iln)
1214 {
1215         basicblock *o_bptr;
1216         s4 len;
1217         instruction *o_iptr;
1218         instruction *n_iptr;
1219         inline_node *nextcall;
1220         basicblock *n_bptr;
1221         inline_block_map *bm;
1222         int i;
1223         int icount;
1224         jitdata *resultjd;
1225         jitdata *origjd;
1226         char indent[100]; /* XXX debug */
1227         s4 retcount;
1228         s4 retidx;
1229
1230         assert(iln);
1231
1232         resultjd = iln->ctx->resultjd;
1233         origjd = iln->jd;
1234
1235         n_bptr = NULL;
1236         nextcall = iln->children;
1237
1238         /* XXX debug */
1239         for (i=0; i<iln->depth; ++i)
1240                 indent[i] = '\t';
1241         indent[i] = 0;
1242         iln->indent = indent;
1243
1244         DOLOG( printf("%srewriting: ", indent); method_println(iln->m);
1245                    printf("%s(passthrough: %d+%d)\n",
1246                                 indent, iln->n_passthroughcount - iln->n_selfpassthroughcount,
1247                                 iln->n_passthroughcount); );
1248
1249         /* set memory cursors */
1250
1251         iln->inlined_iinstr_cursor = iln->inlined_iinstr;
1252         iln->inlined_basicblocks_cursor = iln->inlined_basicblocks;
1253
1254         /* allocate temporary buffers */
1255
1256         iln->javalocals = DMNEW(s4, iln->jd->maxlocals);
1257
1258         /* loop over basic blocks */
1259
1260         o_bptr = iln->jd->basicblocks;
1261         for (; o_bptr; o_bptr = o_bptr->next) {
1262
1263                 if (o_bptr->flags < BBREACHED) {
1264
1265                         /* ignore the dummy end block */
1266
1267                         if (o_bptr->icount == 0 && o_bptr->next == NULL) {
1268                                 /* enter the following block as translation, for exception handler ranges */
1269                                 inline_block_translation(iln, o_bptr, iln->inlined_basicblocks_cursor);
1270                                 continue;
1271                         }
1272
1273                         DOLOG(
1274                         printf("%s* skipping old L%03d (flags=%d, type=%d, oid=%d) of ",
1275                                         indent,
1276                                         o_bptr->nr, o_bptr->flags, o_bptr->type,
1277                                         o_bptr->indepth);
1278                         method_println(iln->m);
1279                         );
1280
1281                         n_bptr = create_body_block(iln, o_bptr, iln->varmap);
1282
1283                         /* enter it in the blockmap */
1284
1285                         inline_block_translation(iln, o_bptr, n_bptr);
1286
1287                         close_body_block(iln, n_bptr, o_bptr, iln->varmap, 0, -1);
1288                         continue;
1289                 }
1290
1291                 len = o_bptr->icount;
1292                 o_iptr = o_bptr->iinstr;
1293
1294                 DOLOG(
1295                 printf("%s* rewriting old L%03d (flags=%d, type=%d, oid=%d) of ",
1296                                 indent,
1297                                 o_bptr->nr, o_bptr->flags, o_bptr->type,
1298                                 o_bptr->indepth);
1299                 method_println(iln->m);
1300                 show_basicblock(iln->jd, o_bptr, SHOW_STACK);
1301                 );
1302
1303                 if (iln->blockbefore || o_bptr != iln->jd->basicblocks) {
1304                         /* create an inlined clone of this block */
1305
1306                         n_bptr = create_body_block(iln, o_bptr, iln->varmap);
1307                         icount = 0;
1308
1309                         /* enter it in the blockmap */
1310
1311                         inline_block_translation(iln, o_bptr, n_bptr);
1312
1313                         /* initialize the javalocals */
1314
1315                         MCOPY(iln->javalocals, n_bptr->javalocals, s4, iln->jd->maxlocals);
1316                 }
1317                 else {
1318                         s4 *jl;
1319
1320                         /* continue caller block */
1321
1322                         n_bptr = iln->inlined_basicblocks_cursor - 1;
1323                         icount = n_bptr->icount;
1324
1325                         /* translate the javalocals */
1326
1327                         jl = translate_javalocals(iln, o_bptr->javalocals);
1328                         iln->inline_start_instruction->sx.s23.s3.inlineinfo->javalocals_start = jl;
1329
1330                         MCOPY(iln->javalocals, jl, s4, iln->jd->maxlocals);
1331                 }
1332
1333                 /* iterate over the ICMDs of this block */
1334
1335                 retcount = 0;
1336                 retidx = UNUSED;
1337
1338                 while (--len >= 0) {
1339
1340                         DOLOG( fputs(indent, stdout); show_icmd(iln->jd, o_iptr, false,  SHOW_STACK);
1341                                    printf("\n") );
1342
1343                         /* handle calls that will be inlined */
1344
1345                         if (nextcall && o_iptr == nextcall->callerins) {
1346
1347                                 /* write the inlining prolog */
1348
1349                                 (void) emit_inlining_prolog(iln, nextcall, o_iptr, iln->varmap);
1350                                 icount += nextcall->prolog_instructioncount;
1351
1352                                 /* end current block, or glue blocks together */
1353
1354                                 n_bptr->icount = icount;
1355
1356                                 if (nextcall->blockbefore) {
1357                                         close_prolog_block(iln, n_bptr, nextcall);
1358                                 }
1359                                 else {
1360                                         /* XXX */
1361                                 }
1362
1363                                 /* check if the result is a local variable */
1364
1365                                 if (nextcall->m->parseddesc->returntype.type != TYPE_VOID
1366                                                 && o_iptr->dst.varindex < iln->jd->localcount)
1367                                 {
1368                                         nextcall->n_resultlocal = iln->varmap[o_iptr->dst.varindex];
1369                                 }
1370                                 else
1371                                         nextcall->n_resultlocal = -1;
1372
1373                                 /* set memory pointers in the callee */
1374
1375                                 nextcall->inlined_iinstr = iln->inlined_iinstr_cursor;
1376                                 nextcall->inlined_basicblocks = iln->inlined_basicblocks_cursor;
1377
1378                                 /* recurse */
1379
1380                                 DOLOG( printf("%sentering inline ", indent);
1381                                            show_icmd(origjd, o_iptr, false, SHOW_STACK); printf("\n") );
1382
1383                                 rewrite_method(nextcall);
1384
1385                                 DOLOG( printf("%sleaving inline ", indent);
1386                                            show_icmd(origjd, o_iptr, false, SHOW_STACK); printf("\n") );
1387
1388                                 /* update memory cursors */
1389
1390                                 assert(nextcall->inlined_iinstr_cursor
1391                                                 <= iln->inlined_iinstr_cursor + nextcall->cumul_instructioncount);
1392                                 assert(nextcall->inlined_basicblocks_cursor
1393                                                 == iln->inlined_basicblocks_cursor + nextcall->cumul_basicblockcount);
1394                                 iln->inlined_iinstr_cursor = nextcall->inlined_iinstr_cursor;
1395                                 iln->inlined_basicblocks_cursor = nextcall->inlined_basicblocks_cursor;
1396
1397                                 /* start new block, or glue blocks together */
1398
1399                                 if (nextcall->blockafter) {
1400                                         n_bptr = create_epilog_block(iln, nextcall, iln->varmap);
1401                                         icount = 0;
1402                                 }
1403                                 else {
1404                                         n_bptr = iln->inlined_basicblocks_cursor - 1;
1405                                         icount = n_bptr->icount;
1406                                         /* XXX */
1407                                 }
1408
1409                                 /* emit inlining epilog */
1410
1411                                 emit_inlining_epilog(iln, nextcall, o_iptr);
1412                                 icount += nextcall->epilog_instructioncount;
1413
1414                                 /* proceed to next call */
1415
1416                                 nextcall = nextcall->next;
1417                         }
1418                         else {
1419                                 n_iptr = (iln->inlined_iinstr_cursor++);
1420                                 assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1421
1422                                 switch (o_iptr->opc) {
1423                                         case ICMD_RETURN:
1424                                                 if (iln->depth == 0)
1425                                                         goto default_clone;
1426                                                 goto return_tail;
1427
1428                                         case ICMD_IRETURN:
1429                                         case ICMD_ARETURN:
1430                                         case ICMD_LRETURN:
1431                                         case ICMD_FRETURN:
1432                                         case ICMD_DRETURN:
1433                                                 if (iln->depth == 0)
1434                                                         goto default_clone;
1435                                                 retcount = 1;
1436                                                 retidx = iln->varmap[o_iptr->s1.varindex];
1437                                                 if (iln->n_resultlocal != -1) {
1438                                                         /* store result in a local variable */
1439
1440                                                         DOLOG( printf("USING RESULTLOCAL %d\n", iln->n_resultlocal); );
1441                                                         /* This relies on the same sequence of types for */
1442                                                         /* ?STORE and ?RETURN opcodes.                   */
1443                                                         n_iptr->opc = ICMD_ISTORE + (o_iptr->opc - ICMD_IRETURN);
1444                                                         n_iptr->s1.varindex = retidx;
1445                                                         n_iptr->dst.varindex = iln->n_resultlocal;
1446                                                         n_iptr->sx.s23.s3.javaindex = UNUSED; /* XXX set real javaindex? */
1447
1448                                                         retcount = 0;
1449                                                         retidx = UNUSED;
1450
1451                                                         n_iptr = (iln->inlined_iinstr_cursor++);
1452                                                         assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1453                                                         icount++;
1454                                                 }
1455                                                 else if ((retidx < resultjd->localcount && iln->blockafter)
1456                                                                 || !iln->blockafter) /* XXX do we really always need the MOVE? */
1457                                                 {
1458                                                         /* local must not become outvar, insert a MOVE */
1459
1460                                                         n_iptr->opc = ICMD_MOVE;
1461                                                         n_iptr->s1.varindex = retidx;
1462                                                         retidx = inline_new_temp_variable(resultjd,
1463                                                                                                                           resultjd->var[retidx].type);
1464                                                         n_iptr->dst.varindex = retidx;
1465
1466                                                         n_iptr = (iln->inlined_iinstr_cursor++);
1467                                                         assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1468                                                         icount++;
1469                                                 }
1470 return_tail:
1471                                                 if (iln->blockafter) {
1472                                                         n_iptr->opc = ICMD_GOTO;
1473                                                         n_iptr->dst.block = INLINE_RETURN_REFERENCE(iln);
1474                                                         inline_add_block_reference(iln, &(n_iptr->dst.block));
1475                                                 }
1476                                                 else {
1477                                                         n_iptr->opc = ICMD_NOP;
1478                                                 }
1479                                                 break;
1480 #if 0
1481                                                 if (o_bptr->next == NULL || (o_bptr->next->icount==0 && o_bptr->next->next == NULL)) {
1482                                                         n_iptr->opc = ICMD_NOP;
1483                                                         break;
1484                                                 }
1485                                                 goto default_clone;
1486                                                 break;
1487 #endif
1488
1489                                         default:
1490 default_clone:
1491                                                 inline_clone_instruction(iln, resultjd, iln->jd, iln->varmap, o_iptr, n_iptr);
1492                                 }
1493
1494                                 DOLOG( fputs(indent, stdout); show_icmd(resultjd, n_iptr, false, SHOW_STACK);
1495                                            printf("\n"););
1496
1497                                 icount++;
1498                         }
1499
1500                         o_iptr++;
1501                 }
1502
1503                 /* end of basic block */
1504
1505                 if (iln->blockafter || (o_bptr->next && o_bptr->next->next)) {
1506                         close_body_block(iln, n_bptr, o_bptr, iln->varmap, retcount, retidx);
1507                         n_bptr->icount = icount;
1508
1509                         DOLOG( printf("closed body block:\n");
1510                                    show_basicblock(resultjd, n_bptr, SHOW_STACK); );
1511                 }
1512                 else {
1513                         n_bptr->icount = icount;
1514                         assert(iln->parent);
1515                         if (retidx != UNUSED)
1516                                 iln->parent->varmap[iln->callerins->dst.varindex] = retidx;
1517                 }
1518         }
1519
1520         bm = iln->ctx->blockmap;
1521         for (i=0; i<iln->ctx->blockmap_index; ++i, ++bm) {
1522                 assert(bm->iln && bm->o_block && bm->n_block);
1523                 if (bm->iln == iln)
1524                         inline_resolve_block_refs(&(iln->refs), bm->o_block, bm->n_block);
1525         }
1526
1527 #if !defined(NDEBUG)
1528         if (iln->refs) {
1529                 inline_target_ref *ref;
1530                 ref = iln->refs;
1531                 while (ref) {
1532                         if (!iln->depth || *(ref->ref) != INLINE_RETURN_REFERENCE(iln)) {
1533                                 DOLOG( printf("XXX REMAINING REF at depth %d: %p\n", iln->depth,
1534                                            (void*)*(ref->ref)) );
1535                                 assert(false);
1536                         }
1537                         ref = ref->next;
1538                 }
1539         }
1540 #endif
1541 }
1542
1543
1544 static exception_entry * inline_exception_tables(inline_node *iln,
1545                                                                                                  exception_entry *n_extable,
1546                                                                                                  exception_entry **prevextable)
1547 {
1548         inline_node *child;
1549         inline_node *scope;
1550         exception_entry *et;
1551
1552         assert(iln);
1553         assert(n_extable);
1554         assert(prevextable);
1555
1556         child = iln->children;
1557         if (child) {
1558                 do {
1559                         n_extable = inline_exception_tables(child, n_extable, prevextable);
1560                         child = child->next;
1561                 } while (child != iln->children);
1562         }
1563
1564         et = iln->jd->exceptiontable;
1565         for (; et != NULL; et = et->down) {
1566                 assert(et);
1567                 MZERO(n_extable, exception_entry, 1);
1568                 n_extable->start     = inline_map_block(iln, et->start  , iln);
1569                 n_extable->end       = inline_map_block(iln, et->end    , iln);
1570                 n_extable->handler   = inline_map_block(iln, et->handler, iln);
1571                 n_extable->catchtype = et->catchtype;
1572
1573                 if (*prevextable) {
1574                         (*prevextable)->down = n_extable;
1575                 }
1576                 *prevextable = n_extable;
1577
1578                 n_extable++;
1579         }
1580
1581         if (iln->handler_monitorexit) {
1582                 exception_entry **activehandlers;
1583
1584                 MZERO(n_extable, exception_entry, 1);
1585                 n_extable->start   = iln->inlined_basicblocks;
1586                 n_extable->end     = iln->inlined_basicblocks_cursor;
1587                 n_extable->handler = iln->handler_monitorexit;
1588                 n_extable->catchtype.any = NULL; /* finally */
1589
1590                 if (*prevextable) {
1591                         (*prevextable)->down = n_extable;
1592                 }
1593                 *prevextable = n_extable;
1594
1595                 n_extable++;
1596
1597                 /* We have to protect the created handler with the same handlers */
1598                 /* that protect the method body itself.                          */
1599
1600                 for (scope = iln; scope->parent != NULL; scope = scope->parent) {
1601
1602                         activehandlers = scope->o_handlers;
1603                         assert(activehandlers);
1604
1605                         while (*activehandlers) {
1606
1607                                 assert(scope->parent);
1608
1609                                 MZERO(n_extable, exception_entry, 1);
1610                                 n_extable->start     = iln->handler_monitorexit;
1611                                 n_extable->end       = iln->handler_monitorexit + 1; /* XXX ok in this case? */
1612                                 n_extable->handler   = inline_map_block(scope->parent,
1613                                                                                                                 (*activehandlers)->handler,
1614                                                                                                                 scope->parent);
1615                                 n_extable->catchtype = (*activehandlers)->catchtype;
1616
1617                                 if (*prevextable) {
1618                                         (*prevextable)->down = n_extable;
1619                                 }
1620                                 *prevextable = n_extable;
1621
1622                                 n_extable++;
1623                                 activehandlers++;
1624                         }
1625                 }
1626         }
1627
1628         return n_extable;
1629 }
1630
1631
1632 static void inline_locals(inline_node *iln)
1633 {
1634         inline_node *child;
1635
1636         assert(iln);
1637
1638         iln->varmap = create_variable_map(iln);
1639
1640         child = iln->children;
1641         if (child) {
1642                 do {
1643                         inline_locals(child);
1644                         child = child->next;
1645                 } while (child != iln->children);
1646         }
1647
1648         if (iln->regdata->memuse > iln->ctx->resultjd->rd->memuse)
1649                 iln->ctx->resultjd->rd->memuse = iln->regdata->memuse;
1650         if (iln->regdata->argintreguse > iln->ctx->resultjd->rd->argintreguse)
1651                 iln->ctx->resultjd->rd->argintreguse = iln->regdata->argintreguse;
1652         if (iln->regdata->argfltreguse > iln->ctx->resultjd->rd->argfltreguse)
1653                 iln->ctx->resultjd->rd->argfltreguse = iln->regdata->argfltreguse;
1654 }
1655
1656
1657 static void inline_interface_variables(inline_node *iln)
1658 {
1659         basicblock *bptr;
1660         jitdata *resultjd;
1661         s4 i;
1662         varinfo *v;
1663
1664         resultjd = iln->ctx->resultjd;
1665
1666         resultjd->interface_map = DMNEW(interface_info, 5*iln->ctx->maxinoutdepth);
1667         for (i=0; i<5*iln->ctx->maxinoutdepth; ++i)
1668                 resultjd->interface_map[i].flags = UNUSED;
1669
1670         for (bptr = resultjd->basicblocks; bptr != NULL; bptr = bptr->next) {
1671                 assert(bptr->indepth  <= iln->ctx->maxinoutdepth);
1672                 assert(bptr->outdepth <= iln->ctx->maxinoutdepth);
1673
1674                 for (i=0; i<bptr->indepth; ++i) {
1675                         v = &(resultjd->var[bptr->invars[i]]);
1676                         v->flags |= INOUT;
1677                         v->flags &= ~PREALLOC;
1678                         v->flags &= ~INMEMORY;
1679                         assert(bptr->invars[i] >= resultjd->localcount);
1680
1681                         if (resultjd->interface_map[5*i + v->type].flags == UNUSED) {
1682                                 resultjd->interface_map[5*i + v->type].flags = v->flags;
1683                         }
1684                         else {
1685                                 resultjd->interface_map[5*i + v->type].flags |= v->flags;
1686                         }
1687                 }
1688
1689                 for (i=0; i<bptr->outdepth; ++i) {
1690                         v = &(resultjd->var[bptr->outvars[i]]);
1691                         v->flags |= INOUT;
1692                         v->flags &= ~PREALLOC;
1693                         v->flags &= ~INMEMORY;
1694                         assert(bptr->outvars[i] >= resultjd->localcount);
1695
1696                         if (resultjd->interface_map[5*i + v->type].flags == UNUSED) {
1697                                 resultjd->interface_map[5*i + v->type].flags = v->flags;
1698                         }
1699                         else {
1700                                 resultjd->interface_map[5*i + v->type].flags |= v->flags;
1701                         }
1702                 }
1703         }
1704 }
1705
1706
1707 static void inline_write_exception_handlers(inline_node *master, inline_node *iln)
1708 {
1709         basicblock *n_bptr;
1710         instruction *n_ins;
1711         inline_node *child;
1712         builtintable_entry *bte;
1713         s4 exvar;
1714         s4 syncvar;
1715         s4 i;
1716
1717         child = iln->children;
1718         if (child) {
1719                 do {
1720                         inline_write_exception_handlers(master, child);
1721                         child = child->next;
1722                 } while (child != iln->children);
1723         }
1724
1725         if (iln->synchronize) {
1726                 /* create the monitorexit handler */
1727                 n_bptr = create_block(master, iln, iln, NULL, NULL,
1728                                                           iln->n_passthroughcount + 1);
1729                 n_bptr->type = BBTYPE_EXH;
1730                 n_bptr->flags = BBFINISHED;
1731
1732                 exvar = inline_new_variable(master->ctx->resultjd, TYPE_ADR, 0);
1733                 n_bptr->invars[iln->n_passthroughcount] = exvar;
1734
1735                 iln->handler_monitorexit = n_bptr;
1736
1737                 /* ACONST / ALOAD */
1738
1739                 n_ins = master->inlined_iinstr_cursor++;
1740                 if (iln->m->flags & ACC_STATIC) {
1741                         n_ins->opc = ICMD_ACONST;
1742                         n_ins->sx.val.c.cls = iln->m->class;
1743                         n_ins->flags.bits = INS_FLAG_CLASS;
1744                 }
1745                 else {
1746                         n_ins->opc = ICMD_ALOAD;
1747                         n_ins->s1.varindex = iln->synclocal;
1748                         assert(n_ins->s1.varindex != UNUSED);
1749                 }
1750                 /* XXX could be PREALLOCed for  builtin call */
1751                 syncvar = inline_new_variable(master->ctx->resultjd, TYPE_ADR, 0);
1752                 n_ins->dst.varindex = syncvar;
1753                 n_ins->line = 0;
1754
1755                 /* MONITOREXIT */
1756
1757                 bte = builtintable_get_internal(LOCK_monitor_exit);
1758
1759                 n_ins = master->inlined_iinstr_cursor++;
1760                 n_ins->opc = ICMD_BUILTIN;
1761                 n_ins->s1.argcount = 1 + iln->n_passthroughcount + 1;
1762                 n_ins->sx.s23.s2.args = DMNEW(s4, n_ins->s1.argcount);
1763                 n_ins->sx.s23.s2.args[0] = syncvar;
1764                 for (i=0; i < iln->n_passthroughcount + 1; ++i) {
1765                         n_ins->sx.s23.s2.args[1 + i] = n_bptr->invars[i];
1766                 }
1767                 n_ins->sx.s23.s3.bte = bte;
1768                 n_ins->line = 0;
1769
1770                 /* ATHROW */
1771
1772                 n_ins = master->inlined_iinstr_cursor++;
1773                 n_ins->opc = ICMD_ATHROW;
1774                 n_ins->flags.bits = 0;
1775                 n_ins->s1.varindex = exvar;
1776                 n_ins->line = 0;
1777
1778                 /* close basic block */
1779
1780                 close_block(iln, iln, n_bptr, iln->n_passthroughcount);
1781                 n_bptr->icount = 3;
1782         }
1783 }
1784
1785
1786 /* second pass driver *********************************************************/
1787
1788 static bool test_inlining(inline_node *iln, jitdata *jd,
1789                 jitdata **resultjd)
1790 {
1791         instruction *n_ins;
1792         basicblock *n_bb;
1793         basicblock *n_bptr;
1794         exception_entry *n_ext;
1795         exception_entry *prevext;
1796         codegendata *n_cd;
1797         jitdata *n_jd;
1798         s4 i;
1799
1800
1801 #if !defined(NDEBUG)
1802         static int debug_verify_inlined_code = 1; /* XXX */
1803         static int debug_compile_inlined_code_counter = 0;
1804 #endif
1805
1806         DOLOG( dump_inline_tree(iln, 0); );
1807
1808         assert(iln && jd && resultjd);
1809
1810         *resultjd = jd;
1811
1812         n_ins = DMNEW(instruction, iln->cumul_instructioncount);
1813         MZERO(n_ins, instruction, iln->cumul_instructioncount);
1814         iln->inlined_iinstr = n_ins;
1815
1816         n_bb = DMNEW(basicblock, iln->cumul_basicblockcount);
1817         MZERO(n_bb, basicblock, iln->cumul_basicblockcount);
1818         iln->inlined_basicblocks = n_bb;
1819
1820         iln->ctx->blockmap = DMNEW(inline_block_map, iln->cumul_blockmapcount);
1821
1822         n_jd = jit_jitdata_new(iln->m);
1823         n_jd->flags = jd->flags;
1824         iln->ctx->resultjd = n_jd;
1825
1826         reg_setup(n_jd);
1827
1828         /* create the local_map */
1829
1830         n_jd->local_map = DMNEW(s4, 5*iln->cumul_maxlocals);
1831         for (i=0; i<5*iln->cumul_maxlocals; ++i)
1832                 n_jd->local_map[i] = UNUSED;
1833
1834         /* create / coalesce local variables */
1835
1836         n_jd->varcount = 0;
1837         n_jd->vartop = 0;
1838         n_jd->var = NULL;
1839
1840         inline_locals(iln);
1841
1842         n_jd->localcount = n_jd->vartop;
1843
1844         /* extra variables for verification (DEBUG) */
1845
1846 #if !defined(NDEBUG)
1847         if (debug_verify_inlined_code) {
1848                 n_jd->vartop   += VERIFIER_EXTRA_LOCALS + VERIFIER_EXTRA_VARS + 100 /* XXX m->maxstack */;
1849                 if (n_jd->vartop > n_jd->varcount) {
1850                         /* XXX why? */
1851                         n_jd->var = DMREALLOC(n_jd->var, varinfo, n_jd->varcount, n_jd->vartop);
1852                         n_jd->varcount = n_jd->vartop;
1853                 }
1854         }
1855 #endif
1856
1857         /* write inlined code */
1858
1859         rewrite_method(iln);
1860
1861         /* create exception handlers */
1862
1863         inline_write_exception_handlers(iln, iln);
1864
1865         /* write the dummy end block */
1866
1867         n_bptr = create_block(iln, iln, iln, NULL, NULL, 0);
1868         n_bptr->flags = BBUNDEF;
1869         n_bptr->type = BBTYPE_STD;
1870
1871         /* store created code in jitdata */
1872
1873         n_jd->basicblocks = iln->inlined_basicblocks;
1874         n_jd->basicblockindex = NULL;
1875         n_jd->instructioncount = iln->cumul_instructioncount;
1876         n_jd->instructions = iln->inlined_iinstr;
1877
1878         /* link the basic blocks (dummy end block is not counted) */
1879
1880         n_jd->basicblockcount = (iln->inlined_basicblocks_cursor - iln->inlined_basicblocks) - 1;
1881         for (i=0; i<n_jd->basicblockcount + 1; ++i)
1882                 n_jd->basicblocks[i].next = &(n_jd->basicblocks[i+1]);
1883         if (i)
1884                 n_jd->basicblocks[i-1].next = NULL;
1885
1886         /* check basicblock numbers */
1887
1888 #if !defined(NDEBUG)
1889         jit_check_basicblock_numbers(n_jd);
1890 #endif
1891
1892         /* create the exception table */
1893
1894         if (iln->cumul_exceptiontablelength) {
1895                 exception_entry *tableend;
1896
1897                 n_ext = DMNEW(exception_entry, iln->cumul_exceptiontablelength);
1898                 prevext = NULL;
1899                 tableend = inline_exception_tables(iln, n_ext, &prevext);
1900                 assert(tableend == n_ext + iln->cumul_exceptiontablelength);
1901                 if (prevext)
1902                         prevext->down = NULL;
1903
1904                 n_jd->exceptiontablelength = iln->cumul_exceptiontablelength;
1905                 n_jd->exceptiontable = n_ext;
1906         }
1907         else {
1908                 n_ext = NULL;
1909         }
1910
1911         /*******************************************************************************/
1912
1913         n_cd = n_jd->cd;
1914         memcpy(n_cd, jd->cd, sizeof(codegendata));
1915
1916         n_cd->method = NULL; /* XXX */
1917         n_jd->maxlocals = iln->cumul_maxlocals;
1918         n_jd->maxinterfaces = iln->ctx->maxinoutdepth;
1919
1920         inline_post_process(n_jd);
1921
1922         inline_interface_variables(iln);
1923
1924 #if defined(ENABLE_VERIFIER) && !defined(NDEBUG)
1925         if (debug_verify_inlined_code) {
1926                 debug_verify_inlined_code = 0;
1927                 DOLOG( printf("VERIFYING INLINED RESULT...\n"); fflush(stdout); );
1928                 if (!typecheck(n_jd)) {
1929                         *exceptionptr = NULL;
1930                         DOLOG( printf("XXX INLINED RESULT DID NOT PASS VERIFIER XXX\n") );
1931                         return false;
1932                 }
1933                 else {
1934                         DOLOG( printf("VERIFICATION PASSED.\n") );
1935                 }
1936                 debug_verify_inlined_code = 1;
1937         }
1938 #endif /* defined(ENABLE_VERIFIER) */
1939
1940         /* we need bigger free memory stacks (XXX these should not be allocated in reg_setup) */
1941
1942         n_jd->rd->freemem = DMNEW(s4, iln->ctx->maxinoutdepth + 1000) /* XXX max vars/block */;
1943 #if defined(HAS_4BYTE_STACKSLOT)
1944         n_jd->rd->freemem_2 = DMNEW(s4, iln->ctx->maxinoutdepth + 1000) /* XXX max vars/block */;
1945 #endif
1946
1947 #if !defined(NDEBUG)
1948         if (n_jd->instructioncount >= inline_debug_min_size
1949                         && n_jd->instructioncount <= inline_debug_max_size)
1950         {
1951            if (debug_compile_inlined_code_counter >= inline_debug_start_counter
1952                            && debug_compile_inlined_code_counter <= inline_debug_end_counter)
1953 #endif /* NDEBUG */
1954            {
1955                         *resultjd = n_jd;
1956
1957 #if !defined(NDEBUG)
1958                         inline_count_methods++;
1959
1960                         /* inline_debug_log++; */
1961                         DOLOG(
1962                         printf("==== %d.INLINE ==================================================================\n", debug_compile_inlined_code_counter);
1963                         printf("\ninline tree:\n");
1964                         dump_inline_tree(iln, 0);
1965                         n_jd->flags |= JITDATA_FLAG_SHOWINTERMEDIATE | JITDATA_FLAG_SHOWDISASSEMBLE;
1966                         /* debug_dump_inlined_code(iln, n_method, n_cd, n_rd); */
1967                         printf("-------- DONE -----------------------------------------------------------\n");
1968                         fflush(stdout);
1969                         );
1970                         /* inline_debug_log--; */
1971 #endif
1972            }
1973
1974 #if !defined(NDEBUG)
1975                 debug_compile_inlined_code_counter++;
1976         }
1977 #endif
1978         return true;
1979 }
1980
1981
1982 /* first pass: build inlining tree ********************************************/
1983
1984 static bool inline_analyse_callee(inline_node *caller,
1985                                                                   methodinfo *callee,
1986                                                                   basicblock *callerblock,
1987                                                                   instruction *calleriptr,
1988                                                                   s4 callerpc,
1989                                                                   exception_entry **handlers,
1990                                                                   s4 nhandlers)
1991 {
1992         inline_node *cn;              /* the callee inline_node */
1993         s4           argi;
1994         bool         isstatic;
1995         s4           i, j;
1996         basicblock  *bptr;
1997
1998         /* create an inline tree node */
1999
2000         cn = DNEW(inline_node);
2001         MZERO(cn, inline_node, 1);
2002
2003         cn->ctx = caller->ctx;
2004         cn->m = callee;
2005         cn->synchronize = (callee->flags & ACC_SYNCHRONIZED);
2006         isstatic = (callee->flags & ACC_STATIC);
2007
2008         /* get the intermediate representation of the callee */
2009
2010         if (!inline_jit_compile(cn))
2011                 return false;
2012
2013         /* info about the call site */
2014
2015         cn->depth = caller->depth + 1;
2016         cn->callerblock = callerblock;
2017         cn->callerins = calleriptr;
2018         cn->callerpc = callerpc;
2019         cn->o_handlers = handlers;
2020         cn->n_handlercount = caller->n_handlercount + nhandlers;
2021
2022         /* determine if we need basic block boundaries before/after */
2023
2024         cn->blockbefore = false;
2025         cn->blockafter = false;
2026
2027         if (cn->jd->branchtoentry)
2028                 cn->blockbefore = true;
2029
2030         if (cn->jd->branchtoend)
2031                 cn->blockafter = true;
2032
2033         if (cn->jd->returncount > 1)
2034                 cn->blockafter = true;
2035
2036         /* XXX make safer and reusable (maybe store last real block) */
2037         for (bptr = cn->jd->basicblocks; bptr && bptr->next && bptr->next->next; bptr = bptr->next)
2038                 ;
2039
2040         if (cn->jd->returnblock != bptr)
2041                 cn->blockafter = true;
2042
2043         /* info about the callee */
2044
2045         cn->localsoffset = caller->localsoffset + caller->m->maxlocals;
2046         cn->prolog_instructioncount = callee->parseddesc->paramcount + 1;
2047         cn->epilog_instructioncount = 1; /* INLINE_END */
2048         cn->extra_instructioncount = 0;
2049
2050         /* we need a CHECKNULL for instance methods */
2051
2052         if (!isstatic)
2053                 cn->prolog_instructioncount += 1;
2054
2055         /* deal with synchronized callees */
2056
2057         if (cn->synchronize) {
2058                 methoddesc         *md;
2059                 builtintable_entry *bte;
2060
2061                 /* we need basic block boundaries because of the handler */
2062
2063                 cn->blockbefore = true;
2064                 cn->blockafter = true;
2065
2066                 /* for synchronized static methods                 */
2067                 /* we need an ACONST, MONITORENTER in the prolog   */
2068                 /* and ACONST, MONITOREXIT in the epilog           */
2069
2070                 /* for synchronized instance methods               */
2071                 /* we need an COPY, MONITORENTER in the prolog     */
2072                 /* and MONITOREXIT in the epilog                   */
2073
2074                 if (isstatic) {
2075                         cn->prolog_instructioncount += 2;
2076                         cn->epilog_instructioncount += 2;
2077                 }
2078                 else {
2079                         cn->prolog_instructioncount += 2;
2080                         cn->epilog_instructioncount += 1;
2081                         cn->localsoffset += 1;
2082                 }
2083
2084                 /* and exception handler */
2085                 /* ALOAD, builtin_monitorexit, ATHROW */
2086
2087                 cn->extra_instructioncount += 3;
2088
2089                 /* exception table entries */
2090
2091                 caller->cumul_exceptiontablelength += 1 + cn->n_handlercount;
2092
2093                 /* we must call the builtins */
2094
2095                 bte = builtintable_get_internal(LOCK_monitor_enter);
2096                 md = bte->md;
2097                 if (md->memuse > cn->regdata->memuse)
2098                         cn->regdata->memuse = md->memuse;
2099                 if (md->argintreguse > cn->regdata->argintreguse)
2100                         cn->regdata->argintreguse = md->argintreguse;
2101
2102                 bte = builtintable_get_internal(LOCK_monitor_exit);
2103                 md = bte->md;
2104                 if (md->memuse > cn->regdata->memuse)
2105                         cn->regdata->memuse = md->memuse;
2106                 if (md->argintreguse > cn->regdata->argintreguse)
2107                         cn->regdata->argintreguse = md->argintreguse;
2108
2109                 caller->ctx->calls_others = true;
2110         }
2111
2112         /* determine pass-through variables */
2113
2114         i = calleriptr->s1.argcount - callee->parseddesc->paramcount; /* max # of pass-though vars */
2115
2116         cn->n_passthroughvars = DMNEW(s4, i);
2117         j = 0;
2118         for (argi = calleriptr->s1.argcount - 1; argi >= callee->parseddesc->paramcount; --argi) {
2119                 s4 idx = calleriptr->sx.s23.s2.args[argi];
2120                 if (idx >= caller->jd->localcount) {
2121                         cn->n_passthroughvars[j] = idx;
2122                         j++;
2123                 }
2124                 else {
2125                         DOLOG( printf("PASSING THROUGH LOCAL VARIABLE %d\n", idx); );
2126                 }
2127         }
2128         assert(j <= i);
2129         cn->n_selfpassthroughcount = j;
2130         cn->n_passthroughcount = caller->n_passthroughcount + cn->n_selfpassthroughcount;
2131
2132         /* insert the node into the inline tree */
2133
2134         insert_inline_node(caller, cn);
2135
2136         /* analyse recursively */
2137
2138         if (!inline_inline_intern(callee, cn))
2139                 return false;
2140
2141         /* subtract one block if we continue the caller block */
2142
2143         if (!cn->blockbefore)
2144                 cn->cumul_basicblockcount -= 1;
2145
2146         /* add exception handler block for synchronized callees */
2147
2148         if (cn->synchronize) {
2149                 caller->ctx->master->cumul_basicblockcount++;
2150                 caller->ctx->master->cumul_blockmapcount++;
2151         }
2152
2153         /* cumulate counters */
2154
2155         caller->cumul_instructioncount += cn->prolog_instructioncount;
2156         caller->cumul_instructioncount += cn->epilog_instructioncount;
2157         caller->cumul_instructioncount += cn->extra_instructioncount;
2158         caller->cumul_instructioncount += cn->cumul_instructioncount - 1 /*invoke*/;
2159
2160         caller->cumul_basicblockcount += cn->cumul_basicblockcount;
2161         caller->cumul_blockmapcount += cn->cumul_blockmapcount;
2162         caller->cumul_exceptiontablelength += cn->cumul_exceptiontablelength;
2163         if (cn->cumul_maxlocals > caller->cumul_maxlocals)
2164                 caller->cumul_maxlocals = cn->cumul_maxlocals;
2165
2166         if (caller->cumul_basicblockcount > 10*caller->ctx->master->jd->basicblockcount) {
2167 #if 0
2168                 printf("STOPPING to avoid code explosion (%d blocks)\n",
2169                                 caller->cumul_basicblockcount);
2170 #endif
2171                 return false;
2172         }
2173
2174         /* XXX extra block after inlined call */
2175         if (cn->blockafter) {
2176                 caller->cumul_basicblockcount += 1;
2177                 caller->cumul_blockmapcount += 1;
2178         }
2179
2180         return true;
2181 }
2182
2183
2184 static bool inline_inline_intern(methodinfo *m, inline_node *iln)
2185 {
2186         basicblock *bptr;
2187         s4 len;
2188         instruction *iptr;
2189         int opcode;                                   /* invocation opcode */
2190         methodinfo *callee;
2191         inline_node *active;
2192         exception_entry **handlers;
2193         exception_entry *ex;
2194         s4 nhandlers;
2195         jitdata *mjd;
2196         bool speculative;
2197
2198         assert(iln);
2199
2200         mjd = iln->jd;
2201
2202         /* initialize cumulative counters */
2203
2204         iln->cumul_maxlocals = iln->localsoffset + m->maxlocals;
2205         iln->cumul_exceptiontablelength += mjd->exceptiontablelength;
2206
2207         /* iterate over basic blocks */
2208
2209         for (bptr = mjd->basicblocks; bptr; bptr = bptr->next) {
2210
2211                 /* ignore dummy end blocks (but count them for the blockmap) */
2212
2213                 iln->cumul_blockmapcount++;
2214                 if (bptr->icount > 0 || bptr->next != NULL)
2215                         iln->cumul_basicblockcount++;
2216
2217                 /* skip dead code */
2218
2219                 if (bptr->flags < BBREACHED)
2220                         continue;
2221
2222                 /* allocate the buffer of active exception handlers */
2223                 /* XXX this wastes some memory, but probably it does not matter */
2224
2225                 handlers = DMNEW(exception_entry*, mjd->exceptiontablelength + 1);
2226
2227                 /* determine the active exception handlers for this block     */
2228                 /* XXX maybe the handlers of a block should be part of our IR */
2229                 /* XXX this should share code with the type checkers          */
2230                 nhandlers = 0;
2231                 for (ex = mjd->exceptiontable; ex ; ex = ex->down) {
2232                         if ((ex->start->nr <= bptr->nr) && (ex->end->nr > bptr->nr)) {
2233                                 handlers[nhandlers++] = ex;
2234                         }
2235                 }
2236                 handlers[nhandlers] = NULL;
2237
2238                 len = bptr->icount;
2239                 iptr = bptr->iinstr;
2240
2241                 iln->instructioncount += len;
2242                 iln->cumul_instructioncount += len;
2243
2244                 for (; --len >= 0; ++iptr) {
2245
2246                         opcode = iptr->opc;
2247
2248                         switch (opcode) {
2249                                 /****************************************/
2250                                 /* INVOKATIONS                          */
2251
2252                                 case ICMD_INVOKEVIRTUAL:
2253                                 case ICMD_INVOKESPECIAL:
2254                                 case ICMD_INVOKESTATIC:
2255                                 case ICMD_INVOKEINTERFACE:
2256
2257                                         if (!INSTRUCTION_IS_UNRESOLVED(iptr)) {
2258                                                 callee = iptr->sx.s23.s3.fmiref->p.method;
2259
2260 #if 0
2261                                                 if (
2262                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Code_Table") == 0
2263                                                                 && (strcmp(callee->name->text, "of") == 0
2264                                                                  || strcmp(callee->name->text, "set") == 0))
2265                                                         ||
2266                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Compressor") == 0
2267                                                                 && (strcmp(callee->name->text, "output") == 0))
2268                                                         ||
2269                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Decompressor") == 0
2270                                                                 && (strcmp(callee->name->text, "getcode") == 0))
2271                                                         ||
2272                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Output_Buffer") == 0
2273                                                                 && (strcmp(callee->name->text, "putbyte") == 0))
2274                                                         ||
2275                                                         (strcmp(callee->class->name->text, "spec/benchmarks/_201_compress/Input_Buffer") == 0
2276                                                                 && (strcmp(callee->name->text, "getbyte") == 0
2277                                                                  || strcmp(callee->name->text, "readbytes") == 0
2278                                                                  ))
2279                                                         )
2280                                                         goto force_inline;
2281
2282                                                 if (callee->jcodelength > 0)
2283                                                         goto dont_inline;
2284 #endif
2285
2286                                                 if (callee->flags & ACC_NATIVE)
2287                                                         goto dont_inline;
2288
2289                                                 if (iln->depth >= 3)
2290                                                         goto dont_inline;
2291
2292                                                 if ((callee->flags & (ACC_STATIC | ACC_FINAL | ACC_PRIVATE)
2293                                                                         || opcode == ICMD_INVOKESPECIAL)) {
2294                                                         speculative = false;
2295                                                         goto maybe_inline;
2296                                                 }
2297
2298                                                 /* XXX search single implementation for abstract monomorphics */
2299                                                 if ((callee->flags & (ACC_METHOD_MONOMORPHIC | ACC_METHOD_IMPLEMENTED
2300                                                                                                                                          | ACC_ABSTRACT))
2301                                                                           == (ACC_METHOD_MONOMORPHIC | ACC_METHOD_IMPLEMENTED)) {
2302                                                         /* XXX */
2303                                                         if (0) {
2304                                                                 DOLOG( printf("SPECULATIVE INLINE: "); method_println(callee); );
2305                                                                 speculative = true;
2306                                                                 goto maybe_inline;
2307                                                         }
2308                                                 }
2309
2310                                                 /* polymorphic call site */
2311                                                 goto dont_inline;
2312
2313 maybe_inline:
2314                                                 for (active = iln; active; active = active->parent) {
2315                                                         if (callee == active->m) {
2316                                                                 DOLOG( printf("RECURSIVE!\n") );
2317                                                                 goto dont_inline;
2318                                                         }
2319                                                 }
2320 #if 0
2321 force_inline:
2322 #endif
2323                                                 if (!inline_analyse_callee(iln, callee,
2324                                                                         bptr,
2325                                                                         iptr,
2326                                                                         iln->instructioncount - len - 1 /* XXX ugly */,
2327                                                                         handlers,
2328                                                                         nhandlers))
2329                                                         return false;
2330
2331                                                 if (speculative)
2332                                                         method_add_assumption_monomorphic(callee, iln->ctx->master->m);
2333                                         }
2334 dont_inline:
2335                                         break;
2336
2337                                 case ICMD_RETURN:
2338                                 case ICMD_IRETURN:
2339                                 case ICMD_ARETURN:
2340                                 case ICMD_LRETURN:
2341                                 case ICMD_FRETURN:
2342                                 case ICMD_DRETURN:
2343                                         /* extra ICMD_MOVE may be necessary */
2344                                         iln->cumul_instructioncount++;
2345                                         break;
2346                         }
2347                 }
2348
2349                 /* end of basic block */
2350         }
2351
2352         return true;
2353 }
2354
2355
2356 /* post processing ************************************************************/
2357
2358 #define POSTPROCESS_SRC(varindex)  live[varindex]--
2359 #define POSTPROCESS_DST(varindex)  live[varindex]++
2360
2361 #define POSTPROCESS_SRCOP(s)  POSTPROCESS_SRC(iptr->s.varindex)
2362 #define POSTPROCESS_DSTOP(d)  POSTPROCESS_DST(iptr->d.varindex)
2363
2364 #define MARKSAVED(varindex)  jd->var[varindex].flags |= SAVEDVAR
2365
2366 #define MARK_ALL_SAVED                                               \
2367     do {                                                             \
2368         for (i=0; i<jd->vartop; ++i)                                 \
2369             if (live[i])                                             \
2370                 MARKSAVED(i);                                        \
2371     } while (0)
2372
2373 static void inline_post_process(jitdata *jd)
2374 {
2375         basicblock *bptr;
2376         instruction *iptr;
2377         instruction *iend;
2378         s4 i;
2379         icmdtable_entry_t *icmdt;
2380         s4 *live;
2381         methoddesc *md;
2382         builtintable_entry *bte;
2383
2384         /* reset the SAVEDVAR flag of all variables */
2385
2386         for (i=0; i<jd->vartop; ++i)
2387                 jd->var[i].flags &= ~SAVEDVAR;
2388
2389         /* allocate the life counters */
2390
2391         live = DMNEW(s4, jd->vartop);
2392         MZERO(live, s4, jd->vartop);
2393
2394         /* iterate over all basic blocks */
2395
2396         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
2397                 if (bptr->flags < BBREACHED)
2398                         continue;
2399
2400                 /* make invars live */
2401
2402                 for (i=0; i<bptr->indepth; ++i)
2403                         POSTPROCESS_DST(bptr->invars[i]);
2404
2405                 iptr = bptr->iinstr;
2406                 iend = iptr + bptr->icount;
2407
2408                 for (; iptr < iend; ++iptr) {
2409
2410                         icmdt = &(icmd_table[iptr->opc]);
2411
2412                         switch (icmdt->dataflow) {
2413                                 case DF_3_TO_0:
2414                                         POSTPROCESS_SRCOP(sx.s23.s3);
2415                                 case DF_2_TO_0:
2416                                         POSTPROCESS_SRCOP(sx.s23.s2);
2417                                 case DF_1_TO_0:
2418                                         POSTPROCESS_SRCOP(s1);
2419                                 case DF_0_TO_0:
2420                                         if (icmdt->flags & ICMDTABLE_CALLS) {
2421                                                 jd->isleafmethod = false;
2422                                                 MARK_ALL_SAVED;
2423                                         }
2424                                         break;
2425
2426                                 case DF_2_TO_1:
2427                                         POSTPROCESS_SRCOP(sx.s23.s2);
2428                                 case DF_1_TO_1:
2429                                 case DF_MOVE:
2430                                         POSTPROCESS_SRCOP(s1);
2431                                 case DF_0_TO_1:
2432                                         if (icmdt->flags & ICMDTABLE_CALLS) {
2433                                                 jd->isleafmethod = false;
2434                                                 MARK_ALL_SAVED;
2435                                         }
2436                                 case DF_COPY:
2437                                         POSTPROCESS_DSTOP(dst);
2438                                         break;
2439
2440                                 case DF_N_TO_1:
2441                                         for (i=0; i<iptr->s1.argcount; ++i) {
2442                                                 POSTPROCESS_SRC(iptr->sx.s23.s2.args[i]);
2443                                         }
2444                                         if (icmdt->flags & ICMDTABLE_CALLS) {
2445                                                 jd->isleafmethod = false;
2446                                                 MARK_ALL_SAVED;
2447                                         }
2448                                         POSTPROCESS_DSTOP(dst);
2449                                         break;
2450
2451                                 case DF_INVOKE:
2452                                         INSTRUCTION_GET_METHODDESC(iptr, md);
2453                 post_process_call:
2454                                         jd->isleafmethod = false;
2455                                         for (i=0; i<md->paramcount; ++i) {
2456                                                 POSTPROCESS_SRC(iptr->sx.s23.s2.args[i]);
2457                                         }
2458                                         for (; i<iptr->s1.argcount; ++i) {
2459                                                 MARKSAVED(iptr->sx.s23.s2.args[i]);
2460                                         }
2461                                         if (md->returntype.type != TYPE_VOID)
2462                                                 POSTPROCESS_DSTOP(dst);
2463                                         break;
2464
2465                                 case DF_BUILTIN:
2466                                         bte = iptr->sx.s23.s3.bte;
2467                                         md = bte->md;
2468                                         goto post_process_call;
2469
2470                                 default:
2471                                         assert(0);
2472                         }
2473
2474                 } /* end instruction loop */
2475
2476                 /* consume outvars */
2477
2478                 for (i=0; i<bptr->outdepth; ++i)
2479                         POSTPROCESS_SRC(bptr->outvars[i]);
2480
2481 #if !defined(NDEBUG)
2482                 for (i=jd->localcount; i < jd->vartop; ++i)
2483                         assert(live[i] == 0);
2484 #endif
2485
2486         } /* end basic block loop */
2487 }
2488
2489
2490 /* main driver function *******************************************************/
2491
2492 bool inline_inline(jitdata *jd, jitdata **resultjd)
2493 {
2494         inline_node *iln;
2495         methodinfo *m;
2496
2497         m = jd->m;
2498
2499         *resultjd = jd;
2500
2501         DOLOG( printf("==== INLINE ==================================================================\n");
2502                    show_method(jd, SHOW_STACK); );
2503
2504         iln = DNEW(inline_node);
2505         MZERO(iln, inline_node, 1);
2506
2507         iln->ctx = DNEW(inline_context);
2508         MZERO(iln->ctx, inline_context, 1);
2509         iln->ctx->master = iln;
2510         iln->ctx->calls_others = false;
2511         iln->m = m;
2512         iln->jd = jd;
2513         iln->regdata = jd->rd;
2514         iln->ctx->next_debugnr = 1; /* XXX debug */
2515
2516         iln->blockbefore = true;
2517         iln->blockafter = true;
2518
2519         /* we cannot use m->instructioncount because it may be greater than
2520          * the actual number of instructions in the basic blocks. */
2521         iln->instructioncount = 0;
2522         iln->cumul_instructioncount = 0;
2523         iln->cumul_basicblockcount = 1 /* dummy end block */;
2524
2525         if (inline_inline_intern(m, iln)) {
2526
2527                 DOLOG( printf("==== TEST INLINE =============================================================\n"); );
2528
2529                 if (iln->children)
2530                         test_inlining(iln, jd, resultjd);
2531         }
2532
2533         DOLOG( printf("-------- DONE -----------------------------------------------------------\n");
2534         fflush(stdout); );
2535
2536         return true;
2537 }
2538
2539 /*
2540  * These are local overrides for various environment variables in Emacs.
2541  * Please do not remove this and leave it at the end of the file, where
2542  * Emacs will automagically detect them.
2543  * ---------------------------------------------------------------------
2544  * Local variables:
2545  * mode: c
2546  * indent-tabs-mode: t
2547  * c-basic-offset: 4
2548  * tab-width: 4
2549  * End:
2550  * vim:noexpandtab:sw=4:ts=4:
2551  */