1 /* src/vm/jit/inline/inline.c - method inlining
3 Copyright (C) 1996-2005, 2006, 2007, 2008
4 CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
6 This file is part of CACAO.
8 This program is free software; you can redistribute it and/or
9 modify it under the terms of the GNU General Public License as
10 published by the Free Software Foundation; either version 2, or (at
11 your option) any later version.
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
35 #include "mm/memory.h"
37 #include "threads/lock-common.h"
38 #include "threads/thread.hpp"
40 #include "toolbox/logging.h"
42 #include "vm/builtin.h"
43 #include "vm/global.h"
44 #include "vm/initialize.h"
46 #include "vm/jit/jit.h"
47 #include "vm/jit/parse.h"
48 #include "vm/jit/reg.h"
49 #include "vm/jit/show.h"
50 #include "vm/jit/stack.h"
52 #include "vm/jit/inline/inline.h"
53 #include "vm/jit/loop/loop.h"
55 #include "vm/jit/verify/typecheck.h"
57 #include "vmcore/class.h"
58 #include "vmcore/method.h"
59 #include "vmcore/options.h"
60 #include "vmcore/statistics.h"
63 /* algorithm tuning constants *************************************************/
65 /* Algorithm Selection */
66 /* Define exactly one of the following three to select the inlining */
69 /*#define INLINE_DEPTH_FIRST*/
70 /*#define INLINE_BREADTH_FIRST*/
71 #define INLINE_KNAPSACK
73 /* Parameters for knapsack heuristics: */
75 #if defined(INLINE_KNAPSACK)
77 #define INLINE_COUNTDOWN_INIT 1000
78 #define INLINE_COST_OFFSET -16
79 #define INLINE_COST_BUDGET 100
80 /*#define INLINE_WEIGHT_BUDGET 5.0*/
81 /*#define INLINE_ADD_NEGATIVE_TO_BUDGET*/
82 /*#define INLINE_MAX_DEPTH 3*/
83 /*#define INLINE_DIVIDE_COST_BY_FREQ */
87 /* Parameters for depth-first heuristics: */
89 #if defined(INLINE_DEPTH_FIRST)
91 #define INLINE_MAX_DEPTH 3
92 #define INLINE_MAX_BLOCK_EXPANSION 10
93 /*#define INLINE_MAX_ICMD_EXPANSION 10*/
94 /*#define INLINE_CANCEL_ON_THRESHOLD*/
98 /* Parameters for breadth-first heuristics: */
100 #if defined(INLINE_BREADTH_FIRST)
102 /*#define INLINE_MAX_BLOCK_EXPANSION 10*/
103 #define INLINE_MAX_ICMD_EXPANSION 5
108 /* debugging ******************************************************************/
111 #define INLINE_VERBOSE
112 #define DOLOG(code) do{ if (opt_TraceInlining >= 2) { code; } }while(0)
113 #define DOLOG_SHORT(code) do{ if (opt_TraceInlining >= 1) { code; } }while(0)
118 #if defined(ENABLE_VERIFIER) && !defined(NDEBUG)
119 /* Define this to verify the resulting code after inlining. */
120 /* Note: This is only useful for development and may require patches to the */
122 /* #define INLINE_VERIFY_RESULT */
126 /* types **********************************************************************/
128 typedef struct inline_node inline_node;
129 typedef struct inline_target_ref inline_target_ref;
130 typedef struct inline_context inline_context;
131 typedef struct inline_block_map inline_block_map;
132 typedef struct inline_site inline_site;
133 typedef struct inline_candidate inline_candidate;
140 inline_node *children;
141 inline_node *next; /* next node at this depth */
142 inline_node *prev; /* prev node at this depth */
143 int depth; /* inlining depth, 0 for root */
145 /* info about the call site (if depth > 0)*/
146 inline_node *parent; /* node of the caller (NULL for root) */
147 basicblock *callerblock; /* original block containing the INVOKE* */
148 instruction *callerins; /* the original INVOKE* instruction */
150 s4 *n_passthroughvars;
151 int n_passthroughcount;
152 int n_selfpassthroughcount; /* # of pass-through vars of the call itself */
153 exception_entry **o_handlers;
154 int n_handlercount; /* # of handlers protecting this call */
156 int synclocal; /* variable used for synchr., or UNUSED */
157 bool isstatic; /* this is a static call */
159 bool blockbefore; /* block boundary before inlined body? */
160 bool blockafter; /* block boundary after inlined body? */
162 /* info about the callee */
164 int prolog_instructioncount; /* # of ICMDs in the inlining prolog */
165 int epilog_instructioncount; /* # of ICMDs in the inlining epilog */
166 int extra_instructioncount;
167 int extra_exceptiontablelength; /* # of extra handlers to put in caller */
168 bool synchronize; /* do we have to synchronize enter/exit? */
169 basicblock *handler_monitorexit; /* handler for synchronized inlinees */
172 /* cumulative values */
173 int cumul_instructioncount; /* ICMDs in this node and its children */
174 int cumul_basicblockcount; /* BBs started by this node and its children */
175 int cumul_basicblockcount_root; /* BBs that have to be added to the root */
176 /* node if this node is inlined */
177 int cumul_blockmapcount;
179 int cumul_exceptiontablelength;
182 instruction *inlined_iinstr;
183 instruction *inlined_iinstr_cursor;
184 basicblock *inlined_basicblocks;
185 basicblock *inlined_basicblocks_cursor;
188 registerdata *regdata;
191 inline_target_ref *refs;
192 instruction *inline_start_instruction;
200 struct inline_target_ref {
201 inline_target_ref *next;
210 struct inline_block_map {
216 struct inline_context {
221 inline_candidate *candidates;
223 int next_block_number;
224 inline_block_map *blockmap;
231 int next_debugnr; /* XXX debug */
235 bool speculative; /* true, if inlining would be speculative */
236 bool inlined; /* true, if this site has been inlined */
238 basicblock *bptr; /* basic block containing the call site */
239 instruction *iptr; /* the invocation instruction */
240 exception_entry **handlers; /* active handlers at the call site */
241 s4 nhandlers; /* number of active handlers */
242 s4 pc; /* PC of the invocation instruction */
245 struct inline_candidate {
246 inline_candidate *next;
256 /* prototypes *****************************************************************/
258 static bool inline_analyse_code(inline_node *iln);
259 static void inline_post_process(jitdata *jd);
262 /* debug helpers **************************************************************/
265 #include "inline_debug.inc"
269 /* statistics *****************************************************************/
271 /*#define INLINE_STATISTICS*/
274 #define INLINE_STATISTICS
277 #if defined(INLINE_STATISTICS)
278 int inline_stat_roots = 0;
279 int inline_stat_roots_transformed = 0;
280 int inline_stat_inlined_nodes = 0;
281 int inline_stat_max_depth = 0;
283 void inline_print_stats()
285 printf("inlining statistics:\n");
286 printf(" roots analysed : %d\n", inline_stat_roots);
287 printf(" roots transformed: %d\n", inline_stat_roots_transformed);
288 printf(" inlined nodes : %d\n", inline_stat_inlined_nodes);
289 printf(" max depth : %d\n", inline_stat_max_depth);
294 /* compilation of callees *****************************************************/
296 static bool inline_jit_compile_intern(jitdata *jd)
300 /* XXX should share code with jit.c */
304 /* XXX initialize the static function's class */
308 /* call the compiler passes ***********************************************/
310 /* call parse pass */
312 DOLOG( log_message_class("Parsing ", m->clazz) );
317 /* call stack analysis pass */
319 if (!stack_analyse(jd)) {
327 static bool inline_jit_compile(inline_node *iln)
333 /* XXX should share code with jit.c */
339 /* enter a monitor on the method */
341 LOCK_MONITOR_ENTER(m);
343 /* allocate jitdata structure and fill it */
345 jd = jit_jitdata_new(m);
348 jd->flags = 0; /* XXX */
350 /* initialize the register allocator */
354 /* setup the codegendata memory */
356 /* XXX do a pseudo setup */
357 jd->cd = DNEW(codegendata);
358 MZERO(jd->cd, codegendata, 1);
360 /* XXX uses too much dump memory codegen_setup(jd); */
362 /* now call internal compile function */
364 r = inline_jit_compile_intern(jd);
367 iln->regdata = jd->rd;
370 /* free some memory */
373 #if defined(ENABLE_JIT)
374 # if defined(ENABLE_INTRP)
382 /* leave the monitor */
384 LOCK_MONITOR_EXIT(m);
390 /* inlining tree handling *****************************************************/
392 static void inline_insert_inline_node(inline_node *parent, inline_node *child)
397 assert(parent && child);
399 child->parent = parent;
401 child->debugnr = parent->ctx->next_debugnr++; /* XXX debug */
403 first = parent->children;
405 /* insert as only node */
406 parent->children = child;
412 /* {there is at least one child already there} */
414 /* XXX is this search necessary, or could we always add at the end? */
417 while (succ->callerpc < child->callerpc) {
420 /* insert as last node */
421 child->prev = first->prev;
423 child->prev->next = child;
424 child->next->prev = child;
429 assert(succ->callerpc > child->callerpc);
431 /* insert before succ */
433 child->prev = succ->prev;
435 child->prev->next = child;
436 child->next->prev = child;
438 if (parent->children == succ)
439 parent->children = child;
443 static void inline_remove_inline_node(inline_node *parent, inline_node *child)
447 assert(child->parent == parent);
449 if (child->prev == child) {
450 /* remove the only child node */
451 parent->children = NULL;
454 child->prev->next = child->next;
455 child->next->prev = child->prev;
457 if (parent->children == child)
458 parent->children = child->next;
463 /* inlining candidate handling ************************************************/
465 #if defined(INLINE_KNAPSACK) || defined(INLINE_BREADTH_FIRST)
466 static void inline_add_candidate(inline_context *ctx,
471 inline_candidate **link;
472 inline_candidate *cand;
474 cand = DNEW(inline_candidate);
475 #if defined(INLINE_DIVIDE_COST_BY_FREQ)
476 cand->freq = INLINE_COUNTDOWN_INIT - callee->hitcountdown;
480 #if defined(INLINE_KNAPSACK)
481 cand->cost = callee->jcodelength + INLINE_COST_OFFSET;
483 #if defined(INLINE_BREADTH_FIRST)
484 cand->cost = caller->depth;
486 cand->caller = caller;
487 cand->callee = callee;
490 cand->weight = (double)cand->cost / cand->freq;
492 for (link = &(ctx->candidates); ; link = &((*link)->next)) {
493 if (!*link || (*link)->weight > cand->weight) {
500 #endif /* defined(INLINE_KNAPSACK) || defined(INLINE_BREADTH_FIRST) */
502 #if defined(INLINE_KNAPSACK) || defined(INLINE_BREADTH_FIRST)
503 static inline_candidate * inline_pick_best_candidate(inline_context *ctx)
505 inline_candidate *cand;
507 cand = ctx->candidates;
510 ctx->candidates = cand->next;
514 #endif /* defined(INLINE_KNAPSACK) || defined(INLINE_BREADTH_FIRST) */
516 #if !defined(NDEBUG) && (defined(INLINE_KNAPSACK) || defined(INLINE_BREADTH_FIRST))
517 static void inline_candidate_println(inline_candidate *cand)
519 printf("%10g (%5d / %5d) depth %2d ",
520 cand->weight, cand->cost, cand->freq, cand->caller->depth + 1);
521 method_println(cand->callee);
523 #endif /* !defined(NDEBUG) && (defined(INLINE_KNAPSACK) || defined(INLINE_BREADTH_FIRST)) */
526 #if !defined(NDEBUG) && (defined(INLINE_KNAPSACK) || defined(INLINE_BREADTH_FIRST))
527 static void inline_candidates_println(inline_context *ctx)
529 inline_candidate *cand;
531 for (cand = ctx->candidates; cand != NULL; cand = cand->next) {
533 inline_candidate_println(cand);
536 #endif /* !defined(NDEBUG) && (defined(INLINE_KNAPSACK) || defined(INLINE_BREADTH_FIRST)) */
539 /* variable handling **********************************************************/
541 static s4 inline_new_variable(jitdata *jd, s4 type, s4 flags)
546 index = jd->vartop++;
547 if (index >= jd->varcount) {
548 newcount = jd->vartop * 2; /* XXX */
549 jd->var = DMREALLOC(jd->var, varinfo, jd->varcount, newcount);
550 MZERO(jd->var + jd->varcount, varinfo, (newcount - jd->varcount));
551 jd->varcount = newcount;
554 jd->var[index].type = type;
555 jd->var[index].flags = flags;
561 static s4 inline_new_variable_clone(jitdata *jd, jitdata *origjd, s4 origidx)
566 v = &(origjd->var[origidx]);
568 newidx = inline_new_variable(jd, v->type, v->flags);
570 jd->var[newidx].vv = v->vv;
576 static s4 inline_new_temp_variable(jitdata *jd, s4 type)
578 return inline_new_variable(jd, type, 0);
582 static s4 inline_translate_variable(jitdata *jd, jitdata *origjd, s4 *varmap, s4 index)
589 idx = inline_new_variable_clone(jd, origjd, index);
597 static s4 *create_variable_map(inline_node *callee)
606 /* create the variable mapping */
608 varmap = DMNEW(s4, callee->jd->varcount);
609 for (i=0; i<callee->jd->varcount; ++i)
612 /* translate local variables */
614 for (i=0; i<callee->m->maxlocals; ++i) {
615 for (t=0; t<5; ++t) {
616 varindex = callee->jd->local_map[5*i + t];
617 if (varindex == UNUSED)
620 v = &(callee->jd->var[varindex]);
621 assert(v->type == t || v->type == TYPE_VOID); /* XXX stack leaves VOID */
622 v->type = t; /* XXX restore if it is TYPE_VOID */
624 avail = callee->ctx->resultjd->local_map[5*(callee->localsoffset + i) + t];
626 if (avail == UNUSED) {
627 avail = inline_new_variable_clone(callee->ctx->resultjd, callee->jd, varindex);
628 callee->ctx->resultjd->local_map[5*(callee->localsoffset + i) + t] = avail;
631 varmap[varindex] = avail;
635 /* for synchronized instance methods we need an extra local */
637 if (callee->synchronize && !(callee->m->flags & ACC_STATIC)) {
638 n_javaindex = callee->localsoffset - 1;
639 assert(n_javaindex >= 0);
640 assert(callee->parent);
641 assert(n_javaindex == callee->parent->localsoffset + callee->parent->m->maxlocals);
643 avail = callee->ctx->resultjd->local_map[5*n_javaindex + TYPE_ADR];
645 if (avail == UNUSED) {
646 avail = inline_new_variable(callee->ctx->resultjd, TYPE_ADR, 0);
647 callee->ctx->resultjd->local_map[5*n_javaindex + TYPE_ADR] = avail;
650 callee->synclocal = avail;
653 callee->synclocal = UNUSED;
660 /* basic block translation ****************************************************/
662 #define INLINE_RETURN_REFERENCE(callee) \
663 ( (basicblock *) (ptrint) (0x333 + (callee)->depth) )
666 static void inline_add_block_reference(inline_node *iln, basicblock **blockp)
668 inline_target_ref *ref;
670 ref = DNEW(inline_target_ref);
671 ref->ref.block = blockp;
672 ref->isnumber = false;
673 ref->next = iln->refs;
679 static void inline_add_blocknr_reference(inline_node *iln, s4 *nrp)
681 inline_target_ref *ref;
683 ref = DNEW(inline_target_ref);
685 ref->isnumber = true;
686 ref->next = iln->refs;
692 static void inline_block_translation(inline_node *iln, basicblock *o_bptr, basicblock *n_bptr)
697 assert(ctx->blockmap_index < ctx->master->cumul_blockmapcount);
699 ctx->blockmap[ctx->blockmap_index].iln = iln;
700 ctx->blockmap[ctx->blockmap_index].o_block = o_bptr;
701 ctx->blockmap[ctx->blockmap_index].n_block = n_bptr;
703 ctx->blockmap_index++;
707 static basicblock * inline_map_block(inline_node *iln,
709 inline_node *targetiln)
711 inline_block_map *bm;
712 inline_block_map *bmend;
720 bm = iln->ctx->blockmap;
721 bmend = bm + iln->ctx->blockmap_index;
724 assert(bm->iln && bm->o_block && bm->n_block);
725 if (bm->o_block == o_block && bm->iln == targetiln)
731 return NULL; /* not reached */
735 static void inline_resolve_block_refs(inline_target_ref **refs,
740 inline_target_ref *ref;
741 inline_target_ref *prev;
744 for (ref = *refs; ref != NULL; ref = ref->next) {
745 if (ref->isnumber && !returnref) {
746 if (*(ref->ref.nr) == JAVALOCAL_FROM_RETADDR(o_bptr->nr)) {
747 *(ref->ref.nr) = JAVALOCAL_FROM_RETADDR(n_bptr->nr);
752 if (*(ref->ref.block) == o_bptr) {
753 *(ref->ref.block) = n_bptr;
764 /* remove this ref */
767 prev->next = ref->next;
776 /* basic block creation *******************************************************/
778 static basicblock * create_block(inline_node *container,
793 assert(indepth >= 0);
795 n_bptr = container->inlined_basicblocks_cursor++;
797 assert((n_bptr - container->inlined_basicblocks) < container->cumul_basicblockcount);
799 BASICBLOCK_INIT(n_bptr, iln->m);
801 n_bptr->iinstr = container->inlined_iinstr_cursor;
802 n_bptr->next = n_bptr + 1;
803 n_bptr->nr = container->ctx->next_block_number++;
804 n_bptr->indepth = indepth;
805 n_bptr->flags = BBFINISHED; /* XXX */
807 /* set the inlineinfo of the new block */
809 if (iln->inline_start_instruction)
810 n_bptr->inlineinfo = iln->inline_start_instruction->sx.s23.s3.inlineinfo;
812 if (indepth > container->ctx->maxinoutdepth)
813 container->ctx->maxinoutdepth = indepth;
816 n_bptr->invars = DMNEW(s4, indepth);
819 for (i=0; i<indepth; ++i)
820 n_bptr->invars[i] = -1; /* XXX debug */
822 /* pass-through variables enter the block */
824 outer = inner->parent;
825 while (outer != NULL) {
826 depth = outer->n_passthroughcount;
828 assert(depth + inner->n_selfpassthroughcount <= indepth);
830 for (i=0; i<inner->n_selfpassthroughcount; ++i) {
831 varidx = inner->n_passthroughvars[i];
833 inline_new_variable_clone(container->ctx->resultjd,
836 n_bptr->invars[depth + i] = newvaridx;
837 outer->varmap[varidx] = newvaridx;
840 outer = outer->parent;
844 n_bptr->invars = NULL;
847 /* XXX for the verifier. should not be here */
852 dv = DMNEW(varinfo, iln->ctx->resultjd->localcount + VERIFIER_EXTRA_LOCALS);
853 MZERO(dv, varinfo, iln->ctx->resultjd->localcount + VERIFIER_EXTRA_LOCALS);
854 n_bptr->inlocals = dv;
861 static s4 *translate_javalocals(inline_node *iln, s4 *javalocals)
866 jl = DMNEW(s4, iln->jd->maxlocals);
868 for (i=0; i<iln->jd->maxlocals; ++i) {
871 j = inline_translate_variable(iln->ctx->resultjd, iln->jd, iln->varmap, j);
876 /* an encoded returnAddress value - must be relocated */
877 inline_add_blocknr_reference(iln, &(jl[i]));
886 static basicblock * create_body_block(inline_node *iln,
887 basicblock *o_bptr, s4 *varmap)
892 n_bptr = create_block(iln, iln, iln,
893 o_bptr->indepth + iln->n_passthroughcount);
895 n_bptr->type = o_bptr->type;
896 n_bptr->flags = o_bptr->flags;
897 n_bptr->bitflags = o_bptr->bitflags;
899 /* resolve references to this block */
901 inline_resolve_block_refs(&(iln->refs), o_bptr, n_bptr, false);
903 /* translate the invars of the original block */
905 for (i=0; i<o_bptr->indepth; ++i) {
906 n_bptr->invars[iln->n_passthroughcount + i] =
907 inline_translate_variable(iln->ctx->resultjd, iln->jd,
912 /* translate javalocals info (not for dead code) */
914 if (n_bptr->flags >= BBREACHED)
915 n_bptr->javalocals = translate_javalocals(iln, o_bptr->javalocals);
921 static basicblock * create_epilog_block(inline_node *caller, inline_node *callee, s4 *varmap)
927 /* number of return variables */
929 retcount = (callee->n_resultlocal == -1
930 && callee->m->parseddesc->returntype.type != TYPE_VOID) ? 1 : 0;
932 /* start the epilog block */
934 n_bptr = create_block(caller, caller, callee,
935 callee->n_passthroughcount + retcount);
937 /* resolve references to the return block */
939 inline_resolve_block_refs(&(callee->refs),
940 INLINE_RETURN_REFERENCE(callee),
944 /* return variable */
947 idx = inline_new_variable(caller->ctx->resultjd,
948 callee->m->parseddesc->returntype.type, 0 /* XXX */);
949 n_bptr->invars[callee->n_passthroughcount] = idx;
950 varmap[callee->callerins->dst.varindex] = idx;
955 n_bptr->javalocals = DMNEW(s4, caller->jd->maxlocals);
956 MCOPY(n_bptr->javalocals, caller->javalocals, s4, caller->jd->maxlocals);
958 /* set block flags & type */
960 n_bptr->flags = /* XXX original block flags */ BBFINISHED;
961 n_bptr->type = BBTYPE_STD;
967 static void close_block(inline_node *iln, inline_node *inner, basicblock *n_bptr, s4 outdepth)
974 n_bptr->outdepth = outdepth;
975 n_bptr->outvars = DMNEW(s4, outdepth);
977 for (i=0; i<outdepth; ++i)
978 n_bptr->outvars[i] = 0; /* XXX debug */
980 if (outdepth > iln->ctx->maxinoutdepth)
981 iln->ctx->maxinoutdepth = outdepth;
983 /* pass-through variables leave the block */
985 outer = inner->parent;
986 while (outer != NULL) {
987 depth = outer->n_passthroughcount;
989 assert(depth + inner->n_selfpassthroughcount <= outdepth);
991 for (i=0; i<inner->n_selfpassthroughcount; ++i) {
992 varidx = inner->n_passthroughvars[i];
993 n_bptr->outvars[depth + i] =
994 inline_translate_variable(iln->ctx->resultjd,
1000 outer = outer->parent;
1005 static void close_prolog_block(inline_node *iln,
1007 inline_node *nextcall)
1009 /* XXX add original outvars! */
1010 close_block(iln, nextcall, n_bptr, nextcall->n_passthroughcount);
1012 /* pass-through variables */
1014 DOLOG( printf("closed prolog block:\n");
1015 show_basicblock(iln->ctx->resultjd, n_bptr, SHOW_STACK); );
1019 static void close_body_block(inline_node *iln,
1028 close_block(iln, iln, n_bptr, iln->n_passthroughcount + o_bptr->outdepth + retcount);
1030 /* translate the outvars of the original block */
1032 /* XXX reuse code */
1033 for (i=0; i<o_bptr->outdepth; ++i) {
1034 n_bptr->outvars[iln->n_passthroughcount + i] =
1035 inline_translate_variable(iln->ctx->resultjd, iln->jd, varmap,
1036 o_bptr->outvars[i]);
1039 /* set the return variable, if any */
1042 assert(retidx >= 0);
1043 n_bptr->outvars[iln->n_passthroughcount + o_bptr->outdepth] = retidx;
1048 /* inlined code generation ****************************************************/
1050 static instruction * inline_instruction(inline_node *iln,
1052 instruction *o_iptr)
1054 instruction *n_iptr;
1056 n_iptr = (iln->inlined_iinstr_cursor++);
1057 assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1059 n_iptr->opc = opcode;
1060 n_iptr->flags.bits = o_iptr->flags.bits & INS_FLAG_ID_MASK;
1061 n_iptr->line = o_iptr->line;
1066 static void inline_generate_sync_builtin(inline_node *iln,
1067 inline_node *callee,
1068 instruction *o_iptr,
1075 if (callee->m->flags & ACC_STATIC) {
1077 syncvar = inline_new_temp_variable(iln->ctx->resultjd, TYPE_ADR);
1079 n_ins = inline_instruction(iln, ICMD_ACONST, o_iptr);
1080 n_ins->sx.val.c.cls = callee->m->clazz;
1081 n_ins->dst.varindex = syncvar;
1082 n_ins->flags.bits |= INS_FLAG_CLASS;
1085 syncvar = instancevar;
1088 assert(syncvar != UNUSED);
1090 /* MONITORENTER / MONITOREXIT */
1092 n_ins = inline_instruction(iln, ICMD_BUILTIN, o_iptr);
1093 n_ins->sx.s23.s3.bte = builtintable_get_internal(func);
1094 n_ins->s1.argcount = 1; /* XXX add through-vars */
1095 n_ins->sx.s23.s2.args = DMNEW(s4, 1);
1096 n_ins->sx.s23.s2.args[0] = syncvar;
1099 static s4 emit_inlining_prolog(inline_node *iln,
1100 inline_node *callee,
1101 instruction *o_iptr,
1104 methodinfo *calleem;
1110 insinfo_inline *insinfo;
1113 assert(iln && callee && o_iptr);
1115 calleem = callee->m;
1116 md = calleem->parseddesc;
1118 /* INLINE_START instruction */
1120 n_ins = inline_instruction(iln, ICMD_INLINE_START, o_iptr);
1122 insinfo = DNEW(insinfo_inline);
1123 insinfo->method = callee->m;
1124 insinfo->outer = iln->m;
1125 insinfo->synclocal = callee->synclocal;
1126 insinfo->synchronize = callee->synchronize;
1127 insinfo->javalocals_start = NULL;
1128 insinfo->javalocals_end = NULL;
1130 /* info about stack vars live at the INLINE_START */
1132 insinfo->throughcount = callee->n_passthroughcount;
1133 insinfo->paramcount = md->paramcount;
1134 insinfo->stackvarscount = o_iptr->s1.argcount;
1135 insinfo->stackvars = DMNEW(s4, insinfo->stackvarscount);
1136 for (i=0; i<insinfo->stackvarscount; ++i)
1137 insinfo->stackvars[i] = iln->varmap[o_iptr->sx.s23.s2.args[i]];
1139 /* info about the surrounding inlining */
1141 if (iln->inline_start_instruction)
1142 insinfo->parent = iln->inline_start_instruction->sx.s23.s3.inlineinfo;
1144 insinfo->parent = NULL;
1146 /* finish the INLINE_START instruction */
1148 n_ins->sx.s23.s3.inlineinfo = insinfo;
1149 callee->inline_start_instruction = n_ins;
1151 DOLOG( printf("%sprolog: ", iln->indent);
1152 show_icmd(iln->ctx->resultjd, n_ins, false, SHOW_STACK); printf("\n"); );
1154 /* handle parameters for the inlined callee */
1156 localindex = callee->localsoffset + md->paramslots;
1158 for (i=md->paramcount-1; i>=0; --i) {
1161 type = md->paramtypes[i].type;
1163 localindex -= IS_2_WORD_TYPE(type) ? 2 : 1;
1164 assert(callee->regdata);
1166 /* translate the argument variable */
1168 varindex = varmap[o_iptr->sx.s23.s2.args[i]];
1169 assert(varindex != UNUSED);
1171 /* remove preallocation from the argument variable */
1173 iln->ctx->resultjd->var[varindex].flags &= ~(PREALLOC | INMEMORY);
1175 /* check the instance slot against NULL */
1176 /* we don't need that for <init> methods, as the verifier */
1177 /* ensures that they are only called for an uninit. object */
1178 /* (which may not be NULL). */
1180 if (!callee->isstatic && i == 0 && calleem->name != utf_init) {
1181 assert(type == TYPE_ADR);
1182 n_ins = inline_instruction(iln, ICMD_CHECKNULL, o_iptr);
1183 n_ins->s1.varindex = varindex;
1184 n_ins->dst.varindex = n_ins->s1.varindex;
1187 /* store argument into local variable of inlined callee */
1189 if (callee->jd->local_map[5*(localindex - callee->localsoffset) + type] != UNUSED)
1191 /* this value is used in the callee */
1193 if (i == 0 && callee->synclocal != UNUSED) {
1194 /* we also need it for synchronization, so copy it */
1195 assert(type == TYPE_ADR);
1196 n_ins = inline_instruction(iln, ICMD_COPY, o_iptr);
1199 n_ins = inline_instruction(iln, ICMD_ISTORE + type, o_iptr);
1200 n_ins->sx.s23.s3.javaindex = UNUSED;
1202 n_ins->s1.varindex = varindex;
1203 n_ins->dst.varindex = iln->ctx->resultjd->local_map[5*localindex + type];
1204 assert(n_ins->dst.varindex != UNUSED);
1206 else if (i == 0 && callee->synclocal != UNUSED) {
1207 /* the value is not used inside the callee, but we need it for */
1208 /* synchronization */
1209 /* XXX In this case it actually makes no sense to create a */
1210 /* separate synchronization variable. */
1212 n_ins = inline_instruction(iln, ICMD_NOP, o_iptr);
1215 /* this value is not used, pop it */
1217 n_ins = inline_instruction(iln, ICMD_POP, o_iptr);
1218 n_ins->s1.varindex = varindex;
1221 DOLOG( printf("%sprolog: ", iln->indent);
1222 show_icmd(iln->ctx->resultjd, n_ins, false, SHOW_STACK); printf("\n"); );
1225 /* COPY for synchronized instance methods */
1227 if (callee->synclocal != UNUSED) {
1228 n_ins = inline_instruction(iln, ICMD_COPY, o_iptr);
1229 n_ins->s1.varindex = varmap[o_iptr->sx.s23.s2.args[0]];
1230 n_ins->dst.varindex = callee->synclocal;
1232 assert(n_ins->s1.varindex != UNUSED);
1235 if (callee->synchronize) {
1236 inline_generate_sync_builtin(iln, callee, o_iptr,
1237 (callee->isstatic) ? UNUSED : varmap[o_iptr->sx.s23.s2.args[0]],
1238 LOCK_monitor_enter);
1241 /* INLINE_BODY instruction */
1243 n_ins = inline_instruction(iln, ICMD_INLINE_BODY, callee->jd->basicblocks[0].iinstr);
1244 n_ins->sx.s23.s3.inlineinfo = insinfo;
1250 static void emit_inlining_epilog(inline_node *iln, inline_node *callee, instruction *o_iptr)
1255 assert(iln && callee && o_iptr);
1256 assert(callee->inline_start_instruction);
1258 if (callee->synchronize) {
1259 inline_generate_sync_builtin(iln, callee, o_iptr,
1264 /* INLINE_END instruction */
1266 n_ins = inline_instruction(iln, ICMD_INLINE_END, o_iptr);
1267 n_ins->sx.s23.s3.inlineinfo = callee->inline_start_instruction->sx.s23.s3.inlineinfo;
1269 /* set the javalocals */
1271 jl = DMNEW(s4, iln->jd->maxlocals);
1272 MCOPY(jl, iln->javalocals, s4, iln->jd->maxlocals);
1273 n_ins->sx.s23.s3.inlineinfo->javalocals_end = jl;
1275 DOLOG( printf("%sepilog: ", iln->indent);
1276 show_icmd(iln->ctx->resultjd, n_ins, false, SHOW_STACK); printf("\n"); );
1280 #define TRANSLATE_VAROP(vo) \
1281 n_iptr->vo.varindex = inline_translate_variable(jd, origjd, varmap, n_iptr->vo.varindex)
1284 static void inline_clone_instruction(inline_node *iln,
1288 instruction *o_iptr,
1289 instruction *n_iptr)
1291 icmdtable_entry_t *icmdt;
1292 builtintable_entry *bte;
1295 branch_target_t *table;
1296 lookup_target_t *lookup;
1301 icmdt = &(icmd_table[o_iptr->opc]);
1303 switch (icmdt->dataflow) {
1308 TRANSLATE_VAROP(sx.s23.s3);
1310 TRANSLATE_VAROP(sx.s23.s2);
1312 TRANSLATE_VAROP(s1);
1316 TRANSLATE_VAROP(sx.s23.s2);
1320 TRANSLATE_VAROP(s1);
1322 TRANSLATE_VAROP(dst);
1326 n_iptr->sx.s23.s2.args = DMNEW(s4, n_iptr->s1.argcount);
1327 for (i=0; i<n_iptr->s1.argcount; ++i) {
1328 n_iptr->sx.s23.s2.args[i] =
1329 inline_translate_variable(jd, origjd, varmap,
1330 o_iptr->sx.s23.s2.args[i]);
1332 TRANSLATE_VAROP(dst);
1336 INSTRUCTION_GET_METHODDESC(n_iptr, md);
1338 n_iptr->s1.argcount += iln->n_passthroughcount;
1339 n_iptr->sx.s23.s2.args = DMNEW(s4, n_iptr->s1.argcount);
1340 for (i=0; i<o_iptr->s1.argcount; ++i) {
1341 n_iptr->sx.s23.s2.args[i] =
1342 inline_translate_variable(jd, origjd, varmap,
1343 o_iptr->sx.s23.s2.args[i]);
1345 for (scope = iln; scope != NULL; scope = scope->parent) {
1346 for (j = 0; j < scope->n_selfpassthroughcount; ++j) {
1347 n_iptr->sx.s23.s2.args[i++] =
1348 scope->parent->varmap[scope->n_passthroughvars[j]];
1351 if (md->returntype.type != TYPE_VOID)
1352 TRANSLATE_VAROP(dst);
1356 bte = n_iptr->sx.s23.s3.bte;
1364 switch (icmdt->controlflow) {
1366 TRANSLATE_VAROP(s1); /* XXX should be handled by data-flow */
1370 inline_add_block_reference(iln, &(n_iptr->dst.block));
1374 inline_add_block_reference(iln, &(n_iptr->sx.s23.s3.jsrtarget.block));
1378 i = n_iptr->sx.s23.s3.tablehigh - n_iptr->sx.s23.s2.tablelow + 1 + 1 /* default */;
1380 table = DMNEW(branch_target_t, i);
1381 MCOPY(table, o_iptr->dst.table, branch_target_t, i);
1382 n_iptr->dst.table = table;
1385 inline_add_block_reference(iln, &(table->block));
1391 inline_add_block_reference(iln, &(n_iptr->sx.s23.s3.lookupdefault.block));
1393 i = n_iptr->sx.s23.s2.lookupcount;
1394 lookup = DMNEW(lookup_target_t, i);
1395 MCOPY(lookup, o_iptr->dst.lookup, lookup_target_t, i);
1396 n_iptr->dst.lookup = lookup;
1399 inline_add_block_reference(iln, &(lookup->target.block));
1405 /* XXX move this to dataflow section? */
1407 switch (n_iptr->opc) {
1410 if (n_iptr->flags.bits & INS_FLAG_RETADDR)
1411 inline_add_blocknr_reference(iln, &(n_iptr->sx.s23.s2.retaddrnr));
1418 stack_javalocals_store(n_iptr, iln->javalocals);
1424 static void inline_rewrite_method(inline_node *iln)
1428 instruction *o_iptr;
1429 instruction *n_iptr;
1430 inline_node *nextcall;
1432 inline_block_map *bm;
1437 char indent[100]; /* XXX debug */
1443 resultjd = iln->ctx->resultjd;
1447 nextcall = iln->children;
1450 for (i=0; i<iln->depth; ++i)
1453 iln->indent = indent;
1455 DOLOG( printf("%srewriting: ", indent); method_println(iln->m);
1456 printf("%s(passthrough: %d+%d)\n",
1457 indent, iln->n_passthroughcount - iln->n_selfpassthroughcount,
1458 iln->n_passthroughcount); );
1460 /* set memory cursors */
1462 iln->inlined_iinstr_cursor = iln->inlined_iinstr;
1463 iln->inlined_basicblocks_cursor = iln->inlined_basicblocks;
1465 /* allocate temporary buffers */
1467 iln->javalocals = DMNEW(s4, iln->jd->maxlocals);
1469 /* loop over basic blocks */
1471 o_bptr = iln->jd->basicblocks;
1472 for (; o_bptr; o_bptr = o_bptr->next) {
1474 if (o_bptr->flags < BBREACHED) {
1476 /* ignore the dummy end block */
1478 if (o_bptr->icount == 0 && o_bptr->next == NULL) {
1479 /* enter the following block as translation, for exception handler ranges */
1480 inline_block_translation(iln, o_bptr, iln->inlined_basicblocks_cursor);
1485 printf("%s* skipping old L%03d (flags=%d, type=%d, oid=%d) of ",
1487 o_bptr->nr, o_bptr->flags, o_bptr->type,
1489 method_println(iln->m);
1492 n_bptr = create_body_block(iln, o_bptr, iln->varmap);
1494 /* enter it in the blockmap */
1496 inline_block_translation(iln, o_bptr, n_bptr);
1498 close_body_block(iln, n_bptr, o_bptr, iln->varmap, 0, -1);
1502 len = o_bptr->icount;
1503 o_iptr = o_bptr->iinstr;
1506 printf("%s* rewriting old L%03d (flags=%d, type=%d, oid=%d) of ",
1508 o_bptr->nr, o_bptr->flags, o_bptr->type,
1510 method_println(iln->m);
1511 show_basicblock(iln->jd, o_bptr, SHOW_STACK);
1514 if (iln->blockbefore || o_bptr != iln->jd->basicblocks) {
1515 /* create an inlined clone of this block */
1517 n_bptr = create_body_block(iln, o_bptr, iln->varmap);
1520 /* enter it in the blockmap */
1522 inline_block_translation(iln, o_bptr, n_bptr);
1524 /* initialize the javalocals */
1526 MCOPY(iln->javalocals, n_bptr->javalocals, s4, iln->jd->maxlocals);
1531 /* continue caller block */
1533 n_bptr = iln->inlined_basicblocks_cursor - 1;
1534 icount = n_bptr->icount;
1536 /* translate the javalocals */
1538 jl = translate_javalocals(iln, o_bptr->javalocals);
1539 iln->inline_start_instruction->sx.s23.s3.inlineinfo->javalocals_start = jl;
1541 MCOPY(iln->javalocals, jl, s4, iln->jd->maxlocals);
1544 /* iterate over the ICMDs of this block */
1549 while (--len >= 0) {
1551 DOLOG( fputs(indent, stdout); show_icmd(iln->jd, o_iptr, false, SHOW_STACK);
1554 /* handle calls that will be inlined */
1556 if (nextcall && o_iptr == nextcall->callerins) {
1558 /* write the inlining prolog */
1560 (void) emit_inlining_prolog(iln, nextcall, o_iptr, iln->varmap);
1561 icount += nextcall->prolog_instructioncount;
1563 /* end current block, or glue blocks together */
1565 n_bptr->icount = icount;
1567 if (nextcall->blockbefore) {
1568 close_prolog_block(iln, n_bptr, nextcall);
1574 /* check if the result is a local variable */
1576 if (nextcall->m->parseddesc->returntype.type != TYPE_VOID
1577 && o_iptr->dst.varindex < iln->jd->localcount)
1579 nextcall->n_resultlocal = iln->varmap[o_iptr->dst.varindex];
1582 nextcall->n_resultlocal = -1;
1584 /* set memory pointers in the callee */
1586 nextcall->inlined_iinstr = iln->inlined_iinstr_cursor;
1587 nextcall->inlined_basicblocks = iln->inlined_basicblocks_cursor;
1591 DOLOG( printf("%sentering inline ", indent);
1592 show_icmd(origjd, o_iptr, false, SHOW_STACK); printf("\n") );
1594 inline_rewrite_method(nextcall);
1596 DOLOG( printf("%sleaving inline ", indent);
1597 show_icmd(origjd, o_iptr, false, SHOW_STACK); printf("\n") );
1599 /* update memory cursors */
1601 assert(nextcall->inlined_iinstr_cursor
1602 <= iln->inlined_iinstr_cursor + nextcall->cumul_instructioncount);
1603 assert(nextcall->inlined_basicblocks_cursor
1604 == iln->inlined_basicblocks_cursor + nextcall->cumul_basicblockcount);
1605 iln->inlined_iinstr_cursor = nextcall->inlined_iinstr_cursor;
1606 iln->inlined_basicblocks_cursor = nextcall->inlined_basicblocks_cursor;
1608 /* start new block, or glue blocks together */
1610 if (nextcall->blockafter) {
1611 n_bptr = create_epilog_block(iln, nextcall, iln->varmap);
1615 n_bptr = iln->inlined_basicblocks_cursor - 1;
1616 icount = n_bptr->icount;
1620 /* emit inlining epilog */
1622 emit_inlining_epilog(iln, nextcall, o_iptr);
1623 icount += nextcall->epilog_instructioncount;
1625 /* proceed to next call */
1627 nextcall = nextcall->next;
1630 n_iptr = (iln->inlined_iinstr_cursor++);
1631 assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1633 switch (o_iptr->opc) {
1635 if (iln->depth == 0)
1644 if (iln->depth == 0)
1647 retidx = iln->varmap[o_iptr->s1.varindex];
1648 if (iln->n_resultlocal != -1) {
1649 /* store result in a local variable */
1651 DOLOG( printf("USING RESULTLOCAL %d\n", iln->n_resultlocal); );
1652 /* This relies on the same sequence of types for */
1653 /* ?STORE and ?RETURN opcodes. */
1654 n_iptr->opc = ICMD_ISTORE + (o_iptr->opc - ICMD_IRETURN);
1655 n_iptr->s1.varindex = retidx;
1656 n_iptr->dst.varindex = iln->n_resultlocal;
1657 n_iptr->sx.s23.s3.javaindex = UNUSED; /* XXX set real javaindex? */
1662 n_iptr = (iln->inlined_iinstr_cursor++);
1663 assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1666 else if ((retidx < resultjd->localcount && iln->blockafter)
1667 || !iln->blockafter) /* XXX do we really always need the MOVE? */
1669 /* local must not become outvar, insert a MOVE */
1671 n_iptr->opc = ICMD_MOVE;
1672 n_iptr->s1.varindex = retidx;
1673 retidx = inline_new_temp_variable(resultjd,
1674 resultjd->var[retidx].type);
1675 n_iptr->dst.varindex = retidx;
1677 n_iptr = (iln->inlined_iinstr_cursor++);
1678 assert((n_iptr - iln->inlined_iinstr) < iln->cumul_instructioncount);
1682 if (iln->blockafter) {
1683 n_iptr->opc = ICMD_GOTO;
1684 n_iptr->dst.block = INLINE_RETURN_REFERENCE(iln);
1685 inline_add_block_reference(iln, &(n_iptr->dst.block));
1688 n_iptr->opc = ICMD_NOP;
1692 if (o_bptr->next == NULL || (o_bptr->next->icount==0 && o_bptr->next->next == NULL)) {
1693 n_iptr->opc = ICMD_NOP;
1702 inline_clone_instruction(iln, resultjd, iln->jd, iln->varmap, o_iptr, n_iptr);
1705 DOLOG( fputs(indent, stdout); show_icmd(resultjd, n_iptr, false, SHOW_STACK);
1714 /* end of basic block */
1716 if (iln->blockafter || (o_bptr->next && o_bptr->next->next)) {
1717 close_body_block(iln, n_bptr, o_bptr, iln->varmap, retcount, retidx);
1718 n_bptr->icount = icount;
1720 DOLOG( printf("closed body block:\n");
1721 show_basicblock(resultjd, n_bptr, SHOW_STACK); );
1724 n_bptr->icount = icount;
1725 assert(iln->parent);
1726 if (retidx != UNUSED)
1727 iln->parent->varmap[iln->callerins->dst.varindex] = retidx;
1731 bm = iln->ctx->blockmap;
1732 for (i=0; i<iln->ctx->blockmap_index; ++i, ++bm) {
1733 assert(bm->iln && bm->o_block && bm->n_block);
1735 inline_resolve_block_refs(&(iln->refs), bm->o_block, bm->n_block, false);
1738 #if !defined(NDEBUG)
1740 inline_target_ref *ref;
1743 if (!iln->depth || ref->isnumber || *(ref->ref.block) != INLINE_RETURN_REFERENCE(iln)) {
1744 DOLOG( printf("XXX REMAINING REF at depth %d: %p\n", iln->depth,
1745 (void*)*(ref->ref.block)) );
1755 static exception_entry * inline_exception_tables(inline_node *iln,
1756 exception_entry *n_extable,
1757 exception_entry **prevextable)
1761 exception_entry *et;
1765 assert(prevextable);
1767 child = iln->children;
1770 n_extable = inline_exception_tables(child, n_extable, prevextable);
1771 child = child->next;
1772 } while (child != iln->children);
1775 et = iln->jd->exceptiontable;
1776 for (; et != NULL; et = et->down) {
1778 MZERO(n_extable, exception_entry, 1);
1779 n_extable->start = inline_map_block(iln, et->start , iln);
1780 n_extable->end = inline_map_block(iln, et->end , iln);
1781 n_extable->handler = inline_map_block(iln, et->handler, iln);
1782 n_extable->catchtype = et->catchtype;
1785 (*prevextable)->down = n_extable;
1787 *prevextable = n_extable;
1792 if (iln->handler_monitorexit) {
1793 exception_entry **activehandlers;
1795 MZERO(n_extable, exception_entry, 1);
1796 n_extable->start = iln->inlined_basicblocks;
1797 n_extable->end = iln->inlined_basicblocks_cursor;
1798 n_extable->handler = iln->handler_monitorexit;
1799 n_extable->catchtype.any = NULL; /* finally */
1802 (*prevextable)->down = n_extable;
1804 *prevextable = n_extable;
1808 /* We have to protect the created handler with the same handlers */
1809 /* that protect the method body itself. */
1811 for (scope = iln; scope->parent != NULL; scope = scope->parent) {
1813 activehandlers = scope->o_handlers;
1814 assert(activehandlers);
1816 while (*activehandlers) {
1818 assert(scope->parent);
1820 MZERO(n_extable, exception_entry, 1);
1821 n_extable->start = iln->handler_monitorexit;
1822 n_extable->end = iln->handler_monitorexit + 1; /* XXX ok in this case? */
1823 n_extable->handler = inline_map_block(scope->parent,
1824 (*activehandlers)->handler,
1826 n_extable->catchtype = (*activehandlers)->catchtype;
1829 (*prevextable)->down = n_extable;
1831 *prevextable = n_extable;
1843 static void inline_locals(inline_node *iln)
1849 iln->varmap = create_variable_map(iln);
1851 child = iln->children;
1854 inline_locals(child);
1855 child = child->next;
1856 } while (child != iln->children);
1859 if (iln->regdata->memuse > iln->ctx->resultjd->rd->memuse)
1860 iln->ctx->resultjd->rd->memuse = iln->regdata->memuse;
1861 if (iln->regdata->argintreguse > iln->ctx->resultjd->rd->argintreguse)
1862 iln->ctx->resultjd->rd->argintreguse = iln->regdata->argintreguse;
1863 if (iln->regdata->argfltreguse > iln->ctx->resultjd->rd->argfltreguse)
1864 iln->ctx->resultjd->rd->argfltreguse = iln->regdata->argfltreguse;
1868 static void inline_interface_variables(inline_node *iln)
1875 resultjd = iln->ctx->resultjd;
1877 resultjd->interface_map = DMNEW(interface_info, 5*iln->ctx->maxinoutdepth);
1878 for (i=0; i<5*iln->ctx->maxinoutdepth; ++i)
1879 resultjd->interface_map[i].flags = UNUSED;
1881 for (bptr = resultjd->basicblocks; bptr != NULL; bptr = bptr->next) {
1882 assert(bptr->indepth <= iln->ctx->maxinoutdepth);
1883 assert(bptr->outdepth <= iln->ctx->maxinoutdepth);
1885 for (i=0; i<bptr->indepth; ++i) {
1886 v = &(resultjd->var[bptr->invars[i]]);
1888 if (v->type == TYPE_RET)
1889 v->flags |= PREALLOC;
1891 v->flags &= ~PREALLOC;
1892 v->flags &= ~INMEMORY;
1893 assert(bptr->invars[i] >= resultjd->localcount);
1895 if (resultjd->interface_map[5*i + v->type].flags == UNUSED) {
1896 resultjd->interface_map[5*i + v->type].flags = v->flags;
1899 resultjd->interface_map[5*i + v->type].flags |= v->flags;
1903 for (i=0; i<bptr->outdepth; ++i) {
1904 v = &(resultjd->var[bptr->outvars[i]]);
1906 if (v->type == TYPE_RET)
1907 v->flags |= PREALLOC;
1909 v->flags &= ~PREALLOC;
1910 v->flags &= ~INMEMORY;
1911 assert(bptr->outvars[i] >= resultjd->localcount);
1913 if (resultjd->interface_map[5*i + v->type].flags == UNUSED) {
1914 resultjd->interface_map[5*i + v->type].flags = v->flags;
1917 resultjd->interface_map[5*i + v->type].flags |= v->flags;
1924 static void inline_write_exception_handlers(inline_node *master, inline_node *iln)
1929 builtintable_entry *bte;
1934 child = iln->children;
1937 inline_write_exception_handlers(master, child);
1938 child = child->next;
1939 } while (child != iln->children);
1942 if (iln->synchronize) {
1943 /* create the monitorexit handler */
1944 n_bptr = create_block(master, iln, iln,
1945 iln->n_passthroughcount + 1);
1946 n_bptr->type = BBTYPE_EXH;
1947 n_bptr->flags = BBFINISHED;
1949 exvar = inline_new_variable(master->ctx->resultjd, TYPE_ADR, 0);
1950 n_bptr->invars[iln->n_passthroughcount] = exvar;
1952 iln->handler_monitorexit = n_bptr;
1954 /* ACONST / ALOAD */
1956 n_ins = master->inlined_iinstr_cursor++;
1957 if (iln->m->flags & ACC_STATIC) {
1958 n_ins->opc = ICMD_ACONST;
1959 n_ins->sx.val.c.cls = iln->m->clazz;
1960 n_ins->flags.bits = INS_FLAG_CLASS;
1963 n_ins->opc = ICMD_ALOAD;
1964 n_ins->s1.varindex = iln->synclocal;
1965 assert(n_ins->s1.varindex != UNUSED);
1967 /* XXX could be PREALLOCed for builtin call */
1968 syncvar = inline_new_variable(master->ctx->resultjd, TYPE_ADR, 0);
1969 n_ins->dst.varindex = syncvar;
1974 bte = builtintable_get_internal(LOCK_monitor_exit);
1976 n_ins = master->inlined_iinstr_cursor++;
1977 n_ins->opc = ICMD_BUILTIN;
1978 n_ins->s1.argcount = 1 + iln->n_passthroughcount + 1;
1979 n_ins->sx.s23.s2.args = DMNEW(s4, n_ins->s1.argcount);
1980 n_ins->sx.s23.s2.args[0] = syncvar;
1981 for (i=0; i < iln->n_passthroughcount + 1; ++i) {
1982 n_ins->sx.s23.s2.args[1 + i] = n_bptr->invars[i];
1984 n_ins->sx.s23.s3.bte = bte;
1989 n_ins = master->inlined_iinstr_cursor++;
1990 n_ins->opc = ICMD_ATHROW;
1991 n_ins->flags.bits = 0;
1992 n_ins->s1.varindex = exvar;
1995 /* close basic block */
1997 close_block(iln, iln, n_bptr, iln->n_passthroughcount);
2003 /* second pass driver *********************************************************/
2005 static bool inline_transform(inline_node *iln, jitdata *jd)
2010 exception_entry *n_ext;
2011 exception_entry *prevext;
2015 #if defined(INLINE_VERIFY_RESULT)
2016 static int debug_verify_inlined_code = 1;
2018 #if defined(ENABLE_INLINING_DEBUG) || !defined(NDEBUG)
2019 static int debug_counter = 0;
2022 DOLOG( dump_inline_tree(iln, 0); );
2026 n_ins = DMNEW(instruction, iln->cumul_instructioncount);
2027 MZERO(n_ins, instruction, iln->cumul_instructioncount);
2028 iln->inlined_iinstr = n_ins;
2030 n_bb = DMNEW(basicblock, iln->cumul_basicblockcount);
2031 MZERO(n_bb, basicblock, iln->cumul_basicblockcount);
2032 iln->inlined_basicblocks = n_bb;
2034 iln->ctx->blockmap = DMNEW(inline_block_map, iln->cumul_blockmapcount);
2036 n_jd = jit_jitdata_new(iln->m);
2037 n_jd->flags = jd->flags;
2038 n_jd->code->optlevel = jd->code->optlevel;
2039 iln->ctx->resultjd = n_jd;
2043 /* create the local_map */
2045 n_jd->local_map = DMNEW(s4, 5*iln->cumul_maxlocals);
2046 for (i=0; i<5*iln->cumul_maxlocals; ++i)
2047 n_jd->local_map[i] = UNUSED;
2049 /* create / coalesce local variables */
2057 n_jd->localcount = n_jd->vartop;
2059 /* extra variables for verification (debugging) */
2061 #if defined(INLINE_VERIFY_RESULT)
2062 if (debug_verify_inlined_code) {
2063 n_jd->vartop += VERIFIER_EXTRA_LOCALS + VERIFIER_EXTRA_VARS + 100 /* XXX m->maxstack */;
2064 if (n_jd->vartop > n_jd->varcount) {
2066 n_jd->var = DMREALLOC(n_jd->var, varinfo, n_jd->varcount, n_jd->vartop);
2067 n_jd->varcount = n_jd->vartop;
2070 #endif /* defined(INLINE_VERIFY_RESULT) */
2072 /* write inlined code */
2074 inline_rewrite_method(iln);
2076 /* create exception handlers */
2078 inline_write_exception_handlers(iln, iln);
2080 /* write the dummy end block */
2082 n_bptr = create_block(iln, iln, iln, 0);
2083 n_bptr->flags = BBUNDEF;
2084 n_bptr->type = BBTYPE_STD;
2086 /* store created code in jitdata */
2088 n_jd->basicblocks = iln->inlined_basicblocks;
2089 n_jd->instructioncount = iln->cumul_instructioncount;
2090 n_jd->instructions = iln->inlined_iinstr;
2092 /* link the basic blocks (dummy end block is not counted) */
2094 n_jd->basicblockcount = (iln->inlined_basicblocks_cursor - iln->inlined_basicblocks) - 1;
2095 for (i=0; i<n_jd->basicblockcount + 1; ++i)
2096 n_jd->basicblocks[i].next = &(n_jd->basicblocks[i+1]);
2098 n_jd->basicblocks[i-1].next = NULL;
2100 /* check basicblock numbers */
2102 #if !defined(NDEBUG)
2103 jit_check_basicblock_numbers(n_jd);
2106 /* create the exception table */
2108 if (iln->cumul_exceptiontablelength) {
2109 exception_entry *tableend;
2111 n_ext = DMNEW(exception_entry, iln->cumul_exceptiontablelength);
2113 tableend = inline_exception_tables(iln, n_ext, &prevext);
2114 assert(tableend == n_ext + iln->cumul_exceptiontablelength);
2116 prevext->down = NULL;
2118 n_jd->exceptiontablelength = iln->cumul_exceptiontablelength;
2119 n_jd->exceptiontable = n_ext;
2125 /*******************************************************************************/
2128 memcpy(n_cd, jd->cd, sizeof(codegendata));
2130 n_cd->method = NULL; /* XXX */
2131 n_jd->maxlocals = iln->cumul_maxlocals;
2132 n_jd->maxinterfaces = iln->ctx->maxinoutdepth;
2134 inline_post_process(n_jd);
2136 inline_interface_variables(iln);
2138 /* for debugging, verify the inlined result */
2140 #if defined(INLINE_VERIFY_RESULT)
2141 if (debug_verify_inlined_code) {
2142 debug_verify_inlined_code = 0;
2143 DOLOG( printf("VERIFYING INLINED RESULT...\n"); fflush(stdout); );
2144 if (!typecheck(n_jd)) {
2145 exceptions_clear_exception();
2146 DOLOG( printf("XXX INLINED RESULT DID NOT PASS VERIFIER XXX\n") );
2150 DOLOG( printf("VERIFICATION PASSED.\n") );
2152 debug_verify_inlined_code = 1;
2154 #endif /* defined(INLINE_VERIFY_RESULT) */
2156 /* we need bigger free memory stacks (XXX these should not be allocated in reg_setup) */
2158 n_jd->rd->freemem = DMNEW(s4, iln->ctx->maxinoutdepth + 1000) /* XXX max vars/block */;
2159 #if defined(HAS_4BYTE_STACKSLOT)
2160 n_jd->rd->freemem_2 = DMNEW(s4, iln->ctx->maxinoutdepth + 1000) /* XXX max vars/block */;
2163 #if defined(ENABLE_INLINING_DEBUG) || !defined(NDEBUG)
2164 if ( (n_jd->instructioncount >= opt_InlineMinSize)
2165 && (n_jd->instructioncount <= opt_InlineMaxSize))
2167 if (debug_counter < opt_InlineCount)
2168 #endif /* defined(ENABLE_INLINING_DEBUG) || !defined(NDEBUG) */
2170 /* install the inlined result */
2172 *jd->code = *n_jd->code;
2173 n_jd->code = jd->code;
2176 /* statistics and logging */
2178 #if !defined(NDEBUG)
2179 inline_stat_roots++;
2182 printf("==== %d.INLINE ==================================================================\n",
2184 printf("\ninline tree:\n");
2185 dump_inline_tree(iln, 0);
2186 n_jd->flags |= JITDATA_FLAG_SHOWINTERMEDIATE | JITDATA_FLAG_SHOWDISASSEMBLE;
2187 /* debug_dump_inlined_code(iln, n_method, n_cd, n_rd); */
2188 printf("-------- DONE -----------------------------------------------------------\n");
2194 #if defined(ENABLE_INLINING_DEBUG) || !defined(NDEBUG)
2202 /******************************************************************************/
2203 /* FIRST PASS: build inlining tree */
2204 /******************************************************************************/
2207 /* inline_pre_parse_heuristics *************************************************
2209 Perform heuristic checks whether a call site should be inlined.
2210 These checks are evaluated before the callee has been parsed and analysed.
2213 caller...........inlining node of the caller
2214 callee...........the called method
2215 site.............information on the call site
2218 true........consider for inlining
2219 false.......don't inline
2221 *******************************************************************************/
2223 static bool inline_pre_parse_heuristics(const inline_node *caller,
2224 const methodinfo *callee,
2227 #if defined(INLINE_MAX_DEPTH)
2228 if (caller->depth >= INLINE_MAX_DEPTH)
2236 /* inline_post_parse_heuristics ************************************************
2238 Perform heuristic checks whether a call site should be inlined.
2239 These checks are evaluated after the callee has been parsed and analysed.
2242 caller...........inlining node of the caller (const)
2243 callee...........the called method (const)
2246 true........consider for inlining
2247 false.......don't inline
2249 *******************************************************************************/
2251 static bool inline_post_parse_heuristics(const inline_node *caller,
2252 const inline_node *callee)
2258 /* inline_afterwards_heuristics ************************************************
2260 Perform heuristic checks whether a call site should be inlined.
2261 These checks are evaluated after the inlining plan for the callee has
2265 caller...........inlining node of the caller (const)
2266 callee...........the called method (const)
2269 true........consider for inlining
2270 false.......don't inline
2272 *******************************************************************************/
2274 static bool inline_afterwards_heuristics(const inline_node *caller,
2275 const inline_node *callee)
2277 inline_node *cumulator;
2279 #if defined(INLINE_DEPTH_FIRST)
2282 cumulator = caller->ctx->master;
2286 #if defined(INLINE_MAX_BLOCK_EXPANSION)
2287 || (cumulator->cumul_basicblockcount + callee->cumul_basicblockcount
2288 > INLINE_MAX_BLOCK_EXPANSION*caller->ctx->master->jd->basicblockcount)
2290 #if defined(INLINE_MAX_ICMD_EXPANSION)
2291 || (cumulator->cumul_instructioncount + callee->cumul_instructioncount
2292 > INLINE_MAX_ICMD_EXPANSION*caller->ctx->master->jd->instructioncount)
2303 /* inline_is_monomorphic *******************************************************
2305 Check if the given call site can be proven to be monomorphic.
2308 callee...........the called method
2309 call.............the invocation instruction
2312 site->speculative.....flags whether the inlining is speculative
2313 (only defined if return value is true)
2316 true if the call site is (currently) monomorphic,
2317 false if not or unknown
2319 *******************************************************************************/
2321 static bool inline_is_monomorphic(const methodinfo *callee,
2322 const instruction *call,
2325 if ((callee->flags & (ACC_STATIC | ACC_FINAL | ACC_PRIVATE)
2326 || call->opc == ICMD_INVOKESPECIAL))
2328 site->speculative = false;
2332 /* XXX search single implementation for abstract monomorphics */
2334 if ((callee->flags & (ACC_METHOD_MONOMORPHIC | ACC_METHOD_IMPLEMENTED
2336 == (ACC_METHOD_MONOMORPHIC | ACC_METHOD_IMPLEMENTED))
2339 DOLOG( printf("SPECULATIVE INLINE: "); method_println((methodinfo*)callee); );
2340 site->speculative = true;
2346 /* possibly polymorphic call site */
2352 /* inline_can_inline ***********************************************************
2354 Check if inlining of the given call site is possible.
2357 caller...........inlining node of the caller
2358 callee...........the called method
2359 call.............the invocation instruction
2362 site->speculative.....flags whether the inlining is speculative
2363 (only defined if return value is true)
2366 true if inlining is possible, false if not
2368 *******************************************************************************/
2370 static bool inline_can_inline(const inline_node *caller,
2371 const methodinfo *callee,
2372 const instruction *call,
2375 const inline_node *active;
2377 /* cannot inline native methods */
2379 if (callee->flags & ACC_NATIVE)
2382 /* cannot inline possibly polymorphic calls */
2384 if (!inline_is_monomorphic(callee, call, site))
2387 /* cannot inline recursive calls */
2389 for (active = caller; active; active = active->parent) {
2390 if (callee == active->m) {
2391 DOLOG( printf("RECURSIVE!\n") );
2396 /* inlining is possible */
2402 /* inline_create_callee_node ***************************************************
2404 Create an inlining node for the given callee.
2407 caller...........inlining node of the caller (const)
2408 callee...........the called method
2411 the new inlining node
2413 *******************************************************************************/
2415 static inline_node * inline_create_callee_node(const inline_node *caller,
2418 inline_node *cn; /* the callee inline_node */
2420 cn = DNEW(inline_node);
2421 MZERO(cn, inline_node, 1);
2423 cn->depth = caller->depth + 1;
2424 cn->ctx = caller->ctx;
2426 cn->synchronize = (callee->flags & ACC_SYNCHRONIZED);
2427 cn->isstatic = (callee->flags & ACC_STATIC);
2433 /* inline_set_callee_properties ************************************************
2435 Set properties of the inlined call site.
2438 caller...........inlining node of the caller (const)
2439 cn...............the called method
2440 site.............info about the call site (const)
2443 *cn..............has the properties set
2445 *******************************************************************************/
2447 static void inline_set_callee_properties(const inline_node *caller,
2449 const inline_site *site)
2455 /* set info about the call site */
2457 cn->callerblock = site->bptr;
2458 cn->callerins = site->iptr;
2459 cn->callerpc = site->pc;
2460 cn->o_handlers = site->handlers;
2461 cn->n_handlercount = caller->n_handlercount + site->nhandlers;
2463 /* determine if we need basic block boundaries before/after */
2465 cn->blockbefore = false;
2466 cn->blockafter = false;
2468 if (cn->jd->branchtoentry)
2469 cn->blockbefore = true;
2471 if (cn->jd->branchtoend)
2472 cn->blockafter = true;
2474 if (cn->jd->returncount > 1)
2475 cn->blockafter = true;
2477 /* XXX make safer and reusable (maybe store last real block) */
2478 for (bptr = cn->jd->basicblocks; bptr && bptr->next && bptr->next->next; bptr = bptr->next)
2481 if (cn->jd->returnblock != bptr)
2482 cn->blockafter = true;
2484 /* info about the callee */
2486 cn->localsoffset = caller->localsoffset + caller->m->maxlocals;
2487 cn->prolog_instructioncount = cn->m->parseddesc->paramcount + 2;
2488 cn->epilog_instructioncount = 1; /* INLINE_END */
2489 cn->extra_instructioncount = 0;
2491 /* we need a CHECKNULL for instance methods, except for <init> */
2493 if (!cn->isstatic && cn->m->name != utf_init)
2494 cn->prolog_instructioncount += 1;
2496 /* deal with synchronized callees */
2498 if (cn->synchronize) {
2500 builtintable_entry *bte;
2502 /* we need basic block boundaries because of the handler */
2504 cn->blockbefore = true;
2505 cn->blockafter = true;
2507 /* for synchronized static methods */
2508 /* we need an ACONST, MONITORENTER in the prolog */
2509 /* and ACONST, MONITOREXIT in the epilog */
2511 /* for synchronized instance methods */
2512 /* we need an COPY, MONITORENTER in the prolog */
2513 /* and MONITOREXIT in the epilog */
2516 cn->prolog_instructioncount += 2;
2517 cn->epilog_instructioncount += 2;
2520 cn->prolog_instructioncount += 2;
2521 cn->epilog_instructioncount += 1;
2522 cn->localsoffset += 1;
2525 /* and exception handler */
2526 /* ALOAD, builtin_monitorexit, ATHROW */
2528 cn->extra_instructioncount += 3;
2530 /* exception table entries */
2532 cn->extra_exceptiontablelength = 1 + cn->n_handlercount;
2534 /* add exception handler block */
2536 cn->cumul_basicblockcount_root++;
2538 /* we must call the builtins */
2540 bte = builtintable_get_internal(LOCK_monitor_enter);
2542 if (md->memuse > cn->regdata->memuse)
2543 cn->regdata->memuse = md->memuse;
2544 if (md->argintreguse > cn->regdata->argintreguse)
2545 cn->regdata->argintreguse = md->argintreguse;
2547 bte = builtintable_get_internal(LOCK_monitor_exit);
2549 if (md->memuse > cn->regdata->memuse)
2550 cn->regdata->memuse = md->memuse;
2551 if (md->argintreguse > cn->regdata->argintreguse)
2552 cn->regdata->argintreguse = md->argintreguse;
2555 /* determine pass-through variables */
2557 i = site->iptr->s1.argcount - cn->m->parseddesc->paramcount; /* max # of pass-though vars */
2559 cn->n_passthroughvars = DMNEW(s4, i);
2561 for (argi = site->iptr->s1.argcount - 1; argi >= cn->m->parseddesc->paramcount; --argi) {
2562 s4 idx = site->iptr->sx.s23.s2.args[argi];
2563 if (idx >= caller->jd->localcount) {
2564 cn->n_passthroughvars[j] = idx;
2568 DOLOG( printf("PASSING THROUGH LOCAL VARIABLE %d\n", idx); );
2572 cn->n_selfpassthroughcount = j;
2573 cn->n_passthroughcount = caller->n_passthroughcount + cn->n_selfpassthroughcount;
2577 /* inline_cumulate_counters ****************************************************
2579 Cumulate counters after a node has been decided to become inlined.
2582 caller...........inlining node of the caller
2583 callee...........inlining node of the callee (const)
2586 *caller..........gets cumulated values added
2588 *******************************************************************************/
2590 static void inline_cumulate_counters(inline_node *caller,
2591 const inline_node *cn)
2593 caller->cumul_instructioncount += cn->prolog_instructioncount;
2594 caller->cumul_instructioncount += cn->epilog_instructioncount;
2595 caller->cumul_instructioncount += cn->extra_instructioncount;
2596 caller->cumul_instructioncount += cn->cumul_instructioncount - 1 /*invoke*/;
2598 caller->cumul_basicblockcount += cn->cumul_basicblockcount;
2599 caller->cumul_basicblockcount_root += cn->cumul_basicblockcount_root;
2600 caller->cumul_blockmapcount += cn->cumul_blockmapcount;
2601 caller->cumul_exceptiontablelength += cn->cumul_exceptiontablelength;
2602 caller->cumul_exceptiontablelength += cn->extra_exceptiontablelength;
2604 if (cn->cumul_maxlocals > caller->cumul_maxlocals)
2605 caller->cumul_maxlocals = cn->cumul_maxlocals;
2607 /* XXX extra block after inlined call */
2608 if (cn->blockafter) {
2609 caller->cumul_basicblockcount += 1;
2610 caller->cumul_blockmapcount += 1;
2615 /* inline_analyse_callee *******************************************************
2617 Analyse an inlining candidate callee.
2620 caller...........inlining node of the caller
2621 callee...........the called method
2622 site.............info about the call site
2625 site->inlined....true if the callee has been selected for inlining
2628 the inline node of the callee, or
2629 NULL if an error has occurred (don't use the inlining plan in this case)
2631 *******************************************************************************/
2633 static inline_node * inline_analyse_callee(inline_node *caller,
2637 inline_node *cn; /* the callee inline_node */
2639 /* create an inline tree node */
2641 cn = inline_create_callee_node(caller, callee);
2643 /* get the intermediate representation of the callee */
2645 if (!inline_jit_compile(cn))
2648 /* evaluate heuristics after parsing the callee */
2650 if (!inline_post_parse_heuristics(caller, cn))
2653 /* the call site will be inlined */
2655 site->inlined = true;
2657 /* set info about the call site */
2659 inline_set_callee_properties(caller, cn, site);
2661 /* insert the node into the inline tree */
2663 inline_insert_inline_node(caller, cn);
2665 /* analyse recursively */
2667 if (!inline_analyse_code(cn))
2670 if (!inline_afterwards_heuristics(caller, cn)) {
2671 #if defined(INLINE_CANCEL_ON_THRESHOLD)
2674 inline_remove_inline_node(caller, cn);
2675 caller->ctx->stopped = true;
2676 site->inlined = false;
2681 /* cumulate counters */
2683 #if defined(INLINE_DEPTH_FIRST)
2684 inline_cumulate_counters(caller, cn);
2687 #if defined(INLINE_BREADTH_FIRST)
2689 inline_cumulate_counters(caller, cn);
2690 caller = caller->parent;
2698 /* inline_process_candidate ****************************************************
2700 Process a selected inlining candidate.
2703 cand.............the candidate
2706 true........everything ok
2707 false.......an error has occurred, don't use the plan
2709 *******************************************************************************/
2711 static bool inline_process_candidate(inline_candidate *cand)
2715 cn = inline_analyse_callee(cand->caller,
2722 if (!cand->site.inlined)
2725 /* store assumptions */
2727 if (cand->site.speculative)
2728 method_add_assumption_monomorphic(cand->callee, cand->caller->ctx->master->m);
2734 /* inline_analyse_code *********************************************************
2736 Analyse the intermediate code of the given inlining node.
2739 iln..............the inlining node
2742 *iln.............the inlining plan
2745 true........everything ok
2746 false.......an error has occurred, don't use the plan
2748 *******************************************************************************/
2750 static bool inline_analyse_code(inline_node *iln)
2757 exception_entry **handlers;
2758 exception_entry *ex;
2769 /* initialize cumulative counters */
2771 iln->cumul_maxlocals = iln->localsoffset + m->maxlocals;
2772 iln->cumul_exceptiontablelength += mjd->exceptiontablelength;
2774 /* iterate over basic blocks */
2778 for (bptr = mjd->basicblocks; bptr; bptr = bptr->next) {
2780 /* count the block */
2781 /* ignore dummy end blocks (but count them for the blockmap) */
2783 iln->cumul_blockmapcount++;
2784 if ((bptr != mjd->basicblocks || iln->blockbefore)
2786 (bptr->icount > 0 || bptr->next != NULL))
2787 iln->cumul_basicblockcount++;
2789 /* skip dead code */
2791 if (bptr->flags < BBREACHED)
2794 /* allocate the buffer of active exception handlers */
2795 /* XXX this wastes some memory, but probably it does not matter */
2797 handlers = DMNEW(exception_entry*, mjd->exceptiontablelength + 1);
2799 /* determine the active exception handlers for this block */
2800 /* XXX maybe the handlers of a block should be part of our IR */
2801 /* XXX this should share code with the type checkers */
2803 for (ex = mjd->exceptiontable; ex ; ex = ex->down) {
2804 if ((ex->start->nr <= bptr->nr) && (ex->end->nr > bptr->nr)) {
2805 handlers[nhandlers++] = ex;
2808 handlers[nhandlers] = NULL;
2811 iptr = bptr->iinstr;
2814 iln->cumul_instructioncount += len;
2816 /* iterate over the instructions of the block */
2818 for (; --len >= 0; ++iptr) {
2820 switch (iptr->opc) {
2821 case ICMD_INVOKEVIRTUAL:
2822 case ICMD_INVOKESPECIAL:
2823 case ICMD_INVOKESTATIC:
2824 case ICMD_INVOKEINTERFACE:
2826 if (!INSTRUCTION_IS_UNRESOLVED(iptr) && !iln->ctx->stopped) {
2827 callee = iptr->sx.s23.s3.fmiref->p.method;
2829 if (inline_can_inline(iln, callee, iptr, &site)) {
2830 site.inlined = false;
2833 site.pc = blockendpc - len - 1;
2834 site.handlers = handlers;
2835 site.nhandlers = nhandlers;
2837 if (inline_pre_parse_heuristics(iln, callee, &site)) {
2838 #if defined(INLINE_KNAPSACK) || defined(INLINE_BREADTH_FIRST)
2839 inline_add_candidate(iln->ctx, iln, callee, &site);
2841 inline_candidate cand;
2843 cand.callee = callee;
2846 if (!inline_process_candidate(&cand))
2860 /* extra ICMD_MOVE may be necessary */
2861 iln->cumul_instructioncount++;
2866 /* end of basic block */
2873 static void inline_cumulate_counters_recursive(inline_node *iln)
2877 child = iln->children;
2880 inline_cumulate_counters_recursive(child);
2881 inline_cumulate_counters(iln, child);
2882 child = child->next;
2883 } while (child != iln->children);
2888 /* inline_make_inlining_plan ***************************************************
2890 Make an inlining plan for the given root node
2893 iln..............the root node
2896 *iln.............the inlining plan
2899 true........everything ok
2900 false.......an error has occurred, don't use the plan
2902 *******************************************************************************/
2904 #if defined(INLINE_KNAPSACK)
2905 static bool inline_make_inlining_plan(inline_node *iln)
2907 inline_candidate *cand;
2908 #if defined(INLINE_COST_BUDGET)
2909 s4 budget = INLINE_COST_BUDGET;
2910 # define BUDGETMEMBER cost
2912 #if defined(INLINE_WEIGHT_BUDGET)
2913 double budget = INLINE_WEIGHT_BUDGET;
2914 # define BUDGETMEMBER weight
2917 inline_analyse_code(iln);
2919 DOLOG( printf("candidates in "); method_println(iln->m);
2920 inline_candidates_println(iln->ctx); );
2922 while ((cand = inline_pick_best_candidate(iln->ctx)) != NULL)
2924 if (cand->BUDGETMEMBER <= budget) {
2925 DOLOG( printf(" picking: "); inline_candidate_println(cand); );
2927 if (!inline_process_candidate(cand))
2930 #if !defined(INLINE_ADD_NEGATIVE_TO_BUDGET)
2931 if (cand->BUDGETMEMBER > 0)
2933 budget -= cand->BUDGETMEMBER;
2937 inline_cumulate_counters_recursive(iln);
2941 #endif /* defined(INLINE_KNAPSACK) */
2944 #if defined(INLINE_DEPTH_FIRST)
2945 static bool inline_make_inlining_plan(inline_node *iln)
2947 return inline_analyse_code(iln);
2949 #endif /* defined(INLINE_DEPTH_FIRST) */
2952 #if defined(INLINE_BREADTH_FIRST)
2953 static bool inline_make_inlining_plan(inline_node *iln)
2955 inline_candidate *cand;
2957 inline_analyse_code(iln);
2959 DOLOG( printf("candidates in "); method_println(iln->m);
2960 inline_candidates_println(iln->ctx); );
2962 while (!iln->ctx->stopped
2963 && (cand = inline_pick_best_candidate(iln->ctx)) != NULL)
2965 DOLOG( printf(" picking: "); inline_candidate_println(cand); );
2967 if (!inline_process_candidate(cand))
2973 #endif /* defined(INLINE_BREADTH_FIRST) */
2976 /* statistics *****************************************************************/
2978 #if defined(INLINE_STATISTICS)
2979 static void inline_gather_statistics_recursive(inline_node *iln)
2983 inline_stat_inlined_nodes++;
2985 if (iln->depth > inline_stat_max_depth)
2986 inline_stat_max_depth++;
2988 child = iln->children;
2991 inline_gather_statistics_recursive(child);
2992 child = child->next;
2993 } while (child != iln->children);
2996 #endif /* defined(INLINE_STATISTICS) */
2999 #if defined(INLINE_STATISTICS)
3000 static void inline_gather_statistics(inline_node *iln)
3002 inline_stat_roots_transformed++;
3004 inline_gather_statistics_recursive(iln);
3006 #endif /* defined(INLINE_STATISTICS) */
3009 /* post processing ************************************************************/
3011 #define POSTPROCESS_SRC(varindex) live[varindex]--
3012 #define POSTPROCESS_DST(varindex) live[varindex]++
3014 #define POSTPROCESS_SRCOP(s) POSTPROCESS_SRC(iptr->s.varindex)
3015 #define POSTPROCESS_DSTOP(d) POSTPROCESS_DST(iptr->d.varindex)
3017 #define MARKSAVED(varindex) jd->var[varindex].flags |= SAVEDVAR
3019 #define MARK_ALL_SAVED \
3021 for (i=0; i<jd->vartop; ++i) \
3026 static void inline_post_process(jitdata *jd)
3033 icmdtable_entry_t *icmdt;
3036 builtintable_entry *bte;
3038 /* Get required compiler data. */
3042 /* reset the SAVEDVAR flag of all variables */
3044 for (i=0; i<jd->vartop; ++i)
3045 jd->var[i].flags &= ~SAVEDVAR;
3047 /* allocate the life counters */
3049 live = DMNEW(s4, jd->vartop);
3050 MZERO(live, s4, jd->vartop);
3052 /* iterate over all basic blocks */
3054 for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
3055 if (bptr->flags < BBREACHED)
3058 /* make invars live */
3060 for (i=0; i<bptr->indepth; ++i)
3061 POSTPROCESS_DST(bptr->invars[i]);
3063 iptr = bptr->iinstr;
3064 iend = iptr + bptr->icount;
3066 for (; iptr < iend; ++iptr) {
3068 icmdt = &(icmd_table[iptr->opc]);
3070 switch (icmdt->dataflow) {
3072 POSTPROCESS_SRCOP(sx.s23.s3);
3074 POSTPROCESS_SRCOP(sx.s23.s2);
3076 POSTPROCESS_SRCOP(s1);
3078 if (icmdt->flags & ICMDTABLE_CALLS) {
3079 code_unflag_leafmethod(code);
3085 POSTPROCESS_SRCOP(sx.s23.s2);
3088 POSTPROCESS_SRCOP(s1);
3090 if (icmdt->flags & ICMDTABLE_CALLS) {
3091 code_unflag_leafmethod(code);
3095 POSTPROCESS_DSTOP(dst);
3099 for (i=0; i<iptr->s1.argcount; ++i) {
3100 POSTPROCESS_SRC(iptr->sx.s23.s2.args[i]);
3102 if (icmdt->flags & ICMDTABLE_CALLS) {
3103 code_unflag_leafmethod(code);
3106 POSTPROCESS_DSTOP(dst);
3110 INSTRUCTION_GET_METHODDESC(iptr, md);
3112 code_unflag_leafmethod(code);
3113 for (i=0; i<md->paramcount; ++i) {
3114 POSTPROCESS_SRC(iptr->sx.s23.s2.args[i]);
3116 for (; i<iptr->s1.argcount; ++i) {
3117 MARKSAVED(iptr->sx.s23.s2.args[i]);
3119 if (md->returntype.type != TYPE_VOID)
3120 POSTPROCESS_DSTOP(dst);
3124 bte = iptr->sx.s23.s3.bte;
3126 goto post_process_call;
3132 } /* end instruction loop */
3134 /* consume outvars */
3136 for (i=0; i<bptr->outdepth; ++i)
3137 POSTPROCESS_SRC(bptr->outvars[i]);
3139 #if !defined(NDEBUG)
3140 for (i=jd->localcount; i < jd->vartop; ++i)
3141 assert(live[i] == 0);
3144 } /* end basic block loop */
3148 /* inline_create_root_node *****************************************************
3150 Create the root node of the inlining tree.
3153 jd...............the current jitdata of the root method
3156 the root node of the inlining tree
3158 *******************************************************************************/
3160 static inline_node * inline_create_root_node(jitdata *jd)
3164 iln = DNEW(inline_node);
3165 MZERO(iln, inline_node, 1);
3169 iln->regdata = jd->rd;
3171 iln->blockbefore = true;
3172 iln->blockafter = true;
3174 iln->cumul_instructioncount = 0;
3175 iln->cumul_basicblockcount = 1 /* dummy end block */;
3177 /* create inlining context */
3179 iln->ctx = DNEW(inline_context);
3180 MZERO(iln->ctx, inline_context, 1);
3181 iln->ctx->master = iln;
3182 iln->ctx->next_debugnr = 1; /* XXX debug */
3188 /******************************************************************************/
3189 /* MAIN DRIVER FUNCTION */
3190 /******************************************************************************/
3192 bool inline_inline(jitdata *jd)
3196 DOLOG( printf("==== INLINE ==================================================================\n");
3197 show_method(jd, SHOW_STACK); );
3199 #if defined(INLINE_STATISTICS)
3200 inline_stat_roots++;
3203 iln = inline_create_root_node(jd);
3205 if (inline_make_inlining_plan(iln)) {
3207 /* add blocks to the root node */
3209 iln->cumul_basicblockcount += iln->cumul_basicblockcount_root;
3210 iln->cumul_blockmapcount += iln->cumul_basicblockcount_root;
3212 DOLOG( printf("==== INLINE TRANSFORM ========================================================\n"); );
3215 inline_transform(iln, jd);
3217 #if defined(INLINE_STATISTICS)
3218 inline_gather_statistics(iln);
3222 DOLOG( printf("-------- DONE -----------------------------------------------------------\n");
3229 * These are local overrides for various environment variables in Emacs.
3230 * Please do not remove this and leave it at the end of the file, where
3231 * Emacs will automagically detect them.
3232 * ---------------------------------------------------------------------
3235 * indent-tabs-mode: t
3239 * vim:noexpandtab:sw=4:ts=4: