src/vm/jit/replace.c

   1 /* src/vm/jit/replace.c - on-stack replacement of methods
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id$
  26
  27 */
  28
  29 #include "config.h"
  30 #include "vm/types.h"
  31
  32 #include <assert.h>
  33 #include <stdlib.h>
  34
  35 #include "arch.h"
  36
  37 #include "mm/memory.h"
  38
  39 #include "threads/threads-common.h"
  40
  41 #include "toolbox/logging.h"
  42
  43 #include "vm/stringlocal.h"
  44
  45 #include "vm/jit/abi.h"
  46 #include "vm/jit/asmpart.h"
  47 #include "vm/jit/disass.h"
  48 #include "vm/jit/jit.h"
  49 #include "vm/jit/md.h"
  50 #include "vm/jit/methodheader.h"
  51 #include "vm/jit/replace.h"
  52 #include "vm/jit/show.h"
  53 #include "vm/jit/stack.h"
  54
  55 #include "vmcore/options.h"
  56 #include "vmcore/classcache.h"
  57
  58
  59 #define REPLACE_PATCH_DYNAMIC_CALL
  60 /*#define REPLACE_PATCH_ALL*/
  61
  62 #if defined(ENABLE_VMLOG)
  63 #include <vmlog_cacao.h>
  64 #endif
  65
  66 /*** architecture-dependent configuration *************************************/
  67
  68 /* first unset the macros (default) */
  69 #undef REPLACE_RA_BETWEEN_FRAMES
  70 #undef REPLACE_RA_TOP_OF_FRAME
  71 #undef REPLACE_RA_LINKAGE_AREA
  72 #undef REPLACE_LEAFMETHODS_RA_REGISTER
  73 #undef REPLACE_REG_RA
  74
  75 /* i386, x86_64 and m68k */
  76 #if defined(__I386__) || defined(__X86_64__) || defined(__M68K__)
  77 #define REPLACE_RA_BETWEEN_FRAMES
  78 /* alpha */
  79 #elif defined(__ALPHA__)
  80 #define REPLACE_RA_TOP_OF_FRAME
  81 #define REPLACE_LEAFMETHODS_RA_REGISTER
  82 #define REPLACE_REG_RA REG_RA
  83 /* powerpc */
  84 #elif defined(__POWERPC__)
  85 #define REPLACE_RA_LINKAGE_AREA
  86 #define REPLACE_LEAFMETHODS_RA_REGISTER
  87 #define REPLACE_REG_RA REG_ITMP3 /* the execution state has the LR in itmp3 */
  88 /* s390 */
  89 #elif defined(__S390__)
  90 #define REPLACE_RA_TOP_OF_FRAME
  91 #define REPLACE_REG_RA REG_ITMP3
  92 #endif
  93
  94
  95 /*** configuration of native stack slot size **********************************/
  96
  97 /* XXX this should be in md-abi.h files, probably */
  98
  99 #if defined(HAS_4BYTE_STACKSLOT)
 100 #define SIZE_OF_STACKSLOT      4
 101 #define STACK_SLOTS_PER_FLOAT  2
 102 typedef u4 stackslot_t;
 103 #else
 104 #define SIZE_OF_STACKSLOT      8
 105 #define STACK_SLOTS_PER_FLOAT  1
 106 typedef u8 stackslot_t;
 107 #endif
 108
 109
 110 /*** debugging ****************************************************************/
 111
 112 #if !defined(NDEBUG)
 113 static void java_value_print(s4 type, replace_val_t value);
 114 static void replace_stackframeinfo_println(stackframeinfo *sfi);
 115 #endif
 116
 117 #if !defined(NDEBUG)
 118 #define DOLOG(code)        do{ if (opt_TraceReplacement > 1) { code; } } while(0)
 119 #define DOLOG_SHORT(code)  do{ if (opt_TraceReplacement > 0) { code; } } while(0)
 120 #else
 121 #define DOLOG(code)
 122 #define DOLOG_SHORT(code)
 123 #endif
 124
 125
 126 /*** statistics ***************************************************************/
 127
 128 #define REPLACE_STATISTICS
 129
 130 #if defined(REPLACE_STATISTICS)
 131
 132 static int stat_replacements = 0;
 133 static int stat_frames = 0;
 134 static int stat_recompile = 0;
 135 static int stat_staticpatch = 0;
 136 static int stat_unroll_inline = 0;
 137 static int stat_unroll_call = 0;
 138 static int stat_dist_frames[20] = { 0 };
 139 static int stat_dist_locals[20] = { 0 };
 140 static int stat_dist_locals_adr[10] = { 0 };
 141 static int stat_dist_locals_prim[10] = { 0 };
 142 static int stat_dist_locals_ret[10] = { 0 };
 143 static int stat_dist_locals_void[10] = { 0 };
 144 static int stat_dist_stack[10] = { 0 };
 145 static int stat_dist_stack_adr[10] = { 0 };
 146 static int stat_dist_stack_prim[10] = { 0 };
 147 static int stat_dist_stack_ret[10] = { 0 };
 148 static int stat_methods = 0;
 149 static int stat_rploints = 0;
 150 static int stat_regallocs = 0;
 151 static int stat_dist_method_rplpoints[20] = { 0 };
 152
 153 #define REPLACE_COUNT(cnt)  (cnt)++
 154 #define REPLACE_COUNT_IF(cnt, cond)  do{ if(cond) (cnt)++; } while(0)
 155 #define REPLACE_COUNT_INC(cnt, inc)  ((cnt) += (inc))
 156
 157 #define REPLACE_COUNT_DIST(array, val)                               \
 158     do {                                                             \
 159         int limit = (sizeof(array) / sizeof(int)) - 1;               \
 160         if ((val) < (limit)) (array)[val]++;                         \
 161         else (array)[limit]++;                                       \
 162     } while (0)
 163
 164 static void replace_statistics_source_frame(sourceframe_t *frame);
 165
 166 #else
 167
 168 #define REPLACE_COUNT(cnt)
 169 #define REPLACE_COUNT_IF(cnt, cond)
 170 #define REPLACE_COUNT_INC(cnt, inc)
 171 #define REPLACE_COUNT_DIST(array, val)
 172
 173 #endif /* defined(REPLACE_STATISTICS) */
 174
 175
 176 /*** constants used internally ************************************************/
 177
 178 #define TOP_IS_NORMAL    0
 179 #define TOP_IS_ON_STACK  1
 180 #define TOP_IS_IN_ITMP1  2
 181 #define TOP_IS_VOID      3
 182
 183
 184 /******************************************************************************/
 185 /* PART I: Creating / freeing replacement points                              */
 186 /******************************************************************************/
 187
 188
 189 /* replace_create_replacement_point ********************************************
 190
 191    Create a replacement point.
 192
 193    IN:
 194        jd...............current jitdata
 195            iinfo............inlining info for the current position
 196            rp...............pre-allocated (uninitialized) rplpoint
 197            type.............RPLPOINT_TYPE constant
 198            iptr.............current instruction
 199            *pra.............current rplalloc pointer
 200            javalocals.......the javalocals at the current point
 201            stackvars........the stack variables at the current point
 202            stackdepth.......the stack depth at the current point
 203            paramcount.......number of parameters at the start of stackvars
 204
 205    OUT:
 206        *rpa.............points to the next free rplalloc
 207
 208 *******************************************************************************/
 209
 210 static void replace_create_replacement_point(jitdata *jd,
 211                                                                                          insinfo_inline *iinfo,
 212                                                                                          rplpoint *rp,
 213                                                                                          s4 type,
 214                                                                                          instruction *iptr,
 215                                                                                          rplalloc **pra,
 216                                                                                          s4 *javalocals,
 217                                                                                          s4 *stackvars,
 218                                                                                          s4 stackdepth,
 219                                                                                          s4 paramcount)
 220 {
 221         rplalloc *ra;
 222         s4        i;
 223         varinfo  *v;
 224         s4        index;
 225
 226         ra = *pra;
 227
 228         REPLACE_COUNT(stat_rploints);
 229
 230         rp->method = (iinfo) ? iinfo->method : jd->m;
 231         rp->pc = NULL;        /* set by codegen */
 232         rp->callsize = 0;     /* set by codegen */
 233         rp->regalloc = ra;
 234         rp->flags = 0;
 235         rp->type = type;
 236         rp->id = iptr->flags.bits >> INS_FLAG_ID_SHIFT;
 237
 238         /* XXX unify these two fields */
 239         rp->parent = (iinfo) ? iinfo->rp : NULL;
 240
 241         /* store local allocation info of javalocals */
 242
 243         if (javalocals) {
 244                 for (i = 0; i < rp->method->maxlocals; ++i) {
 245                         index = javalocals[i];
 246                         if (index == UNUSED)
 247                                 continue;
 248
 249                         ra->index = i;
 250                         if (index >= 0) {
 251                                 v = VAR(index);
 252                                 ra->flags = v->flags & (INMEMORY);
 253                                 ra->regoff = v->vv.regoff;
 254                                 ra->type = v->type;
 255                         }
 256                         else {
 257                                 ra->regoff = RETADDR_FROM_JAVALOCAL(index);
 258                                 ra->type = TYPE_RET;
 259                                 ra->flags = 0;
 260                         }
 261                         ra++;
 262                 }
 263         }
 264
 265         /* store allocation info of java stack vars */
 266
 267         for (i = 0; i < stackdepth; ++i) {
 268                 v = VAR(stackvars[i]);
 269                 ra->flags = v->flags & (INMEMORY);
 270                 ra->index = (i < paramcount) ? RPLALLOC_PARAM : RPLALLOC_STACK;
 271                 ra->type  = v->type;
 272                 /* XXX how to handle locals on the stack containing returnAddresses? */
 273                 if (v->type == TYPE_RET) {
 274                         assert(stackvars[i] >= jd->localcount);
 275                         ra->regoff = v->vv.retaddr->nr;
 276                 }
 277                 else
 278                         ra->regoff = v->vv.regoff;
 279                 ra++;
 280         }
 281
 282         /* total number of allocations */
 283
 284         rp->regalloccount = ra - rp->regalloc;
 285
 286         *pra = ra;
 287 }
 288
 289
 290 /* replace_create_inline_start_replacement_point *******************************
 291
 292    Create an INLINE_START replacement point.
 293
 294    IN:
 295        jd...............current jitdata
 296            rp...............pre-allocated (uninitialized) rplpoint
 297            iptr.............current instruction
 298            *pra.............current rplalloc pointer
 299            javalocals.......the javalocals at the current point
 300
 301    OUT:
 302        *rpa.............points to the next free rplalloc
 303
 304    RETURN VALUE:
 305        the insinfo_inline * for the following inlined body
 306
 307 *******************************************************************************/
 308
 309 static insinfo_inline * replace_create_inline_start_replacement_point(
 310                                                                                          jitdata *jd,
 311                                                                                          rplpoint *rp,
 312                                                                                          instruction *iptr,
 313                                                                                          rplalloc **pra,
 314                                                                                          s4 *javalocals)
 315 {
 316         insinfo_inline *calleeinfo;
 317         rplalloc       *ra;
 318
 319         calleeinfo = iptr->sx.s23.s3.inlineinfo;
 320
 321         calleeinfo->rp = rp;
 322
 323         replace_create_replacement_point(jd, calleeinfo->parent, rp,
 324                         RPLPOINT_TYPE_INLINE, iptr, pra,
 325                         javalocals,
 326                         calleeinfo->stackvars, calleeinfo->stackvarscount,
 327                         calleeinfo->paramcount);
 328
 329         if (calleeinfo->synclocal != UNUSED) {
 330                 ra = (*pra)++;
 331                 ra->index  = RPLALLOC_SYNC;
 332                 ra->regoff = jd->var[calleeinfo->synclocal].vv.regoff;
 333                 ra->flags  = jd->var[calleeinfo->synclocal].flags & INMEMORY;
 334                 ra->type   = TYPE_ADR;
 335
 336                 rp->regalloccount++;
 337         }
 338
 339         return calleeinfo;
 340 }
 341
 342
 343 /* replace_create_replacement_points *******************************************
 344
 345    Create the replacement points for the given code.
 346
 347    IN:
 348        jd...............current jitdata, must not have any replacement points
 349
 350    OUT:
 351        code->rplpoints.......set to the list of replacement points
 352            code->rplpointcount...number of replacement points
 353            code->regalloc........list of allocation info
 354            code->regalloccount...total length of allocation info list
 355            code->globalcount.....number of global allocations at the
 356                                  start of code->regalloc
 357
 358    RETURN VALUE:
 359        true.............everything ok
 360        false............an exception has been thrown
 361
 362 *******************************************************************************/
 363
 364 #define CLEAR_javalocals(array, method)                              \
 365     do {                                                             \
 366         for (i=0; i<(method)->maxlocals; ++i)                        \
 367             (array)[i] = UNUSED;                                     \
 368     } while (0)
 369
 370 #define COPY_OR_CLEAR_javalocals(dest, array, method)                \
 371     do {                                                             \
 372         if ((array) != NULL)                                         \
 373             MCOPY((dest), (array), s4, (method)->maxlocals);         \
 374         else                                                         \
 375             CLEAR_javalocals((dest), (method));                      \
 376     } while (0)
 377
 378 #define COUNT_javalocals(array, method, counter)                     \
 379     do {                                                             \
 380         for (i=0; i<(method)->maxlocals; ++i)                        \
 381             if ((array)[i] != UNUSED)                                \
 382                                 (counter)++;                                         \
 383     } while (0)
 384
 385 bool replace_create_replacement_points(jitdata *jd)
 386 {
 387         codeinfo        *code;
 388         registerdata    *rd;
 389         basicblock      *bptr;
 390         int              count;
 391         methodinfo      *m;
 392         rplpoint        *rplpoints;
 393         rplpoint        *rp;
 394         int              alloccount;
 395         rplalloc        *regalloc;
 396         rplalloc        *ra;
 397         int              i;
 398         instruction     *iptr;
 399         instruction     *iend;
 400         s4              *javalocals;
 401         s4              *jl;
 402         methoddesc      *md;
 403         insinfo_inline  *iinfo;
 404         s4               startcount;
 405         s4               firstcount;
 406 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 407         bool             needentry;
 408 #endif
 409
 410         REPLACE_COUNT(stat_methods);
 411
 412         /* get required compiler data */
 413
 414         code = jd->code;
 415         rd   = jd->rd;
 416
 417         /* assert that we wont overwrite already allocated data */
 418
 419         assert(code);
 420         assert(code->m);
 421         assert(code->rplpoints == NULL);
 422         assert(code->rplpointcount == 0);
 423         assert(code->regalloc == NULL);
 424         assert(code->regalloccount == 0);
 425         assert(code->globalcount == 0);
 426
 427         m = code->m;
 428
 429         /* set codeinfo flags */
 430
 431         if (jd->isleafmethod)
 432                 CODE_SETFLAG_LEAFMETHOD(code);
 433
 434         /* in instance methods, we may need a rplpoint at the method entry */
 435
 436 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 437         if (!(m->flags & ACC_STATIC)) {
 438                 jd->basicblocks[0].bitflags |= BBFLAG_REPLACEMENT;
 439                 needentry = true;
 440         }
 441         else {
 442                 needentry = false;
 443         }
 444 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
 445
 446         /* iterate over the basic block list to find replacement points */
 447
 448         count = 0;
 449         alloccount = 0;
 450
 451         javalocals = DMNEW(s4, jd->maxlocals);
 452
 453         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 454
 455                 /* skip dead code */
 456
 457                 if (bptr->flags < BBFINISHED)
 458                         continue;
 459
 460                 /* get info about this block */
 461
 462                 m = bptr->method;
 463                 iinfo = bptr->inlineinfo;
 464
 465                 /* initialize javalocals at the start of this block */
 466
 467                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 468
 469                 /* iterate over the instructions */
 470
 471                 iptr = bptr->iinstr;
 472                 iend = iptr + bptr->icount;
 473                 startcount = count;
 474                 firstcount = count;
 475
 476                 for (; iptr != iend; ++iptr) {
 477                         switch (iptr->opc) {
 478                                 case ICMD_INVOKESTATIC:
 479                                 case ICMD_INVOKESPECIAL:
 480                                 case ICMD_INVOKEVIRTUAL:
 481                                 case ICMD_INVOKEINTERFACE:
 482                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 483                                         count++;
 484                                         COUNT_javalocals(javalocals, m, alloccount);
 485                                         alloccount += iptr->s1.argcount;
 486                                         if (iinfo)
 487                                                 alloccount -= iinfo->throughcount;
 488                                         break;
 489
 490                                 case ICMD_ISTORE:
 491                                 case ICMD_LSTORE:
 492                                 case ICMD_FSTORE:
 493                                 case ICMD_DSTORE:
 494                                 case ICMD_ASTORE:
 495                                         stack_javalocals_store(iptr, javalocals);
 496                                         break;
 497
 498                                 case ICMD_IRETURN:
 499                                 case ICMD_LRETURN:
 500                                 case ICMD_FRETURN:
 501                                 case ICMD_DRETURN:
 502                                 case ICMD_ARETURN:
 503                                         alloccount += 1;
 504                                         /* FALLTHROUGH! */
 505                                 case ICMD_RETURN:
 506                                         count++;
 507                                         break;
 508
 509                                 case ICMD_INLINE_START:
 510                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 511
 512                                         count++;
 513                                         COUNT_javalocals(javalocals, m, alloccount);
 514                                         alloccount += iinfo->stackvarscount;
 515                                         if (iinfo->synclocal != UNUSED)
 516                                                 alloccount++;
 517
 518                                         m = iinfo->method;
 519                                         /* javalocals may be set at next block start, or now */
 520                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 521                                         break;
 522
 523                                 case ICMD_INLINE_BODY:
 524                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 525
 526                                         jl = iinfo->javalocals_start;
 527                                         if (jl == NULL) {
 528                                                 /* get the javalocals from the following block start */
 529                                                 assert(bptr->next);
 530                                                 jl = bptr->next->javalocals;
 531                                         }
 532                                         count++;
 533                                         COUNT_javalocals(jl, m, alloccount);
 534                                         break;
 535
 536                                 case ICMD_INLINE_END:
 537                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 538                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 539                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 540                                         m = iinfo->outer;
 541                                         if (iinfo->javalocals_end)
 542                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 543                                         iinfo = iinfo->parent;
 544                                         break;
 545                         }
 546
 547                         if (iptr == bptr->iinstr)
 548                                 firstcount = count;
 549                 } /* end instruction loop */
 550
 551                 /* create replacement points at targets of backward branches */
 552                 /* We only need the replacement point there, if there is no  */
 553                 /* replacement point inside the block.                       */
 554
 555                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 556 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 557                         int test = (needentry && bptr == jd->basicblocks) ? firstcount : count;
 558 #else
 559                         int test = count;
 560 #endif
 561                         if (test > startcount) {
 562                                 /* we don't need an extra rplpoint */
 563                                 bptr->bitflags &= ~BBFLAG_REPLACEMENT;
 564                         }
 565                         else {
 566                                 count++;
 567                                 alloccount += bptr->indepth;
 568                                 if (bptr->inlineinfo)
 569                                         alloccount -= bptr->inlineinfo->throughcount;
 570
 571                                 COUNT_javalocals(bptr->javalocals, bptr->method, alloccount);
 572                         }
 573                 }
 574
 575         } /* end basicblock loop */
 576
 577         /* if no points were found, there's nothing to do */
 578
 579         if (!count)
 580                 return true;
 581
 582         /* allocate replacement point array and allocation array */
 583
 584         rplpoints = MNEW(rplpoint, count);
 585         regalloc = MNEW(rplalloc, alloccount);
 586         ra = regalloc;
 587
 588         /* initialize replacement point structs */
 589
 590         rp = rplpoints;
 591
 592         /* XXX try to share code with the counting loop! */
 593
 594         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 595                 /* skip dead code */
 596
 597                 if (bptr->flags < BBFINISHED)
 598                         continue;
 599
 600                 /* get info about this block */
 601
 602                 m = bptr->method;
 603                 iinfo = bptr->inlineinfo;
 604
 605                 /* initialize javalocals at the start of this block */
 606
 607                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 608
 609                 /* create replacement points at targets of backward branches */
 610
 611                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 612
 613                         i = (iinfo) ? iinfo->throughcount : 0;
 614                         replace_create_replacement_point(jd, iinfo, rp++,
 615                                         bptr->type, bptr->iinstr, &ra,
 616                                         bptr->javalocals, bptr->invars + i, bptr->indepth - i, 0);
 617
 618                         if (JITDATA_HAS_FLAG_COUNTDOWN(jd))
 619                                 rp[-1].flags |= RPLPOINT_FLAG_COUNTDOWN;
 620                 }
 621
 622                 /* iterate over the instructions */
 623
 624                 iptr = bptr->iinstr;
 625                 iend = iptr + bptr->icount;
 626
 627                 for (; iptr != iend; ++iptr) {
 628                         switch (iptr->opc) {
 629                                 case ICMD_INVOKESTATIC:
 630                                 case ICMD_INVOKESPECIAL:
 631                                 case ICMD_INVOKEVIRTUAL:
 632                                 case ICMD_INVOKEINTERFACE:
 633                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 634
 635                                         i = (iinfo) ? iinfo->throughcount : 0;
 636                                         replace_create_replacement_point(jd, iinfo, rp++,
 637                                                         RPLPOINT_TYPE_CALL, iptr, &ra,
 638                                                         javalocals, iptr->sx.s23.s2.args,
 639                                                         iptr->s1.argcount - i,
 640                                                         md->paramcount);
 641                                         break;
 642
 643                                 case ICMD_ISTORE:
 644                                 case ICMD_LSTORE:
 645                                 case ICMD_FSTORE:
 646                                 case ICMD_DSTORE:
 647                                 case ICMD_ASTORE:
 648                                         stack_javalocals_store(iptr, javalocals);
 649                                         break;
 650
 651                                 case ICMD_IRETURN:
 652                                 case ICMD_LRETURN:
 653                                 case ICMD_FRETURN:
 654                                 case ICMD_DRETURN:
 655                                 case ICMD_ARETURN:
 656                                         replace_create_replacement_point(jd, iinfo, rp++,
 657                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 658                                                         NULL, &(iptr->s1.varindex), 1, 0);
 659                                         break;
 660
 661                                 case ICMD_RETURN:
 662                                         replace_create_replacement_point(jd, iinfo, rp++,
 663                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 664                                                         NULL, NULL, 0, 0);
 665                                         break;
 666
 667                                 case ICMD_INLINE_START:
 668                                         iinfo = replace_create_inline_start_replacement_point(
 669                                                                 jd, rp++, iptr, &ra, javalocals);
 670                                         m = iinfo->method;
 671                                         /* javalocals may be set at next block start, or now */
 672                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 673                                         break;
 674
 675                                 case ICMD_INLINE_BODY:
 676                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 677
 678                                         jl = iinfo->javalocals_start;
 679                                         if (jl == NULL) {
 680                                                 /* get the javalocals from the following block start */
 681                                                 assert(bptr->next);
 682                                                 jl = bptr->next->javalocals;
 683                                         }
 684                                         /* create a non-trappable rplpoint */
 685                                         replace_create_replacement_point(jd, iinfo, rp++,
 686                                                         RPLPOINT_TYPE_BODY, iptr, &ra,
 687                                                         jl, NULL, 0, 0);
 688                                         rp[-1].flags |= RPLPOINT_FLAG_NOTRAP;
 689                                         break;
 690
 691                                 case ICMD_INLINE_END:
 692                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 693                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 694                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 695                                         m = iinfo->outer;
 696                                         if (iinfo->javalocals_end)
 697                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 698                                         iinfo = iinfo->parent;
 699                                         break;
 700                         }
 701                 } /* end instruction loop */
 702         } /* end basicblock loop */
 703
 704         assert((rp - rplpoints) == count);
 705         assert((ra - regalloc) == alloccount);
 706
 707         /* store the data in the codeinfo */
 708
 709         code->rplpoints     = rplpoints;
 710         code->rplpointcount = count;
 711         code->regalloc      = regalloc;
 712         code->regalloccount = alloccount;
 713         code->globalcount   = 0;
 714         code->savedintcount = INT_SAV_CNT - rd->savintreguse;
 715         code->savedfltcount = FLT_SAV_CNT - rd->savfltreguse;
 716 #if defined(HAS_ADDRESS_REGISTER_FILE)
 717         code->savedadrcount = ADR_SAV_CNT - rd->savadrreguse;
 718 #endif
 719         code->memuse        = rd->memuse;
 720         code->stackframesize = jd->cd->stackframesize;
 721
 722         REPLACE_COUNT_DIST(stat_dist_method_rplpoints, count);
 723         REPLACE_COUNT_INC(stat_regallocs, alloccount);
 724
 725         /* everything alright */
 726
 727         return true;
 728 }
 729
 730
 731 /* replace_free_replacement_points *********************************************
 732
 733    Free memory used by replacement points.
 734
 735    IN:
 736        code.............codeinfo whose replacement points should be freed.
 737
 738 *******************************************************************************/
 739
 740 void replace_free_replacement_points(codeinfo *code)
 741 {
 742         assert(code);
 743
 744         if (code->rplpoints)
 745                 MFREE(code->rplpoints,rplpoint,code->rplpointcount);
 746
 747         if (code->regalloc)
 748                 MFREE(code->regalloc,rplalloc,code->regalloccount);
 749
 750         code->rplpoints = NULL;
 751         code->rplpointcount = 0;
 752         code->regalloc = NULL;
 753         code->regalloccount = 0;
 754         code->globalcount = 0;
 755 }
 756
 757
 758 /******************************************************************************/
 759 /* PART II: Activating / deactivating replacement points                      */
 760 /******************************************************************************/
 761
 762
 763 /* replace_activate_replacement_points *****************************************
 764
 765    Activate the replacement points of the given compilation unit. When this
 766    function returns, the replacement points are "armed", so each thread
 767    reaching one of the points will enter the replacement mechanism.
 768
 769    IN:
 770        code.............codeinfo of which replacement points should be
 771                                                 activated
 772            mappable.........if true, only mappable replacement points are
 773                                                 activated
 774
 775 *******************************************************************************/
 776
 777 void replace_activate_replacement_points(codeinfo *code, bool mappable)
 778 {
 779         rplpoint *rp;
 780         s4        i;
 781         s4        count;
 782         s4        index;
 783         u1       *savedmcode;
 784
 785         assert(code->savedmcode == NULL);
 786
 787         /* count trappable replacement points */
 788
 789         count = 0;
 790         index = 0;
 791         i = code->rplpointcount;
 792         rp = code->rplpoints;
 793         for (; i--; rp++) {
 794                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 795                         continue;
 796
 797                 index++;
 798
 799                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 800                         continue;
 801
 802                 count++;
 803         }
 804
 805         /* allocate buffer for saved machine code */
 806
 807         savedmcode = MNEW(u1, count * REPLACEMENT_PATCH_SIZE);
 808         code->savedmcode = savedmcode;
 809         savedmcode += count * REPLACEMENT_PATCH_SIZE;
 810
 811         /* activate trappable replacement points */
 812         /* (in reverse order to handle overlapping points within basic blocks) */
 813
 814         i = code->rplpointcount;
 815         rp = code->rplpoints + i;
 816         while (rp--, i--) {
 817                 assert(!(rp->flags & RPLPOINT_FLAG_ACTIVE));
 818
 819                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 820                         continue;
 821
 822                 index--;
 823
 824                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 825                         continue;
 826
 827                 DOLOG( printf("activate replacement point:\n");
 828                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 829
 830                 savedmcode -= REPLACEMENT_PATCH_SIZE;
 831
 832 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__) || defined(__S390__)) && defined(ENABLE_JIT)
 833                 md_patch_replacement_point(code, index, rp, savedmcode);
 834 #endif
 835                 rp->flags |= RPLPOINT_FLAG_ACTIVE;
 836         }
 837
 838         assert(savedmcode == code->savedmcode);
 839 }
 840
 841
 842 /* replace_deactivate_replacement_points ***************************************
 843
 844    Deactivate a replacement points in the given compilation unit.
 845    When this function returns, the replacement points will be "un-armed",
 846    that is a each thread reaching a point will just continue normally.
 847
 848    IN:
 849        code.............the compilation unit
 850
 851 *******************************************************************************/
 852
 853 void replace_deactivate_replacement_points(codeinfo *code)
 854 {
 855         rplpoint *rp;
 856         s4        i;
 857         s4        count;
 858         u1       *savedmcode;
 859
 860         if (code->savedmcode == NULL) {
 861                 /* disarm countdown points by patching the branches */
 862
 863                 i = code->rplpointcount;
 864                 rp = code->rplpoints;
 865                 for (; i--; rp++) {
 866                         if ((rp->flags & (RPLPOINT_FLAG_ACTIVE | RPLPOINT_FLAG_COUNTDOWN))
 867                                         == RPLPOINT_FLAG_COUNTDOWN)
 868                         {
 869 #if 0
 870                                 *(s4*) (rp->pc + 9) = 0; /* XXX machine dependent! */
 871 #endif
 872                         }
 873                 }
 874                 return;
 875         }
 876
 877         assert(code->savedmcode != NULL);
 878         savedmcode = code->savedmcode;
 879
 880         /* de-activate each trappable replacement point */
 881
 882         i = code->rplpointcount;
 883         rp = code->rplpoints;
 884         count = 0;
 885         for (; i--; rp++) {
 886                 if (!(rp->flags & RPLPOINT_FLAG_ACTIVE))
 887                         continue;
 888
 889                 count++;
 890
 891                 DOLOG( printf("deactivate replacement point:\n");
 892                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 893
 894 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__) || defined(__S390__)) && defined(ENABLE_JIT)
 895                 md_patch_replacement_point(code, -1, rp, savedmcode);
 896 #endif
 897
 898                 rp->flags &= ~RPLPOINT_FLAG_ACTIVE;
 899
 900                 savedmcode += REPLACEMENT_PATCH_SIZE;
 901         }
 902
 903         assert(savedmcode == code->savedmcode + count * REPLACEMENT_PATCH_SIZE);
 904
 905         /* free saved machine code */
 906
 907         MFREE(code->savedmcode, u1, count * REPLACEMENT_PATCH_SIZE);
 908         code->savedmcode = NULL;
 909 }
 910
 911
 912 /******************************************************************************/
 913 /* PART III: The replacement mechanism                                        */
 914 /******************************************************************************/
 915
 916
 917 /* replace_read_value **********************************************************
 918
 919    Read a value with the given allocation from the execution state.
 920
 921    IN:
 922            es...............execution state
 923            ra...............allocation
 924            javaval..........where to put the value
 925
 926    OUT:
 927        *javaval.........the value
 928
 929 *******************************************************************************/
 930
 931 static void replace_read_value(executionstate_t *es,
 932                                                            rplalloc *ra,
 933                                                            replace_val_t *javaval)
 934 {
 935         if (ra->flags & INMEMORY) {
 936                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
 937 #ifdef HAS_4BYTE_STACKSLOT
 938                 if (IS_2_WORD_TYPE(ra->type)) {
 939                         javaval->l = *(u8*)(es->sp + ra->regoff);
 940                 }
 941                 else {
 942 #endif
 943                         javaval->p = *(ptrint*)(es->sp + ra->regoff);
 944 #ifdef HAS_4BYTE_STACKSLOT
 945                 }
 946 #endif
 947         }
 948         else {
 949                 /* allocated register */
 950                 if (IS_FLT_DBL_TYPE(ra->type)) {
 951                         javaval->d = es->fltregs[ra->regoff];
 952
 953                         if (ra->type == TYPE_FLT)
 954                                 javaval->f = javaval->d;
 955                 }
 956 #if defined(HAS_ADDRESS_REGISTER_FILE)
 957                 else if (IS_ADR_TYPE(ra->type)) {
 958                         javaval->p = es->adrregs[ra->regoff];
 959                 }
 960 #endif
 961                 else {
 962 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
 963                         if (ra->type == TYPE_LNG) {
 964                                 javaval->words.lo = es->intregs[GET_LOW_REG(ra->regoff)];
 965                                 javaval->words.hi = es->intregs[GET_HIGH_REG(ra->regoff)];
 966                         }
 967                         else
 968 #endif /* defined(SUPPORT_COMBINE_INTEGER_REGISTERS) */
 969                                 javaval->p = es->intregs[ra->regoff];
 970                 }
 971         }
 972 }
 973
 974
 975 /* replace_write_value *********************************************************
 976
 977    Write a value to the given allocation in the execution state.
 978
 979    IN:
 980            es...............execution state
 981            ra...............allocation
 982            *javaval.........the value
 983
 984 *******************************************************************************/
 985
 986 static void replace_write_value(executionstate_t *es,
 987                                                             rplalloc *ra,
 988                                                             replace_val_t *javaval)
 989 {
 990         if (ra->flags & INMEMORY) {
 991                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
 992 #ifdef HAS_4BYTE_STACKSLOT
 993                 if (IS_2_WORD_TYPE(ra->type)) {
 994                         *(u8*)(es->sp + ra->regoff) = javaval->l;
 995                 }
 996                 else {
 997 #endif
 998                         *(ptrint*)(es->sp + ra->regoff) = javaval->p;
 999 #ifdef HAS_4BYTE_STACKSLOT
1000                 }
1001 #endif
1002         }
1003         else {
1004                 /* allocated register */
1005                 switch (ra->type) {
1006                         case TYPE_FLT:
1007                                 es->fltregs[ra->regoff] = (double) javaval->f;
1008                                 break;
1009                         case TYPE_DBL:
1010                                 es->fltregs[ra->regoff] = javaval->d;
1011                                 break;
1012 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
1013                         case TYPE_LNG:
1014                                 es->intregs[GET_LOW_REG(ra->regoff)] = javaval->words.lo;
1015                                 es->intregs[GET_HIGH_REG(ra->regoff)] = javaval->words.hi;
1016                                 break;
1017 #endif
1018 #if defined(HAS_ADDRESS_REGISTER_FILE)
1019                         case TYPE_ADR:
1020                                 es->adrregs[ra->regoff] = javaval->p;
1021 #endif
1022                         default:
1023                                 es->intregs[ra->regoff] = javaval->p;
1024                 }
1025         }
1026 }
1027
1028
1029 /* replace_new_sourceframe *****************************************************
1030
1031    Allocate a new source frame and insert it at the front of the frame list.
1032
1033    IN:
1034            ss...............the source state
1035
1036    OUT:
1037            ss->frames.......set to new frame (the new head of the frame list).
1038
1039    RETURN VALUE:
1040        returns the new frame
1041
1042 *******************************************************************************/
1043
1044 static sourceframe_t *replace_new_sourceframe(sourcestate_t *ss)
1045 {
1046         sourceframe_t *frame;
1047
1048         frame = DNEW(sourceframe_t);
1049         MZERO(frame, sourceframe_t, 1);
1050
1051         frame->down = ss->frames;
1052         ss->frames = frame;
1053
1054         return frame;
1055 }
1056
1057
1058 /* replace_read_executionstate *************************************************
1059
1060    Read a source frame from the given executions state.
1061    The new source frame is pushed to the front of the frame list of the
1062    source state.
1063
1064    IN:
1065        rp...............replacement point at which `es` was taken
1066            es...............execution state
1067            ss...............the source state to add the source frame to
1068            topframe.........true, if the first (top-most) source frame on the
1069                             stack is to be read
1070
1071    OUT:
1072        *ss..............the source state with the newly created source frame
1073                             added
1074
1075 *******************************************************************************/
1076
1077 static s4 replace_normalize_type_map[] = {
1078 /* RPLPOINT_TYPE_STD    |--> */ RPLPOINT_TYPE_STD,
1079 /* RPLPOINT_TYPE_EXH    |--> */ RPLPOINT_TYPE_STD,
1080 /* RPLPOINT_TYPE_SBR    |--> */ RPLPOINT_TYPE_STD,
1081 /* RPLPOINT_TYPE_CALL   |--> */ RPLPOINT_TYPE_CALL,
1082 /* RPLPOINT_TYPE_INLINE |--> */ RPLPOINT_TYPE_CALL,
1083 /* RPLPOINT_TYPE_RETURN |--> */ RPLPOINT_TYPE_RETURN,
1084 /* RPLPOINT_TYPE_BODY   |--> */ RPLPOINT_TYPE_STD
1085 };
1086
1087
1088 static void replace_read_executionstate(rplpoint *rp,
1089                                                                                 executionstate_t *es,
1090                                                                                 sourcestate_t *ss,
1091                                                                                 bool topframe)
1092 {
1093         methodinfo    *m;
1094         codeinfo      *code;
1095         int            count;
1096         int            i;
1097         rplalloc      *ra;
1098         sourceframe_t *frame;
1099         int            topslot;
1100         stackslot_t   *sp;
1101         stackslot_t   *basesp;
1102
1103         code = code_find_codeinfo_for_pc(rp->pc);
1104         m = rp->method;
1105         topslot = TOP_IS_NORMAL;
1106
1107         /* stack pointer */
1108
1109         sp = (stackslot_t *) es->sp;
1110
1111         /* in some cases the top stack slot is passed in REG_ITMP1 */
1112
1113         if (rp->type == BBTYPE_EXH) {
1114                 topslot = TOP_IS_IN_ITMP1;
1115         }
1116
1117         /* calculate base stack pointer */
1118
1119         basesp = sp + code->stackframesize;
1120
1121         /* create the source frame */
1122
1123         frame = replace_new_sourceframe(ss);
1124         frame->method = rp->method;
1125         frame->id = rp->id;
1126         assert(rp->type >= 0 && rp->type < sizeof(replace_normalize_type_map)/sizeof(s4));
1127         frame->type = replace_normalize_type_map[rp->type];
1128         frame->fromrp = rp;
1129         frame->fromcode = code;
1130
1131         /* read local variables */
1132
1133         count = m->maxlocals;
1134         frame->javalocalcount = count;
1135         frame->javalocals = DMNEW(replace_val_t, count);
1136         frame->javalocaltype = DMNEW(u1, count);
1137
1138         /* mark values as undefined */
1139         for (i=0; i<count; ++i) {
1140 #if !defined(NDEBUG)
1141                 frame->javalocals[i].l = (u8) 0x00dead0000dead00ULL;
1142 #endif
1143                 frame->javalocaltype[i] = TYPE_VOID;
1144         }
1145
1146         /* some entries in the intregs array are not meaningful */
1147         /*es->intregs[REG_ITMP3] = (u8) 0x11dead1111dead11ULL;*/
1148 #if !defined(NDEBUG)
1149         es->intregs[REG_SP   ] = (ptrint) 0x11dead1111dead11ULL;
1150 #ifdef REG_PV
1151         es->intregs[REG_PV   ] = (ptrint) 0x11dead1111dead11ULL;
1152 #endif
1153 #endif /* !defined(NDEBUG) */
1154
1155         /* read javalocals */
1156
1157         count = rp->regalloccount;
1158         ra = rp->regalloc;
1159
1160         while (count && (i = ra->index) >= 0) {
1161                 assert(i < m->maxlocals);
1162                 frame->javalocaltype[i] = ra->type;
1163                 if (ra->type == TYPE_RET)
1164                         frame->javalocals[i].i = ra->regoff;
1165                 else
1166                         replace_read_value(es, ra, frame->javalocals + i);
1167                 ra++;
1168                 count--;
1169         }
1170
1171         /* read instance, if this is the first rplpoint */
1172
1173 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
1174         if (topframe && !(rp->method->flags & ACC_STATIC) && rp == code->rplpoints) {
1175 #if 1
1176                 /* we are at the start of the method body, so if local 0 is set, */
1177                 /* it is the instance.                                           */
1178                 if (frame->javalocaltype[0] == TYPE_ADR)
1179                         frame->instance = frame->javalocals[0];
1180 #else
1181                 rplalloc instra;
1182                 methoddesc *md;
1183
1184                 md = rp->method->parseddesc;
1185                 assert(md->params);
1186                 assert(md->paramcount >= 1);
1187                 instra.type = TYPE_ADR;
1188                 instra.regoff = md->params[0].regoff;
1189                 if (md->params[0].inmemory) {
1190                         instra.flags = INMEMORY;
1191                         instra.regoff += (1 + code->stackframesize) * SIZE_OF_STACKSLOT;
1192                 }
1193                 else {
1194                         instra.flags = 0;
1195                 }
1196                 replace_read_value(es, &instra, &(frame->instance));
1197 #endif
1198         }
1199 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
1200
1201         /* read stack slots */
1202
1203         frame->javastackdepth = count;
1204         frame->javastack = DMNEW(replace_val_t, count);
1205         frame->javastacktype = DMNEW(u1, count);
1206
1207 #if !defined(NDEBUG)
1208         /* mark values as undefined */
1209         for (i=0; i<count; ++i) {
1210                 frame->javastack[i].l = (u8) 0x00dead0000dead00ULL;
1211                 frame->javastacktype[i] = TYPE_VOID;
1212         }
1213 #endif /* !defined(NDEBUG) */
1214
1215         i = 0;
1216
1217         /* the first stack slot is special in SBR and EXH blocks */
1218
1219         if (topslot == TOP_IS_ON_STACK) {
1220                 assert(count);
1221
1222                 assert(ra->index == RPLALLOC_STACK);
1223                 assert(ra->type == TYPE_ADR);
1224                 frame->javastack[i].p = sp[-1];
1225                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1226                 count--;
1227                 i++;
1228                 ra++;
1229         }
1230         else if (topslot == TOP_IS_IN_ITMP1) {
1231                 assert(count);
1232
1233                 assert(ra->index == RPLALLOC_STACK);
1234                 assert(ra->type == TYPE_ADR);
1235                 frame->javastack[i].p = es->intregs[REG_ITMP1];
1236                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1237                 count--;
1238                 i++;
1239                 ra++;
1240         }
1241         else if (topslot == TOP_IS_VOID) {
1242                 assert(count);
1243
1244                 assert(ra->index == RPLALLOC_STACK);
1245                 frame->javastack[i].l = 0;
1246                 frame->javastacktype[i] = TYPE_VOID;
1247                 count--;
1248                 i++;
1249                 ra++;
1250         }
1251
1252         /* read remaining stack slots */
1253
1254         for (; count--; ra++) {
1255                 if (ra->index == RPLALLOC_SYNC) {
1256                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1257
1258                         /* only read synchronization slots when traversing an inline point */
1259
1260                         if (!topframe) {
1261                                 sourceframe_t *calleeframe = frame->down;
1262                                 assert(calleeframe);
1263                                 assert(calleeframe->syncslotcount == 0);
1264                                 assert(calleeframe->syncslots == NULL);
1265
1266                                 calleeframe->syncslotcount = 1;
1267                                 calleeframe->syncslots = DMNEW(replace_val_t, 1);
1268                                 replace_read_value(es,ra,calleeframe->syncslots);
1269                         }
1270
1271                         frame->javastackdepth--;
1272                         continue;
1273                 }
1274
1275                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1276
1277                 /* do not read parameters of calls down the call chain */
1278
1279                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1280                         frame->javastackdepth--;
1281                 }
1282                 else {
1283                         if (ra->type == TYPE_RET)
1284                                 frame->javastack[i].i = ra->regoff;
1285                         else
1286                                 replace_read_value(es,ra,frame->javastack + i);
1287                         frame->javastacktype[i] = ra->type;
1288                         i++;
1289                 }
1290         }
1291 }
1292
1293
1294 /* replace_write_executionstate ************************************************
1295
1296    Pop a source frame from the front of the frame list of the given source state
1297    and write its values into the execution state.
1298
1299    IN:
1300        rp...............replacement point for which execution state should be
1301                             created
1302            es...............the execution state to modify
1303            ss...............the given source state
1304            topframe.........true, if this is the last (top-most) source frame to be
1305                             translated
1306
1307    OUT:
1308        *es..............the execution state derived from the source state
1309
1310 *******************************************************************************/
1311
1312 static void replace_write_executionstate(rplpoint *rp,
1313                                                                                  executionstate_t *es,
1314                                                                                  sourcestate_t *ss,
1315                                                                                  bool topframe)
1316 {
1317         methodinfo     *m;
1318         codeinfo       *code;
1319         int             count;
1320         int             i;
1321         rplalloc       *ra;
1322         sourceframe_t  *frame;
1323         int             topslot;
1324         stackslot_t    *sp;
1325         stackslot_t    *basesp;
1326
1327         code = code_find_codeinfo_for_pc(rp->pc);
1328         m = rp->method;
1329         topslot = TOP_IS_NORMAL;
1330
1331         /* pop a source frame */
1332
1333         frame = ss->frames;
1334         assert(frame);
1335         ss->frames = frame->down;
1336
1337         /* calculate stack pointer */
1338
1339         sp = (stackslot_t *) es->sp;
1340
1341         basesp = sp + code->stackframesize;
1342
1343         /* in some cases the top stack slot is passed in REG_ITMP1 */
1344
1345         if (rp->type == BBTYPE_EXH) {
1346                 topslot = TOP_IS_IN_ITMP1;
1347         }
1348
1349         /* write javalocals */
1350
1351         ra = rp->regalloc;
1352         count = rp->regalloccount;
1353
1354         while (count && (i = ra->index) >= 0) {
1355                 assert(i < m->maxlocals);
1356                 assert(i < frame->javalocalcount);
1357                 assert(ra->type == frame->javalocaltype[i]);
1358                 if (ra->type == TYPE_RET) {
1359                         /* XXX assert that it matches this rplpoint */
1360                 }
1361                 else
1362                         replace_write_value(es, ra, frame->javalocals + i);
1363                 count--;
1364                 ra++;
1365         }
1366
1367         /* write stack slots */
1368
1369         i = 0;
1370
1371         /* the first stack slot is special in SBR and EXH blocks */
1372
1373         if (topslot == TOP_IS_ON_STACK) {
1374                 assert(count);
1375
1376                 assert(ra->index == RPLALLOC_STACK);
1377                 assert(i < frame->javastackdepth);
1378                 assert(frame->javastacktype[i] == TYPE_ADR);
1379                 sp[-1] = frame->javastack[i].p;
1380                 count--;
1381                 i++;
1382                 ra++;
1383         }
1384         else if (topslot == TOP_IS_IN_ITMP1) {
1385                 assert(count);
1386
1387                 assert(ra->index == RPLALLOC_STACK);
1388                 assert(i < frame->javastackdepth);
1389                 assert(frame->javastacktype[i] == TYPE_ADR);
1390                 es->intregs[REG_ITMP1] = frame->javastack[i].p;
1391                 count--;
1392                 i++;
1393                 ra++;
1394         }
1395         else if (topslot == TOP_IS_VOID) {
1396                 assert(count);
1397
1398                 assert(ra->index == RPLALLOC_STACK);
1399                 assert(i < frame->javastackdepth);
1400                 assert(frame->javastacktype[i] == TYPE_VOID);
1401                 count--;
1402                 i++;
1403                 ra++;
1404         }
1405
1406         /* write remaining stack slots */
1407
1408         for (; count--; ra++) {
1409                 if (ra->index == RPLALLOC_SYNC) {
1410                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1411
1412                         /* only write synchronization slots when traversing an inline point */
1413
1414                         if (!topframe) {
1415                                 assert(frame->down);
1416                                 assert(frame->down->syncslotcount == 1); /* XXX need to understand more cases */
1417                                 assert(frame->down->syncslots != NULL);
1418
1419                                 replace_write_value(es,ra,frame->down->syncslots);
1420                         }
1421                         continue;
1422                 }
1423
1424                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1425
1426                 /* do not write parameters of calls down the call chain */
1427
1428                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1429                         /* skip it */
1430                 }
1431                 else {
1432                         assert(i < frame->javastackdepth);
1433                         assert(ra->type == frame->javastacktype[i]);
1434                         if (ra->type == TYPE_RET) {
1435                                 /* XXX assert that it matches this rplpoint */
1436                         }
1437                         else {
1438                                 replace_write_value(es,ra,frame->javastack + i);
1439                         }
1440                         i++;
1441                 }
1442         }
1443
1444         /* set new pc */
1445
1446         es->pc = rp->pc;
1447 }
1448
1449
1450 /* replace_pop_activation_record ***********************************************
1451
1452    Peel a stack frame from the execution state.
1453
1454    *** This function imitates the effects of the method epilog ***
1455    *** and returning from the method call.                     ***
1456
1457    IN:
1458            es...............execution state
1459            frame............source frame, receives synchronization slots
1460
1461    OUT:
1462        *es..............the execution state after popping the stack frame
1463
1464 *******************************************************************************/
1465
1466 u1* replace_pop_activation_record(executionstate_t *es,
1467                                                                   sourceframe_t *frame)
1468 {
1469         u1 *ra;
1470         u1 *pv;
1471         s4 reg;
1472         s4 i;
1473         s4 count;
1474         codeinfo *code;
1475         stackslot_t *basesp;
1476         stackslot_t *sp;
1477
1478         assert(es->code);
1479         assert(frame);
1480
1481         /* read the return address */
1482
1483 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1484         if (CODE_IS_LEAFMETHOD(es->code))
1485                 ra = (u1*) (ptrint) es->intregs[REPLACE_REG_RA];
1486         else
1487 #endif
1488                 ra = md_stacktrace_get_returnaddress(es->sp,
1489                                 SIZE_OF_STACKSLOT * es->code->stackframesize);
1490
1491         DOLOG( printf("return address: %p\n", (void*)ra); );
1492
1493         assert(ra);
1494
1495         /* calculate the base of the stack frame */
1496
1497         sp = (stackslot_t *) es->sp;
1498         basesp = sp + es->code->stackframesize;
1499
1500         /* read slots used for synchronization */
1501
1502         assert(frame->syncslotcount == 0);
1503         assert(frame->syncslots == NULL);
1504         count = code_get_sync_slot_count(es->code);
1505         frame->syncslotcount = count;
1506         frame->syncslots = DMNEW(replace_val_t, count);
1507         for (i=0; i<count; ++i) {
1508                 frame->syncslots[i].p = sp[es->code->memuse + i]; /* XXX */
1509         }
1510
1511         /* restore return address, if part of frame */
1512
1513 #if defined(REPLACE_RA_TOP_OF_FRAME)
1514 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1515         if (!CODE_IS_LEAFMETHOD(es->code))
1516 #endif
1517                 es->intregs[REPLACE_REG_RA] = *--basesp;
1518 #endif /* REPLACE_RA_TOP_OF_FRAME */
1519
1520 #if defined(REPLACE_RA_LINKAGE_AREA)
1521 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1522         if (!CODE_IS_LEAFMETHOD(es->code))
1523 #endif
1524                 es->intregs[REPLACE_REG_RA] = basesp[LA_LR_OFFSET / sizeof(stackslot_t)];
1525 #endif /* REPLACE_RA_LINKAGE_AREA */
1526
1527         /* restore saved int registers */
1528
1529         reg = INT_REG_CNT;
1530         for (i=0; i<es->code->savedintcount; ++i) {
1531                 while (nregdescint[--reg] != REG_SAV)
1532                         ;
1533                 es->intregs[reg] = *--basesp;
1534         }
1535
1536         /* restore saved flt registers */
1537
1538         /* XXX align? */
1539         reg = FLT_REG_CNT;
1540         for (i=0; i<es->code->savedfltcount; ++i) {
1541                 while (nregdescfloat[--reg] != REG_SAV)
1542                         ;
1543                 basesp -= STACK_SLOTS_PER_FLOAT;
1544                 es->fltregs[reg] = *(double*)basesp;
1545         }
1546
1547 #if defined(HAS_ADDRESS_REGISTER_FILE)
1548         /* restore saved adr registers */
1549
1550         reg = ADR_REG_CNT;
1551         for (i=0; i<es->code->savedadrcount; ++i) {
1552                 while (nregdescadr[--reg] != REG_SAV)
1553                         ;
1554                 es->adrregs[reg] = *--basesp;
1555         }
1556 #endif
1557
1558         /* adjust the stackpointer */
1559
1560         es->sp += SIZE_OF_STACKSLOT * es->code->stackframesize;
1561
1562 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1563         es->sp += SIZE_OF_STACKSLOT; /* skip return address */
1564 #endif
1565
1566         /* Set the new pc. Subtract one so we do not hit the replacement point */
1567         /* of the instruction following the call, if there is one.             */
1568
1569         es->pc = ra - 1;
1570
1571         /* find the new codeinfo */
1572
1573         pv = md_codegen_get_pv_from_pc(ra);
1574
1575         DOLOG( printf("PV = %p\n", (void*) pv); );
1576
1577         if (pv == NULL) /* XXX can this really happen? */
1578                 return NULL;
1579
1580         code = *(codeinfo **)(pv + CodeinfoPointer);
1581
1582         DOLOG( printf("CODE = %p\n", (void*) code); );
1583
1584         /* return NULL if we reached native code */
1585
1586         es->pv = pv;
1587         es->code = code;
1588
1589         /* in debugging mode clobber non-saved registers */
1590
1591 #if !defined(NDEBUG)
1592         /* for debugging */
1593         for (i=0; i<INT_REG_CNT; ++i)
1594                 if ((nregdescint[i] != REG_SAV)
1595 #if defined(REG_RA)
1596                                 && (i != REPLACE_REG_RA)
1597 #endif
1598                         )
1599                         es->intregs[i] = (ptrint) 0x33dead3333dead33ULL;
1600         for (i=0; i<FLT_REG_CNT; ++i)
1601                 if (nregdescfloat[i] != REG_SAV)
1602                         *(u8*)&(es->fltregs[i]) = 0x33dead3333dead33ULL;
1603 # if defined(HAS_ADDRESS_REGISTER_FILE)
1604         for (i=0; i<ADR_REG_CNT; ++i)
1605                 if (nregdescadr[i] != REG_SAV)
1606                         es->adrregs[i] = (ptrint) 0x33dead3333dead33ULL;
1607 # endif
1608 #endif /* !defined(NDEBUG) */
1609
1610         return (code) ? ra : NULL;
1611 }
1612
1613
1614 /* replace_patch_method_pointer ************************************************
1615
1616    Patch a method pointer (may be in code, data segment, vftbl, or interface
1617    table).
1618
1619    IN:
1620            mpp..............address of the method pointer to patch
1621            entrypoint.......the new entrypoint of the method
1622            kind.............kind of call to patch, used only for debugging
1623
1624 *******************************************************************************/
1625
1626 static void replace_patch_method_pointer(methodptr *mpp,
1627                                                                                  methodptr entrypoint,
1628                                                                                  const char *kind)
1629 {
1630 #if !defined(NDEBUG)
1631         codeinfo       *oldcode;
1632         codeinfo       *newcode;
1633 #endif
1634
1635         DOLOG( printf("patch method pointer from: %p to %p\n",
1636                                   (void*) *mpp, (void*)entrypoint); );
1637
1638 #if !defined(NDEBUG)
1639         oldcode = *(codeinfo **)((u1*)(*mpp) + CodeinfoPointer);
1640         newcode = *(codeinfo **)((u1*)(entrypoint) + CodeinfoPointer);
1641
1642         DOLOG_SHORT( printf("\tpatch %s %p ", kind, (void*) oldcode);
1643                                  method_println(oldcode->m);
1644                                  printf("\t      with      %p ", (void*) newcode);
1645                                  method_println(newcode->m); );
1646
1647         assert(oldcode->m == newcode->m);
1648 #endif
1649
1650         /* write the new entrypoint */
1651
1652         *mpp = (methodptr) entrypoint;
1653 }
1654
1655
1656 /* replace_patch_class *********************************************************
1657
1658    Patch a method in the given class.
1659
1660    IN:
1661            vftbl............vftbl of the class
1662            m................the method to patch
1663            oldentrypoint....the old entrypoint to replace
1664            entrypoint.......the new entrypoint
1665
1666 *******************************************************************************/
1667
1668 void replace_patch_class(vftbl_t *vftbl,
1669                                                  methodinfo *m,
1670                                                  u1 *oldentrypoint,
1671                                                  u1 *entrypoint)
1672 {
1673         s4                 i;
1674         methodptr         *mpp;
1675         methodptr         *mppend;
1676
1677         /* patch the vftbl of the class */
1678
1679         replace_patch_method_pointer(vftbl->table + m->vftblindex,
1680                                                                  entrypoint,
1681                                                                  "virtual  ");
1682
1683         /* patch the interface tables */
1684
1685         assert(oldentrypoint);
1686
1687         for (i=0; i < vftbl->interfacetablelength; ++i) {
1688                 mpp = vftbl->interfacetable[-i];
1689                 mppend = mpp + vftbl->interfacevftbllength[i];
1690                 for (; mpp != mppend; ++mpp)
1691                         if (*mpp == oldentrypoint) {
1692                                 replace_patch_method_pointer(mpp, entrypoint, "interface");
1693                         }
1694         }
1695 }
1696
1697
1698 /* replace_patch_class_hierarchy ***********************************************
1699
1700    Patch a method in all loaded classes.
1701
1702    IN:
1703            m................the method to patch
1704            oldentrypoint....the old entrypoint to replace
1705            entrypoint.......the new entrypoint
1706
1707 *******************************************************************************/
1708
1709 struct replace_patch_data_t {
1710         methodinfo *m;
1711         u1         *oldentrypoint;
1712         u1         *entrypoint;
1713 };
1714
1715 #define CODEINFO_OF_CODE(entrypoint) \
1716         (*(codeinfo **)((u1*)(entrypoint) + CodeinfoPointer))
1717
1718 #define METHOD_OF_CODE(entrypoint) \
1719         (CODEINFO_OF_CODE(entrypoint)->m)
1720
1721 void replace_patch_callback(classinfo *c, struct replace_patch_data_t *pd)
1722 {
1723         vftbl_t *vftbl = c->vftbl;
1724
1725         if (vftbl != NULL
1726                 && vftbl->vftbllength > pd->m->vftblindex
1727                 && vftbl->table[pd->m->vftblindex] != &asm_abstractmethoderror
1728                 && METHOD_OF_CODE(vftbl->table[pd->m->vftblindex]) == pd->m)
1729         {
1730                 replace_patch_class(c->vftbl, pd->m, pd->oldentrypoint, pd->entrypoint);
1731         }
1732 }
1733
1734 void replace_patch_class_hierarchy(methodinfo *m,
1735                                                                    u1 *oldentrypoint,
1736                                                                    u1 *entrypoint)
1737 {
1738         struct replace_patch_data_t pd;
1739
1740         pd.m = m;
1741         pd.oldentrypoint = oldentrypoint;
1742         pd.entrypoint = entrypoint;
1743
1744         DOLOG_SHORT( printf("patching class hierarchy: ");
1745                              method_println(m); );
1746
1747         classcache_foreach_loaded_class(
1748                         (classcache_foreach_functionptr_t) &replace_patch_callback,
1749                         (void*) &pd);
1750 }
1751
1752
1753 /* replace_patch_future_calls **************************************************
1754
1755    Analyse a call site and depending on the kind of call patch the call, the
1756    virtual function table, or the interface table.
1757
1758    IN:
1759            ra...............return address pointing after the call site
1760            callerframe......source frame of the caller
1761            calleeframe......source frame of the callee, must have been mapped
1762
1763 *******************************************************************************/
1764
1765 void replace_patch_future_calls(u1 *ra,
1766                                                                 sourceframe_t *callerframe,
1767                                                                 sourceframe_t *calleeframe)
1768 {
1769         u1             *patchpos;
1770         methodptr       entrypoint;
1771         methodptr       oldentrypoint;
1772         bool            atentry;
1773         stackframeinfo  sfi;
1774         codeinfo       *calleecode;
1775         methodinfo     *calleem;
1776         java_object_t  *obj;
1777         vftbl_t        *vftbl;
1778
1779         assert(ra);
1780         assert(callerframe->down == calleeframe);
1781
1782         /* get the new codeinfo and the method that shall be entered */
1783
1784         calleecode = calleeframe->tocode;
1785         assert(calleecode);
1786
1787         calleem = calleeframe->method;
1788         assert(calleem == calleecode->m);
1789
1790         entrypoint = (methodptr) calleecode->entrypoint;
1791
1792         /* check if we are at an method entry rplpoint at the innermost frame */
1793
1794         atentry = (calleeframe->down == NULL)
1795                         && !(calleem->flags & ACC_STATIC)
1796                         && (calleeframe->fromrp->id == 0); /* XXX */
1797
1798         /* get the position to patch, in case it was a statically bound call   */
1799
1800         sfi.pv = callerframe->fromcode->entrypoint;
1801         patchpos = md_get_method_patch_address(ra, &sfi, NULL);
1802
1803         if (patchpos == NULL) {
1804                 /* the call was dispatched dynamically */
1805
1806                 /* we can only patch such calls if we are at the entry point */
1807
1808                 if (!atentry)
1809                         return;
1810
1811                 assert((calleem->flags & ACC_STATIC) == 0);
1812
1813                 oldentrypoint = calleeframe->fromcode->entrypoint;
1814
1815                 /* we need to know the instance */
1816
1817                 if (!calleeframe->instance.a) {
1818                         DOLOG_SHORT( printf("WARNING: object instance unknown!\n"); );
1819                         replace_patch_class_hierarchy(calleem, oldentrypoint, entrypoint);
1820                         return;
1821                 }
1822
1823                 /* get the vftbl */
1824
1825                 obj = calleeframe->instance.a;
1826                 vftbl = obj->vftbl;
1827
1828                 assert(vftbl->class->vftbl == vftbl);
1829
1830                 DOLOG_SHORT( printf("\tclass: "); class_println(vftbl->class); );
1831
1832                 replace_patch_class(vftbl, calleem, oldentrypoint, entrypoint);
1833         }
1834         else {
1835                 /* the call was statically bound */
1836
1837                 replace_patch_method_pointer((methodptr *) patchpos, entrypoint, "static   ");
1838         }
1839 }
1840
1841
1842 /* replace_push_activation_record **********************************************
1843
1844    Push a stack frame onto the execution state.
1845
1846    *** This function imitates the effects of a call and the ***
1847    *** method prolog of the callee.                         ***
1848
1849    IN:
1850            es...............execution state
1851            rpcall...........the replacement point at the call site
1852            callerframe......source frame of the caller, or NULL for creating the
1853                             first frame
1854            calleeframe......source frame of the callee, must have been mapped
1855
1856    OUT:
1857        *es..............the execution state after pushing the stack frame
1858
1859 *******************************************************************************/
1860
1861 void replace_push_activation_record(executionstate_t *es,
1862                                                                         rplpoint *rpcall,
1863                                                                         sourceframe_t *callerframe,
1864                                                                         sourceframe_t *calleeframe)
1865 {
1866         s4           reg;
1867         s4           i;
1868         s4           count;
1869         stackslot_t *basesp;
1870         stackslot_t *sp;
1871         u1          *ra;
1872         codeinfo    *calleecode;
1873
1874         assert(es);
1875         assert(!rpcall || callerframe);
1876     assert(!rpcall || rpcall->type == RPLPOINT_TYPE_CALL);
1877         assert(!rpcall || rpcall == callerframe->torp);
1878         assert(calleeframe);
1879         assert(!callerframe || calleeframe == callerframe->down);
1880
1881         /* the compilation unit we are entering */
1882
1883         calleecode = calleeframe->tocode;
1884         assert(calleecode);
1885
1886         /* calculate the return address */
1887
1888         if (rpcall)
1889                 ra = rpcall->pc + rpcall->callsize;
1890         else
1891                 ra = es->pc + 1 /* XXX this is ugly */;
1892
1893         /* write the return address */
1894
1895 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1896         es->sp -= SIZE_OF_STACKSLOT;
1897
1898         *((stackslot_t *)es->sp) = (stackslot_t) ra;
1899 #endif /* REPLACE_RA_BETWEEN_FRAMES */
1900
1901 #if defined(REPLACE_REG_RA)
1902         es->intregs[REPLACE_REG_RA] = (ptrint) ra;
1903 #endif
1904
1905         /* we move into a new code unit */
1906
1907         es->code = calleecode;
1908
1909         /* set the new pc XXX not needed? */
1910
1911         es->pc = calleecode->entrypoint;
1912
1913         /* build the stackframe */
1914
1915         DOLOG( printf("building stackframe of %d words at %p\n",
1916                                   calleecode->stackframesize, (void*)es->sp); );
1917
1918         sp = (stackslot_t *) es->sp;
1919         basesp = sp;
1920
1921         sp -= calleecode->stackframesize;
1922         es->sp = (u1*) sp;
1923
1924         /* in debug mode, invalidate stack frame first */
1925
1926         /* XXX may not invalidate linkage area used by native code! */
1927 #if !defined(NDEBUG) && 0
1928         for (i=0; i<(basesp - sp); ++i) {
1929                 sp[i] = 0xdeaddeadU;
1930         }
1931 #endif
1932
1933         /* save the return address register */
1934
1935 #if defined(REPLACE_RA_TOP_OF_FRAME)
1936 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1937         if (!CODE_IS_LEAFMETHOD(calleecode))
1938 #endif
1939                 *--basesp = (ptrint) ra;
1940 #endif /* REPLACE_RA_TOP_OF_FRAME */
1941
1942 #if defined(REPLACE_RA_LINKAGE_AREA)
1943 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1944         if (!CODE_IS_LEAFMETHOD(calleecode))
1945 #endif
1946                 basesp[LA_LR_OFFSET / sizeof(stackslot_t)] = (ptrint) ra;
1947 #endif /* REPLACE_RA_LINKAGE_AREA */
1948
1949         /* save int registers */
1950
1951         reg = INT_REG_CNT;
1952         for (i=0; i<calleecode->savedintcount; ++i) {
1953                 while (nregdescint[--reg] != REG_SAV)
1954                         ;
1955                 *--basesp = es->intregs[reg];
1956
1957                 /* XXX may not clobber saved regs used by native code! */
1958 #if !defined(NDEBUG) && 0
1959                 es->intregs[reg] = (ptrint) 0x44dead4444dead44ULL;
1960 #endif
1961         }
1962
1963         /* save flt registers */
1964
1965         /* XXX align? */
1966         reg = FLT_REG_CNT;
1967         for (i=0; i<calleecode->savedfltcount; ++i) {
1968                 while (nregdescfloat[--reg] != REG_SAV)
1969                         ;
1970                 basesp -= STACK_SLOTS_PER_FLOAT;
1971                 *(double*)basesp = es->fltregs[reg];
1972
1973                 /* XXX may not clobber saved regs used by native code! */
1974 #if !defined(NDEBUG) && 0
1975                 *(u8*)&(es->fltregs[reg]) = 0x44dead4444dead44ULL;
1976 #endif
1977         }
1978
1979 #if defined(HAS_ADDRESS_REGISTER_FILE)
1980         /* save adr registers */
1981
1982         reg = ADR_REG_CNT;
1983         for (i=0; i<calleecode->savedadrcount; ++i) {
1984                 while (nregdescadr[--reg] != REG_SAV)
1985                         ;
1986                 *--basesp = es->adrregs[reg];
1987
1988                 /* XXX may not clobber saved regs used by native code! */
1989 #if !defined(NDEBUG) && 0
1990                 es->adrregs[reg] = (ptrint) 0x44dead4444dead44ULL;
1991 #endif
1992         }
1993 #endif
1994
1995         /* write slots used for synchronization */
1996
1997         count = code_get_sync_slot_count(calleecode);
1998         assert(count == calleeframe->syncslotcount);
1999         for (i=0; i<count; ++i) {
2000                 sp[calleecode->memuse + i] = calleeframe->syncslots[i].p;
2001         }
2002
2003         /* set the PV */
2004
2005         es->pv = calleecode->entrypoint;
2006
2007         /* redirect future invocations */
2008
2009         if (callerframe && rpcall) {
2010 #if defined(REPLACE_PATCH_ALL)
2011                 if (rpcall->type == callerframe->fromrp->type)
2012 #else
2013                 if (rpcall == callerframe->fromrp)
2014 #endif
2015                         replace_patch_future_calls(ra, callerframe, calleeframe);
2016         }
2017 }
2018
2019
2020 /* replace_find_replacement_point **********************************************
2021
2022    Find the replacement point in the given code corresponding to the
2023    position given in the source frame.
2024
2025    IN:
2026            code.............the codeinfo in which to search the rplpoint
2027            frame............the source frame defining the position to look for
2028            parent...........parent replacement point to match
2029
2030    RETURN VALUE:
2031        the replacement point
2032
2033 *******************************************************************************/
2034
2035 rplpoint * replace_find_replacement_point(codeinfo *code,
2036                                                                                   sourceframe_t *frame,
2037                                                                                   rplpoint *parent)
2038 {
2039         methodinfo *m;
2040         rplpoint *rp;
2041         s4        i;
2042         s4        j;
2043         s4        stacki;
2044         rplalloc *ra;
2045
2046         assert(code);
2047         assert(frame);
2048
2049         DOLOG( printf("searching replacement point for:\n");
2050                    replace_source_frame_println(frame); );
2051
2052         m = frame->method;
2053
2054         DOLOG( printf("code = %p\n", (void*)code); );
2055
2056         rp = code->rplpoints;
2057         i = code->rplpointcount;
2058         while (i--) {
2059                 if (rp->id == frame->id && rp->method == frame->method
2060                                 && rp->parent == parent
2061                                 && replace_normalize_type_map[rp->type] == frame->type)
2062                 {
2063                         /* check if returnAddresses match */
2064                         /* XXX optimize: only do this if JSRs in method */
2065                         DOLOG( printf("checking match for:");
2066                                    replace_replacement_point_println(rp, 1); fflush(stdout); );
2067                         ra = rp->regalloc;
2068                         stacki = 0;
2069                         for (j = rp->regalloccount; j--; ++ra) {
2070                                 if (ra->type == TYPE_RET) {
2071                                         if (ra->index == RPLALLOC_STACK) {
2072                                                 assert(stacki < frame->javastackdepth);
2073                                                 if (frame->javastack[stacki].i != ra->regoff)
2074                                                         goto no_match;
2075                                                 stacki++;
2076                                         }
2077                                         else {
2078                                                 assert(ra->index >= 0 && ra->index < frame->javalocalcount);
2079                                                 if (frame->javalocals[ra->index].i != ra->regoff)
2080                                                         goto no_match;
2081                                         }
2082                                 }
2083                         }
2084
2085                         /* found */
2086                         return rp;
2087                 }
2088 no_match:
2089                 rp++;
2090         }
2091
2092 #if !defined(NDEBUG)
2093         printf("candidate replacement points were:\n");
2094         rp = code->rplpoints;
2095         i = code->rplpointcount;
2096         for (; i--; ++rp) {
2097                 replace_replacement_point_println(rp, 1);
2098         }
2099 #endif
2100
2101         vm_abort("no matching replacement point found");
2102         return NULL; /* NOT REACHED */
2103 }
2104
2105
2106 /* replace_find_replacement_point_for_pc ***************************************
2107
2108    Find the nearest replacement point at or before the given PC.
2109
2110    IN:
2111        code.............compilation unit the PC is in
2112            pc...............the machine code PC
2113
2114    RETURN VALUE:
2115        the replacement point found, or
2116            NULL if no replacement point was found
2117
2118 *******************************************************************************/
2119
2120 rplpoint *replace_find_replacement_point_for_pc(codeinfo *code, u1 *pc)
2121 {
2122         rplpoint *found;
2123         rplpoint *rp;
2124         s4        i;
2125
2126         DOLOG( printf("searching for rp in %p ", (void*)code);
2127                    method_println(code->m); );
2128
2129         found = NULL;
2130
2131         rp = code->rplpoints;
2132         for (i=0; i<code->rplpointcount; ++i, ++rp) {
2133                 DOLOG( replace_replacement_point_println(rp, 2); );
2134                 if (rp->pc <= pc)
2135                         found = rp;
2136         }
2137
2138         return found;
2139 }
2140
2141
2142 /* replace_pop_native_frame ****************************************************
2143
2144    Unroll a native frame in the execution state and create a source frame
2145    for it.
2146
2147    IN:
2148            es...............current execution state
2149            ss...............the current source state
2150            sfi..............stackframeinfo for the native frame
2151
2152    OUT:
2153        es...............execution state after unrolling the native frame
2154            ss...............gets the added native source frame
2155
2156 *******************************************************************************/
2157
2158 static void replace_pop_native_frame(executionstate_t *es,
2159                                                                          sourcestate_t *ss,
2160                                                                          stackframeinfo *sfi)
2161 {
2162         sourceframe_t *frame;
2163         codeinfo      *code;
2164         s4             i,j;
2165
2166         assert(sfi);
2167
2168         frame = replace_new_sourceframe(ss);
2169
2170         frame->sfi = sfi;
2171
2172         /* remember pc and size of native frame */
2173
2174         frame->nativepc = es->pc;
2175         frame->nativeframesize = sfi->sp - es->sp;
2176         assert(frame->nativeframesize >= 0);
2177
2178         /* remember values of saved registers */
2179
2180         j = 0;
2181         for (i=0; i<INT_REG_CNT; ++i) {
2182                 if (nregdescint[i] == REG_SAV)
2183                         frame->nativesavint[j++] = es->intregs[i];
2184         }
2185
2186         j = 0;
2187         for (i=0; i<FLT_REG_CNT; ++i) {
2188                 if (nregdescfloat[i] == REG_SAV)
2189                         frame->nativesavflt[j++] = es->fltregs[i];
2190         }
2191
2192 #if defined(HAS_ADDRESS_REGISTER_FILE)
2193         j = 0;
2194         for (i=0; i<ADR_REG_CNT; ++i) {
2195                 if (nregdescadr[i] == REG_SAV)
2196                         frame->nativesavadr[j++] = es->adrregs[i];
2197         }
2198 #endif
2199
2200         /* restore saved registers */
2201
2202 #if 0
2203         /* XXX we don't have them, yet, in the sfi, so clear them */
2204
2205         for (i=0; i<INT_REG_CNT; ++i) {
2206                 if (nregdescint[i] == REG_SAV)
2207                         es->intregs[i] = 0;
2208         }
2209
2210         for (i=0; i<FLT_REG_CNT; ++i) {
2211                 if (nregdescfloat[i] == REG_SAV)
2212                         es->fltregs[i] = 0.0;
2213         }
2214
2215 # if defined(HAS_ADDRESS_REGISTER_FILE)
2216         for (i=0; i<ADR_REG_CNT; ++i) {
2217                 if (nregdescadr[i] == REG_SAV)
2218                         es->adrregs[i] = 0;
2219         }
2220 # endif
2221 #endif
2222
2223         /* restore pv, pc, and sp */
2224
2225         if (sfi->pv == NULL) {
2226                 /* frame of a native function call */
2227                 es->pv = md_codegen_get_pv_from_pc(sfi->ra);
2228         }
2229         else {
2230                 es->pv = sfi->pv;
2231         }
2232         es->pc = ((sfi->xpc) ? sfi->xpc : sfi->ra) - 1;
2233         es->sp = sfi->sp;
2234
2235         /* find the new codeinfo */
2236
2237         DOLOG( printf("PV = %p\n", (void*) es->pv); );
2238
2239         assert(es->pv != NULL);
2240
2241         code = *(codeinfo **)(es->pv + CodeinfoPointer);
2242
2243         DOLOG( printf("CODE = %p\n", (void*) code); );
2244
2245         es->code = code;
2246 }
2247
2248
2249 /* replace_push_native_frame ***************************************************
2250
2251    Rebuild a native frame onto the execution state and remove its source frame.
2252
2253    Note: The native frame is "rebuild" by setting fields like PC and stack
2254          pointer in the execution state accordingly. Values in the
2255                  stackframeinfo may be modified, but the actual stack frame of the
2256                  native code is not touched.
2257
2258    IN:
2259            es...............current execution state
2260            ss...............the current source state
2261
2262    OUT:
2263        es...............execution state after re-rolling the native frame
2264            ss...............the native source frame is removed
2265
2266 *******************************************************************************/
2267
2268 static void replace_push_native_frame(executionstate_t *es, sourcestate_t *ss)
2269 {
2270         sourceframe_t *frame;
2271         s4             i,j;
2272
2273         assert(es);
2274         assert(ss);
2275
2276         DOLOG( printf("pushing native frame\n"); );
2277
2278         /* remove the frame from the source state */
2279
2280         frame = ss->frames;
2281         assert(frame);
2282         assert(REPLACE_IS_NATIVE_FRAME(frame));
2283
2284         ss->frames = frame->down;
2285
2286         /* assert that the native frame has not moved */
2287
2288         assert(es->sp == frame->sfi->sp);
2289
2290         /* restore saved registers */
2291
2292         j = 0;
2293         for (i=0; i<INT_REG_CNT; ++i) {
2294                 if (nregdescint[i] == REG_SAV)
2295                         es->intregs[i] = frame->nativesavint[j++];
2296         }
2297
2298         j = 0;
2299         for (i=0; i<FLT_REG_CNT; ++i) {
2300                 if (nregdescfloat[i] == REG_SAV)
2301                         es->fltregs[i] = frame->nativesavflt[j++];
2302         }
2303
2304 #if defined(HAS_ADDRESS_REGISTER_FILE)
2305         j = 0;
2306         for (i=0; i<ADR_REG_CNT; ++i) {
2307                 if (nregdescadr[i] == REG_SAV)
2308                         es->adrregs[i] = frame->nativesavadr[j++];
2309         }
2310 #endif
2311
2312         /* skip the native frame on the machine stack */
2313
2314         es->sp -= frame->nativeframesize;
2315
2316         /* set the pc the next frame must return to */
2317
2318         es->pc = frame->nativepc;
2319 }
2320
2321
2322 /* replace_recover_source_state ************************************************
2323
2324    Recover the source state from the given replacement point and execution
2325    state.
2326
2327    IN:
2328        rp...............replacement point that has been reached, if any
2329            sfi..............stackframeinfo, if called from native code
2330            es...............execution state at the replacement point rp
2331
2332    RETURN VALUE:
2333        the source state
2334
2335 *******************************************************************************/
2336
2337 sourcestate_t *replace_recover_source_state(rplpoint *rp,
2338                                                                                         stackframeinfo *sfi,
2339                                                                                     executionstate_t *es)
2340 {
2341         sourcestate_t *ss;
2342         u1            *ra;
2343         bool           locked;
2344 #if defined(REPLACE_STATISTICS)
2345         s4             depth;
2346 #endif
2347
2348         /* create the source frame structure in dump memory */
2349
2350         ss = DNEW(sourcestate_t);
2351         ss->frames = NULL;
2352
2353         /* get the stackframeinfo if none is given */
2354
2355         if (sfi == NULL)
2356                 sfi = STACKFRAMEINFO;
2357
2358         /* each iteration of the loop recovers one source frame */
2359
2360         depth = 0;
2361         locked = false;
2362
2363         while (rp || sfi) {
2364
2365                 DOLOG( replace_executionstate_println(es); );
2366
2367                 /* if we are not at a replacement point, it is a native frame */
2368
2369                 if (rp == NULL) {
2370                         DOLOG( printf("native frame: sfi: "); replace_stackframeinfo_println(sfi); );
2371
2372                         locked = true;
2373                         replace_pop_native_frame(es, ss, sfi);
2374                         sfi = sfi->prev;
2375
2376                         if (es->code == NULL)
2377                                 continue;
2378
2379                         goto after_machine_frame;
2380                 }
2381
2382                 /* read the values for this source frame from the execution state */
2383
2384                 DOLOG( printf("recovering source state for%s:\n",
2385                                         (ss->frames == NULL) ? " TOPFRAME" : "");
2386                            replace_replacement_point_println(rp, 1); );
2387
2388                 replace_read_executionstate(rp, es, ss, ss->frames == NULL);
2389
2390 #if defined(ENABLE_VMLOG)
2391                 vmlog_cacao_unrol_method(ss->frames->method);
2392 #endif
2393
2394 #if defined(REPLACE_STATISTICS)
2395                 REPLACE_COUNT(stat_frames);
2396                 depth++;
2397                 replace_statistics_source_frame(ss->frames);
2398 #endif
2399
2400                 /* in locked areas (below native frames), identity map the frame */
2401
2402                 if (locked) {
2403                         ss->frames->torp = ss->frames->fromrp;
2404                         ss->frames->tocode = ss->frames->fromcode;
2405                 }
2406
2407                 /* unroll to the next (outer) frame */
2408
2409                 if (rp->parent) {
2410                         /* this frame is in inlined code */
2411
2412                         DOLOG( printf("INLINED!\n"); );
2413
2414                         rp = rp->parent;
2415
2416                         assert(rp->type == RPLPOINT_TYPE_INLINE);
2417                         REPLACE_COUNT(stat_unroll_inline);
2418                 }
2419                 else {
2420                         /* this frame had been called at machine-level. pop it. */
2421
2422                         DOLOG( printf("UNWIND\n"); );
2423
2424                         ra = replace_pop_activation_record(es, ss->frames);
2425                         if (ra == NULL) {
2426                                 DOLOG( printf("REACHED NATIVE CODE\n"); );
2427
2428                                 rp = NULL;
2429
2430                                 break; /* XXX remove to activate native frames */
2431                                 continue;
2432                         }
2433
2434                         /* find the replacement point at the call site */
2435
2436 after_machine_frame:
2437                         rp = replace_find_replacement_point_for_pc(es->code, es->pc);
2438
2439                         if (rp == NULL)
2440                                 vm_abort("could not find replacement point while unrolling call");
2441
2442                         DOLOG( printf("found replacement point.\n");
2443                                         replace_replacement_point_println(rp, 1); );
2444
2445                         assert(rp->type == RPLPOINT_TYPE_CALL);
2446                         REPLACE_COUNT(stat_unroll_call);
2447                 }
2448         } /* end loop over source frames */
2449
2450         REPLACE_COUNT_DIST(stat_dist_frames, depth);
2451
2452         return ss;
2453 }
2454
2455
2456 /* replace_map_source_state ****************************************************
2457
2458    Map each source frame in the given source state to a target replacement
2459    point and compilation unit. If no valid code is available for a source
2460    frame, it is (re)compiled.
2461
2462    IN:
2463        ss...............the source state
2464
2465    OUT:
2466        ss...............the source state, modified: The `torp` and `tocode`
2467                             fields of each source frame are set.
2468
2469    RETURN VALUE:
2470        true.............everything went ok
2471            false............an exception has been thrown
2472
2473 *******************************************************************************/
2474
2475 static bool replace_map_source_state(sourcestate_t *ss)
2476 {
2477         sourceframe_t *frame;
2478         codeinfo      *code;
2479         rplpoint      *rp;
2480         rplpoint      *parent; /* parent of inlined rplpoint */
2481 #if defined(REPLACE_STATISTICS)
2482         codeinfo      *oldcode;
2483 #endif
2484
2485         parent = NULL;
2486         code = NULL;
2487
2488         /* iterate over the source frames from outermost to innermost */
2489
2490         for (frame = ss->frames; frame != NULL; frame = frame->down) {
2491
2492                 /* XXX skip native frames */
2493
2494                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
2495                         parent = NULL;
2496                         continue;
2497                 }
2498
2499                 /* map frames which are not already mapped */
2500
2501                 if (frame->tocode) {
2502                         code = frame->tocode;
2503                         rp = frame->torp;
2504                         assert(rp);
2505                 }
2506                 else {
2507                         assert(frame->torp == NULL);
2508
2509                         if (parent == NULL) {
2510                                 /* find code for this frame */
2511
2512 #if defined(REPLACE_STATISTICS)
2513                                 oldcode = frame->method->code;
2514 #endif
2515                                 /* request optimization of hot methods and their callers */
2516
2517                                 if (frame->method->hitcountdown < 0
2518                                         || (frame->down && frame->down->method->hitcountdown < 0))
2519                                         jit_request_optimization(frame->method);
2520
2521                                 code = jit_get_current_code(frame->method);
2522
2523                                 if (code == NULL)
2524                                         return false; /* exception */
2525
2526                                 REPLACE_COUNT_IF(stat_recompile, code != oldcode);
2527                         }
2528
2529                         assert(code);
2530
2531                         /* map this frame */
2532
2533                         rp = replace_find_replacement_point(code, frame, parent);
2534
2535                         frame->tocode = code;
2536                         frame->torp = rp;
2537                 }
2538
2539                 if (rp->type == RPLPOINT_TYPE_CALL) {
2540                         parent = NULL;
2541                 }
2542                 else {
2543                         /* inlining */
2544                         parent = rp;
2545                 }
2546         }
2547
2548         return true;
2549 }
2550
2551
2552 /* replace_build_execution_state_intern ****************************************
2553
2554    Build an execution state for the given (mapped) source state.
2555
2556    !!! CAUTION: This function rewrites the machine stack !!!
2557
2558    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2559
2560    IN:
2561        ss...............the source state. Must have been mapped by
2562                                                 replace_map_source_state before.
2563            es...............the base execution state on which to build
2564
2565    OUT:
2566        *es..............the new execution state
2567
2568 *******************************************************************************/
2569
2570 static void replace_build_execution_state_intern(sourcestate_t *ss,
2571                                                                                                  executionstate_t *es)
2572 {
2573         rplpoint      *rp;
2574         sourceframe_t *prevframe;
2575         rplpoint      *parent;
2576
2577         parent = NULL;
2578         prevframe = NULL;
2579         rp = NULL;
2580
2581         while (ss->frames) {
2582
2583                 if (REPLACE_IS_NATIVE_FRAME(ss->frames)) {
2584                         prevframe = ss->frames;
2585                         replace_push_native_frame(es, ss);
2586                         parent = NULL;
2587                         rp = NULL;
2588                         continue;
2589                 }
2590
2591                 if (parent == NULL) {
2592                         /* create a machine-level stack frame */
2593
2594                         DOLOG( printf("pushing activation record for:\n");
2595                                    if (rp) replace_replacement_point_println(rp, 1);
2596                                    else printf("\tfirst frame\n"); );
2597
2598                         replace_push_activation_record(es, rp, prevframe, ss->frames);
2599
2600                         DOLOG( replace_executionstate_println(es); );
2601                 }
2602
2603                 rp = ss->frames->torp;
2604                 assert(rp);
2605
2606                 DOLOG( printf("creating execution state for%s:\n",
2607                                 (ss->frames->down == NULL) ? " TOPFRAME" : "");
2608                            replace_replacement_point_println(ss->frames->fromrp, 1);
2609                            replace_replacement_point_println(rp, 1); );
2610
2611                 es->code = ss->frames->tocode;
2612                 prevframe = ss->frames;
2613
2614 #if defined(ENABLE_VMLOG)
2615                 vmlog_cacao_rerol_method(ss->frames->method);
2616 #endif
2617
2618                 replace_write_executionstate(rp, es, ss, ss->frames->down == NULL);
2619
2620                 DOLOG( replace_executionstate_println(es); );
2621
2622                 if (rp->type == RPLPOINT_TYPE_CALL) {
2623                         parent = NULL;
2624                 }
2625                 else {
2626                         /* inlining */
2627                         parent = rp;
2628                 }
2629         }
2630 }
2631
2632
2633 /* replace_build_execution_state ***********************************************
2634
2635    This function contains the final phase of replacement. It builds the new
2636    execution state, releases dump memory, and returns to the calling
2637    assembler function which finishes replacement.
2638
2639    NOTE: This function is called from asm_replacement_in, with the stack
2640          pointer at the start of the safe stack area.
2641
2642    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2643
2644    CAUTION: This function and its children must not use a lot of stack!
2645             There are only REPLACE_SAFESTACK_SIZE bytes of C stack
2646                         available.
2647
2648    IN:
2649        st...............the safestack contained the necessary data
2650
2651 *******************************************************************************/
2652
2653 void replace_build_execution_state(replace_safestack_t *st)
2654 {
2655         replace_build_execution_state_intern(st->ss, &(st->es));
2656
2657         DOLOG( replace_executionstate_println(&(st->es)); );
2658
2659         /* release dump area */
2660
2661         dump_release(st->dumpsize);
2662
2663         /* new code is entered after returning */
2664
2665         DOLOG( printf("JUMPING IN!\n"); fflush(stdout); );
2666 }
2667
2668
2669 /* replace_alloc_safestack *****************************************************
2670
2671    Allocate a safe stack area to use during the final phase of replacement.
2672    The returned area is not initialized. This must be done by the caller.
2673
2674    RETURN VALUE:
2675        a newly allocated replace_safestack_t *
2676
2677 *******************************************************************************/
2678
2679 static replace_safestack_t *replace_alloc_safestack()
2680 {
2681         u1 *mem;
2682         replace_safestack_t *st;
2683
2684         mem = MNEW(u1, sizeof(replace_safestack_t) + REPLACE_STACK_ALIGNMENT - 1);
2685
2686         st = (replace_safestack_t *) ((ptrint)(mem + REPLACE_STACK_ALIGNMENT - 1)
2687                                                                                 & ~(REPLACE_STACK_ALIGNMENT - 1));
2688
2689 #if !defined(NDEBUG)
2690         memset(st, 0xa5, sizeof(replace_safestack_t));
2691 #endif
2692
2693         st->mem = mem;
2694
2695         return st;
2696 }
2697
2698
2699 /* replace_free_safestack ******************************************************
2700
2701    Free the given safestack structure, making a copy of the contained
2702    execution state before freeing it.
2703
2704    NOTE: This function is called from asm_replacement_in.
2705
2706    IN:
2707        st...............the safestack to free
2708            tmpes............where to copy the execution state to
2709
2710    OUT:
2711            *tmpes...........receives a copy of st->es
2712
2713 *******************************************************************************/
2714
2715 void replace_free_safestack(replace_safestack_t *st, executionstate_t *tmpes)
2716 {
2717         u1 *mem;
2718
2719         /* copy the executionstate_t to the temporary location */
2720
2721         *tmpes = st->es;
2722
2723         /* get the memory address to free */
2724
2725         mem = st->mem;
2726
2727         /* destroy memory (in debug mode) */
2728
2729 #if !defined(NDEBUG)
2730         memset(st, 0xa5, sizeof(replace_safestack_t));
2731 #endif
2732
2733         /* free the safe stack struct */
2734
2735         MFREE(mem, u1, sizeof(replace_safestack_t) + REPLACE_STACK_ALIGNMENT - 1);
2736 }
2737
2738
2739 /* replace_me_wrapper **********************************************************
2740
2741    TODO: Document me!
2742
2743 *******************************************************************************/
2744
2745 bool replace_me_wrapper(u1 *pc)
2746 {
2747         codeinfo         *code;
2748         rplpoint         *rp;
2749         executionstate_t  es;
2750
2751         /* search the codeinfo for the given PC */
2752
2753         code = code_find_codeinfo_for_pc(pc);
2754         assert(code);
2755
2756         /* search for a replacement point at the given PC */
2757
2758 #if 0
2759         rp = replace_find_replacement_point_for_pc(code, pc);
2760         assert(rp == NULL || rp->pc == pc);
2761 #else
2762         {
2763                 int i;
2764                 rplpoint *rp2;
2765                 rp = NULL;
2766                 for (i=0,rp2=code->rplpoints; i<code->rplpointcount; i++,rp2++) {
2767                         if (rp2->pc == pc)
2768                                 rp = rp2;
2769                 }
2770         }
2771 #endif
2772
2773         /* check if the replacement point is active */
2774
2775         if (rp != NULL && (rp->flags & RPLPOINT_FLAG_ACTIVE)) {
2776
2777                 /*md_replace_executionstate_read(&es, context);*/
2778
2779                 replace_me(rp, &es);
2780
2781                 return true;
2782         }
2783         else
2784                 return false;
2785 }
2786
2787
2788 /* replace_me ******************************************************************
2789
2790    This function is called by asm_replacement_out when a thread reaches
2791    a replacement point. `replace_me` must map the execution state to the
2792    target replacement point and let execution continue there.
2793
2794    This function never returns!
2795
2796    IN:
2797        rp...............replacement point that has been reached
2798            es...............execution state read by asm_replacement_out
2799
2800 *******************************************************************************/
2801
2802 void replace_me(rplpoint *rp, executionstate_t *es)
2803 {
2804         sourcestate_t       *ss;
2805         sourceframe_t       *frame;
2806         s4                   dumpsize;
2807         rplpoint            *origrp;
2808         replace_safestack_t *safestack;
2809
2810         origrp = rp;
2811         es->code = code_find_codeinfo_for_pc(rp->pc);
2812
2813         DOLOG_SHORT( printf("REPLACING(%d %p): (id %d %p) ",
2814                                  stat_replacements, (void*)THREADOBJECT,
2815                                  rp->id, (void*)rp);
2816                                  method_println(es->code->m); );
2817
2818         DOLOG( replace_replacement_point_println(rp, 1);
2819                    replace_executionstate_println(es); );
2820
2821         REPLACE_COUNT(stat_replacements);
2822
2823         /* mark start of dump memory area */
2824
2825         dumpsize = dump_size();
2826
2827         /* recover source state */
2828
2829         ss = replace_recover_source_state(rp, NULL, es);
2830
2831         /* map the source state */
2832
2833         if (!replace_map_source_state(ss))
2834                 vm_abort("exception during method replacement");
2835
2836         DOLOG( replace_sourcestate_println(ss); );
2837
2838         DOLOG_SHORT( replace_sourcestate_println_short(ss); );
2839
2840         /* avoid infinite loops by self-replacement */
2841
2842         frame = ss->frames;
2843         while (frame->down)
2844                 frame = frame->down;
2845
2846         if (frame->torp == origrp) {
2847                 DOLOG_SHORT(
2848                         printf("WARNING: identity replacement, turning off rps to avoid infinite loop\n");
2849                 );
2850                 replace_deactivate_replacement_points(frame->tocode);
2851         }
2852
2853         /* write execution state of new code */
2854
2855         DOLOG( replace_executionstate_println(es); );
2856
2857         /* allocate a safe stack area and copy all needed data there */
2858
2859         safestack = replace_alloc_safestack();
2860
2861         safestack->es = *es;
2862         safestack->ss = ss;
2863         safestack->dumpsize = dumpsize;
2864
2865         /* call the assembler code for the last phase of replacement */
2866
2867 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__) || defined(__S390__)) && defined(ENABLE_JIT)
2868         /*asm_replacement_in(&(safestack->es), safestack);*/
2869 #endif
2870
2871         abort(); /* NOT REACHED */
2872 }
2873
2874
2875 /******************************************************************************/
2876 /* NOTE: No important code below.                                             */
2877 /******************************************************************************/
2878
2879
2880 /* statistics *****************************************************************/
2881
2882 #if defined(REPLACE_STATISTICS)
2883 static void print_freq(FILE *file,int *array,int limit)
2884 {
2885         int i;
2886         int sum = 0;
2887         int cum = 0;
2888         for (i=0; i<limit; ++i)
2889                 sum += array[i];
2890         sum += array[limit];
2891         for (i=0; i<limit; ++i) {
2892                 cum += array[i];
2893                 fprintf(file,"      %3d: %8d (cum %3d%%)\n",
2894                                 i, array[i], (sum) ? ((100*cum)/sum) : 0);
2895         }
2896         fprintf(file,"    >=%3d: %8d\n",limit,array[limit]);
2897 }
2898 #endif /* defined(REPLACE_STATISTICS) */
2899
2900
2901 #if defined(REPLACE_STATISTICS)
2902
2903 #define REPLACE_PRINT_DIST(name, array)                              \
2904     printf("    " name " distribution:\n");                          \
2905     print_freq(stdout, (array), sizeof(array)/sizeof(int) - 1);
2906
2907 void replace_print_statistics(void)
2908 {
2909         printf("replacement statistics:\n");
2910         printf("    # of replacements:   %d\n", stat_replacements);
2911         printf("    # of frames:         %d\n", stat_frames);
2912         printf("    # of recompilations: %d\n", stat_recompile);
2913         printf("    patched static calls:%d\n", stat_staticpatch);
2914         printf("    unrolled inlines:    %d\n", stat_unroll_inline);
2915         printf("    unrolled calls:      %d\n", stat_unroll_call);
2916         REPLACE_PRINT_DIST("frame depth", stat_dist_frames);
2917         REPLACE_PRINT_DIST("locals per frame", stat_dist_locals);
2918         REPLACE_PRINT_DIST("ADR locals per frame", stat_dist_locals_adr);
2919         REPLACE_PRINT_DIST("primitive locals per frame", stat_dist_locals_prim);
2920         REPLACE_PRINT_DIST("RET locals per frame", stat_dist_locals_ret);
2921         REPLACE_PRINT_DIST("void locals per frame", stat_dist_locals_void);
2922         REPLACE_PRINT_DIST("stack slots per frame", stat_dist_stack);
2923         REPLACE_PRINT_DIST("ADR stack slots per frame", stat_dist_stack_adr);
2924         REPLACE_PRINT_DIST("primitive stack slots per frame", stat_dist_stack_prim);
2925         REPLACE_PRINT_DIST("RET stack slots per frame", stat_dist_stack_ret);
2926         printf("\n");
2927         printf("    # of methods:            %d\n", stat_methods);
2928         printf("    # of replacement points: %d\n", stat_rploints);
2929         printf("    # of regallocs:          %d\n", stat_regallocs);
2930         printf("        per rplpoint:        %f\n", (double)stat_regallocs / stat_rploints);
2931         printf("        per method:          %f\n", (double)stat_regallocs / stat_methods);
2932         REPLACE_PRINT_DIST("replacement points per method", stat_dist_method_rplpoints);
2933         printf("\n");
2934
2935 }
2936 #endif /* defined(REPLACE_STATISTICS) */
2937
2938
2939 #if defined(REPLACE_STATISTICS)
2940 static void replace_statistics_source_frame(sourceframe_t *frame)
2941 {
2942         int adr = 0;
2943         int ret = 0;
2944         int prim = 0;
2945         int vd = 0;
2946         int n = 0;
2947         int i;
2948
2949         for (i=0; i<frame->javalocalcount; ++i) {
2950                 switch (frame->javalocaltype[i]) {
2951                         case TYPE_ADR: adr++; break;
2952                         case TYPE_RET: ret++; break;
2953                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
2954                         case TYPE_VOID: vd++; break;
2955                         default: assert(0);
2956                 }
2957                 n++;
2958         }
2959         REPLACE_COUNT_DIST(stat_dist_locals, n);
2960         REPLACE_COUNT_DIST(stat_dist_locals_adr, adr);
2961         REPLACE_COUNT_DIST(stat_dist_locals_void, vd);
2962         REPLACE_COUNT_DIST(stat_dist_locals_ret, ret);
2963         REPLACE_COUNT_DIST(stat_dist_locals_prim, prim);
2964         adr = ret = prim = n = 0;
2965         for (i=0; i<frame->javastackdepth; ++i) {
2966                 switch (frame->javastacktype[i]) {
2967                         case TYPE_ADR: adr++; break;
2968                         case TYPE_RET: ret++; break;
2969                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
2970                 }
2971                 n++;
2972         }
2973         REPLACE_COUNT_DIST(stat_dist_stack, n);
2974         REPLACE_COUNT_DIST(stat_dist_stack_adr, adr);
2975         REPLACE_COUNT_DIST(stat_dist_stack_ret, ret);
2976         REPLACE_COUNT_DIST(stat_dist_stack_prim, prim);
2977 }
2978 #endif /* defined(REPLACE_STATISTICS) */
2979
2980
2981 /* debugging helpers **********************************************************/
2982
2983 /* replace_replacement_point_println *******************************************
2984
2985    Print replacement point info.
2986
2987    IN:
2988        rp...............the replacement point to print
2989
2990 *******************************************************************************/
2991
2992 #if !defined(NDEBUG)
2993
2994 #define TYPECHAR(t)  (((t) >= 0 && (t) <= TYPE_RET) ? show_jit_type_letters[t] : '?')
2995
2996 static char *replace_type_str[] = {
2997         "STD",
2998         "EXH",
2999         "SBR",
3000         "CALL",
3001         "INLINE",
3002         "RETURN",
3003         "BODY"
3004 };
3005
3006 void replace_replacement_point_println(rplpoint *rp, int depth)
3007 {
3008         int j;
3009         int index;
3010
3011         if (!rp) {
3012                 printf("(rplpoint *)NULL\n");
3013                 return;
3014         }
3015
3016         for (j=0; j<depth; ++j)
3017                 putchar('\t');
3018
3019         printf("rplpoint (id %d) %p pc:%p+%d type:%s",
3020                         rp->id, (void*)rp,rp->pc,rp->callsize,
3021                         replace_type_str[rp->type]);
3022         if (rp->flags & RPLPOINT_FLAG_NOTRAP)
3023                 printf(" NOTRAP");
3024         if (rp->flags & RPLPOINT_FLAG_COUNTDOWN)
3025                 printf(" COUNTDOWN");
3026         if (rp->flags & RPLPOINT_FLAG_ACTIVE)
3027                 printf(" ACTIVE");
3028         printf(" parent:%p\n", (void*)rp->parent);
3029         for (j=0; j<depth; ++j)
3030                 putchar('\t');
3031         printf("ra:%d = [",     rp->regalloccount);
3032
3033         for (j=0; j<rp->regalloccount; ++j) {
3034                 if (j)
3035                         putchar(' ');
3036                 index = rp->regalloc[j].index;
3037                 switch (index) {
3038                         case RPLALLOC_STACK: printf("S"); break;
3039                         case RPLALLOC_PARAM: printf("P"); break;
3040                         case RPLALLOC_SYNC : printf("Y"); break;
3041                         default: printf("%d", index);
3042                 }
3043                 printf(":%1c:", TYPECHAR(rp->regalloc[j].type));
3044                 if (rp->regalloc[j].type == TYPE_RET) {
3045                         printf("ret(L%03d)", rp->regalloc[j].regoff);
3046                 }
3047                 else {
3048                         show_allocation(rp->regalloc[j].type, rp->regalloc[j].flags, rp->regalloc[j].regoff);
3049                 }
3050         }
3051
3052         printf("]\n");
3053         for (j=0; j<depth; ++j)
3054                 putchar('\t');
3055         printf("method: ");
3056         method_print(rp->method);
3057
3058         printf("\n");
3059 }
3060 #endif /* !defined(NDEBUG) */
3061
3062
3063 /* replace_show_replacement_points *********************************************
3064
3065    Print replacement point info.
3066
3067    IN:
3068        code.............codeinfo whose replacement points should be printed.
3069
3070 *******************************************************************************/
3071
3072 #if !defined(NDEBUG)
3073 void replace_show_replacement_points(codeinfo *code)
3074 {
3075         int i;
3076         int depth;
3077         rplpoint *rp;
3078         rplpoint *parent;
3079
3080         if (!code) {
3081                 printf("(codeinfo *)NULL\n");
3082                 return;
3083         }
3084
3085         printf("\treplacement points: %d\n",code->rplpointcount);
3086
3087         printf("\ttotal allocations : %d\n",code->regalloccount);
3088         printf("\tsaved int regs    : %d\n",code->savedintcount);
3089         printf("\tsaved flt regs    : %d\n",code->savedfltcount);
3090 #if defined(HAS_ADDRESS_REGISTER_FILE)
3091         printf("\tsaved adr regs    : %d\n",code->savedadrcount);
3092 #endif
3093         printf("\tmemuse            : %d\n",code->memuse);
3094
3095         printf("\n");
3096
3097         for (i=0; i<code->rplpointcount; ++i) {
3098                 rp = code->rplpoints + i;
3099
3100                 depth = 1;
3101                 parent = rp->parent;
3102                 while (parent) {
3103                         depth++;
3104                         parent = parent->parent;
3105                 }
3106                 replace_replacement_point_println(rp, depth);
3107         }
3108 }
3109 #endif
3110
3111
3112 /* replace_executionstate_println **********************************************
3113
3114    Print execution state
3115
3116    IN:
3117        es...............the execution state to print
3118
3119 *******************************************************************************/
3120
3121 #if !defined(NDEBUG)
3122 void replace_executionstate_println(executionstate_t *es)
3123 {
3124         int i;
3125         int slots;
3126         stackslot_t *sp;
3127         int extraslots;
3128
3129         if (!es) {
3130                 printf("(executionstate_t *)NULL\n");
3131                 return;
3132         }
3133
3134         printf("executionstate_t:\n");
3135         printf("\tpc = %p",(void*)es->pc);
3136         printf("  sp = %p",(void*)es->sp);
3137         printf("  pv = %p\n",(void*)es->pv);
3138 #if defined(ENABLE_DISASSEMBLER)
3139         for (i=0; i<INT_REG_CNT; ++i) {
3140                 if (i%4 == 0)
3141                         printf("\t");
3142                 else
3143                         printf(" ");
3144 #if SIZEOF_VOID_P == 8
3145                 printf("%-3s = %016llx",abi_registers_integer_name[i],(unsigned long long)es->intregs[i]);
3146 #else
3147                 printf("%-3s = %08lx",abi_registers_integer_name[i],(unsigned long)es->intregs[i]);
3148 #endif
3149                 if (i%4 == 3)
3150                         printf("\n");
3151         }
3152         for (i=0; i<FLT_REG_CNT; ++i) {
3153                 if (i%4 == 0)
3154                         printf("\t");
3155                 else
3156                         printf(" ");
3157                 printf("F%02d = %016llx",i,(unsigned long long)es->fltregs[i]);
3158                 if (i%4 == 3)
3159                         printf("\n");
3160         }
3161 # if defined(HAS_ADDRESS_REGISTER_FILE)
3162         for (i=0; i<ADR_REG_CNT; ++i) {
3163                 if (i%4 == 0)
3164                         printf("\t");
3165                 else
3166                         printf(" ");
3167                 printf("A%02d = %016llx",i,(unsigned long long)es->adrregs[i]);
3168                 if (i%4 == 3)
3169                         printf("\n");
3170         }
3171 # endif
3172 #endif
3173
3174         sp = (stackslot_t *) es->sp;
3175
3176         extraslots = 2;
3177
3178         if (es->code) {
3179                 methoddesc *md = es->code->m->parseddesc;
3180                 slots = es->code->stackframesize;
3181                 extraslots = 1 + md->memuse;
3182         }
3183         else
3184                 slots = 0;
3185
3186
3187         if (slots) {
3188                 printf("\tstack slots(+%d) at sp:", extraslots);
3189                 for (i=0; i<slots+extraslots; ++i) {
3190                         if (i%4 == 0)
3191                                 printf("\n\t\t");
3192                         printf("M%02d%c", i, (i >= slots) ? '(' : ' ');
3193 #ifdef HAS_4BYTE_STACKSLOT
3194                         printf("%08lx",(unsigned long)*sp++);
3195 #else
3196                         printf("%016llx",(unsigned long long)*sp++);
3197 #endif
3198                         printf("%c", (i >= slots) ? ')' : ' ');
3199                 }
3200                 printf("\n");
3201         }
3202
3203         printf("\tcode: %p", (void*)es->code);
3204         if (es->code != NULL) {
3205                 printf(" stackframesize=%d ", es->code->stackframesize);
3206                 method_print(es->code->m);
3207         }
3208         printf("\n");
3209
3210         printf("\n");
3211 }
3212 #endif
3213
3214 #if !defined(NDEBUG)
3215 static void java_value_print(s4 type, replace_val_t value)
3216 {
3217         java_object_t *obj;
3218         utf           *u;
3219
3220         printf("%016llx",(unsigned long long) value.l);
3221
3222         if (type < 0 || type > TYPE_RET)
3223                 printf(" <INVALID TYPE:%d>", type);
3224         else
3225                 printf(" %s", show_jit_type_names[type]);
3226
3227         if (type == TYPE_ADR && value.a != NULL) {
3228                 obj = value.a;
3229                 putchar(' ');
3230                 utf_display_printable_ascii_classname(obj->vftbl->class->name);
3231
3232                 if (obj->vftbl->class == class_java_lang_String) {
3233                         printf(" \"");
3234                         u = javastring_toutf(obj, false);
3235                         utf_display_printable_ascii(u);
3236                         printf("\"");
3237                 }
3238         }
3239         else if (type == TYPE_INT) {
3240                 printf(" %ld", (long) value.i);
3241         }
3242         else if (type == TYPE_LNG) {
3243                 printf(" %lld", (long long) value.l);
3244         }
3245         else if (type == TYPE_FLT) {
3246                 printf(" %f", value.f);
3247         }
3248         else if (type == TYPE_DBL) {
3249                 printf(" %f", value.d);
3250         }
3251 }
3252 #endif /* !defined(NDEBUG) */
3253
3254
3255 #if !defined(NDEBUG)
3256 void replace_source_frame_println(sourceframe_t *frame)
3257 {
3258         s4 i,j;
3259         s4 t;
3260
3261         if (REPLACE_IS_NATIVE_FRAME(frame)) {
3262                 printf("\tNATIVE\n");
3263                 printf("\tsfi: "); replace_stackframeinfo_println(frame->sfi);
3264                 printf("\tnativepc: %p\n", frame->nativepc);
3265                 printf("\tframesize: %d\n", frame->nativeframesize);
3266
3267                 j = 0;
3268                 for (i=0; i<INT_REG_CNT; ++i) {
3269                         if (nregdescint[i] == REG_SAV)
3270                                 printf("\t%s = %p\n", abi_registers_integer_name[i], (void*)frame->nativesavint[j++]);
3271                 }
3272
3273                 j = 0;
3274                 for (i=0; i<FLT_REG_CNT; ++i) {
3275                         if (nregdescfloat[i] == REG_SAV)
3276                                 printf("\tF%02d = %f\n", i, frame->nativesavflt[j++]);
3277                 }
3278
3279                 printf("\n");
3280                 return;
3281         }
3282
3283         printf("\t");
3284         method_println(frame->method);
3285         printf("\tid: %d\n", frame->id);
3286         printf("\ttype: %s\n", replace_type_str[frame->type]);
3287         printf("\n");
3288
3289         if (frame->instance.a) {
3290                 printf("\tinstance: ");
3291                 java_value_print(TYPE_ADR, frame->instance);
3292                 printf("\n");
3293         }
3294
3295         if (frame->javalocalcount) {
3296                 printf("\tlocals (%d):\n",frame->javalocalcount);
3297                 for (i=0; i<frame->javalocalcount; ++i) {
3298                         t = frame->javalocaltype[i];
3299                         if (t == TYPE_VOID) {
3300                                 printf("\tlocal[ %2d] = void\n",i);
3301                         }
3302                         else {
3303                                 printf("\tlocal[%c%2d] = ",TYPECHAR(t),i);
3304                                 java_value_print(t, frame->javalocals[i]);
3305                                 printf("\n");
3306                         }
3307                 }
3308                 printf("\n");
3309         }
3310
3311         if (frame->javastackdepth) {
3312                 printf("\tstack (depth %d):\n",frame->javastackdepth);
3313                 for (i=0; i<frame->javastackdepth; ++i) {
3314                         t = frame->javastacktype[i];
3315                         if (t == TYPE_VOID) {
3316                                 printf("\tstack[%2d] = void", i);
3317                         }
3318                         else {
3319                                 printf("\tstack[%2d] = ",i);
3320                                 java_value_print(frame->javastacktype[i], frame->javastack[i]);
3321                                 printf("\n");
3322                         }
3323                 }
3324                 printf("\n");
3325         }
3326
3327         if (frame->syncslotcount) {
3328                 printf("\tsynchronization slots (%d):\n",frame->syncslotcount);
3329                 for (i=0; i<frame->syncslotcount; ++i) {
3330                         printf("\tslot[%2d] = ",i);
3331 #ifdef HAS_4BYTE_STACKSLOT
3332                         printf("%08lx\n",(unsigned long) frame->syncslots[i].p);
3333 #else
3334                         printf("%016llx\n",(unsigned long long) frame->syncslots[i].p);
3335 #endif
3336                 }
3337                 printf("\n");
3338         }
3339
3340         if (frame->fromcode) {
3341                 printf("\tfrom %p ", (void*)frame->fromcode);
3342                 method_println(frame->fromcode->m);
3343         }
3344         if (frame->tocode) {
3345                 printf("\tto %p ", (void*)frame->tocode);
3346                 method_println(frame->tocode->m);
3347         }
3348
3349         if (frame->fromrp) {
3350                 printf("\tfrom replacement point:\n");
3351                 replace_replacement_point_println(frame->fromrp, 2);
3352         }
3353         if (frame->torp) {
3354                 printf("\tto replacement point:\n");
3355                 replace_replacement_point_println(frame->torp, 2);
3356         }
3357
3358         printf("\n");
3359 }
3360 #endif /* !defined(NDEBUG) */
3361
3362
3363 /* replace_sourcestate_println *************************************************
3364
3365    Print source state
3366
3367    IN:
3368        ss...............the source state to print
3369
3370 *******************************************************************************/
3371
3372 #if !defined(NDEBUG)
3373 void replace_sourcestate_println(sourcestate_t *ss)
3374 {
3375         int i;
3376         sourceframe_t *frame;
3377
3378         if (!ss) {
3379                 printf("(sourcestate_t *)NULL\n");
3380                 return;
3381         }
3382
3383         printf("sourcestate_t:\n");
3384
3385         for (i=0, frame = ss->frames; frame != NULL; frame = frame->down, ++i) {
3386                 printf("    frame %d:\n", i);
3387                 replace_source_frame_println(frame);
3388         }
3389 }
3390 #endif
3391
3392
3393 /* replace_sourcestate_println_short *******************************************
3394
3395    Print a compact representation of the given source state.
3396
3397    IN:
3398        ss...............the source state to print
3399
3400 *******************************************************************************/
3401
3402 #if !defined(NDEBUG)
3403 void replace_sourcestate_println_short(sourcestate_t *ss)
3404 {
3405         sourceframe_t *frame;
3406
3407         for (frame = ss->frames; frame != NULL; frame = frame->down) {
3408                 printf("\t");
3409
3410                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
3411                         printf("NATIVE (pc %p size %d) ",
3412                                         (void*)frame->nativepc, frame->nativeframesize);
3413                         replace_stackframeinfo_println(frame->sfi);
3414                         continue;
3415                 }
3416
3417                 if (frame->torp) {
3418                         printf("%c", (frame->torp == frame->fromrp) ? '=' : '+');
3419                 }
3420
3421                 printf("%s", replace_type_str[frame->fromrp->type]);
3422
3423                 if (frame->torp && frame->torp->type != frame->fromrp->type)
3424                         printf("->%s", replace_type_str[frame->torp->type]);
3425
3426                 if (frame->tocode != frame->fromcode)
3427                         printf(" (%p->%p/%d) ",
3428                                    (void*) frame->fromcode, (void*) frame->tocode,
3429                                    frame->fromrp->id);
3430                 else
3431                         printf(" (%p/%d) ", (void*) frame->fromcode, frame->fromrp->id);
3432
3433                 method_println(frame->method);
3434         }
3435 }
3436 #endif
3437
3438 #if !defined(NDEBUG)
3439 static void replace_stackframeinfo_println(stackframeinfo *sfi)
3440 {
3441         printf("prev=%p pv=%p sp=%p ra=%p xpc=%p method=",
3442                         (void*)sfi->prev, (void*)sfi->pv, (void*)sfi->sp,
3443                         (void*)sfi->ra, (void*)sfi->xpc);
3444
3445         if (sfi->method)
3446                 method_println(sfi->method);
3447         else
3448                 printf("(nil)\n");
3449 }
3450 #endif
3451
3452 /*
3453  * These are local overrides for various environment variables in Emacs.
3454  * Please do not remove this and leave it at the end of the file, where
3455  * Emacs will automagically detect them.
3456  * ---------------------------------------------------------------------
3457  * Local variables:
3458  * mode: c
3459  * indent-tabs-mode: t
3460  * c-basic-offset: 4
3461  * tab-width: 4
3462  * End:
3463  * vim:noexpandtab:sw=4:ts=4:
3464  */