src/vm/jit/replace.c

   1 /* src/vm/jit/replace.c - on-stack replacement of methods
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id$
  26
  27 */
  28
  29 #include "config.h"
  30 #include "vm/types.h"
  31
  32 #include <assert.h>
  33 #include <stdlib.h>
  34
  35 #include "arch.h"
  36
  37 #if defined(ENABLE_GC_CACAO)
  38 # include "mm/cacao-gc/gc.h"
  39 #endif
  40
  41 #include "mm/memory.h"
  42
  43 #include "threads/threads-common.h"
  44
  45 #include "toolbox/logging.h"
  46
  47 #include "vm/stringlocal.h"
  48
  49 #include "vm/jit/abi.h"
  50 #include "vm/jit/asmpart.h"
  51 #include "vm/jit/disass.h"
  52 #include "vm/jit/jit.h"
  53 #include "vm/jit/md.h"
  54 #include "vm/jit/methodheader.h"
  55 #include "vm/jit/replace.h"
  56 #include "vm/jit/show.h"
  57 #include "vm/jit/stack.h"
  58
  59 #include "vmcore/options.h"
  60 #include "vmcore/classcache.h"
  61
  62
  63 #define REPLACE_PATCH_DYNAMIC_CALL
  64 /*#define REPLACE_PATCH_ALL*/
  65
  66
  67 /*** architecture-dependent configuration *************************************/
  68
  69 /* first unset the macros (default) */
  70 #undef REPLACE_RA_BETWEEN_FRAMES
  71 #undef REPLACE_RA_TOP_OF_FRAME
  72 #undef REPLACE_RA_LINKAGE_AREA
  73 #undef REPLACE_LEAFMETHODS_RA_REGISTER
  74 #undef REPLACE_REG_RA
  75
  76 /* i386 and x86_64 */
  77 #if defined(__I386__) || defined(__X86_64__)
  78 #define REPLACE_RA_BETWEEN_FRAMES
  79 /* alpha */
  80 #elif defined(__ALPHA__)
  81 #define REPLACE_RA_TOP_OF_FRAME
  82 #define REPLACE_LEAFMETHODS_RA_REGISTER
  83 #define REPLACE_REG_RA REG_RA
  84 /* powerpc */
  85 #elif defined(__POWERPC__)
  86 #define REPLACE_RA_LINKAGE_AREA
  87 #define REPLACE_LEAFMETHODS_RA_REGISTER
  88 #define REPLACE_REG_RA REG_ITMP3 /* the execution state has the LR in itmp3 */
  89 #endif
  90
  91
  92 /*** configuration of native stack slot size **********************************/
  93
  94 /* XXX this should be in md-abi.h files, probably */
  95
  96 #if defined(HAS_4BYTE_STACKSLOT)
  97 #define SIZE_OF_STACKSLOT      4
  98 #define STACK_SLOTS_PER_FLOAT  2
  99 typedef u4 stackslot_t;
 100 #else
 101 #define SIZE_OF_STACKSLOT      8
 102 #define STACK_SLOTS_PER_FLOAT  1
 103 typedef u8 stackslot_t;
 104 #endif
 105
 106
 107 /*** debugging ****************************************************************/
 108
 109 /*#define REPLACE_VERBOSE*/
 110
 111 #if !defined(NDEBUG)
 112 static void java_value_print(s4 type, replace_val_t value);
 113 static void replace_stackframeinfo_println(stackframeinfo *sfi);
 114 #endif
 115
 116 #if !defined(NDEBUG) && defined(REPLACE_VERBOSE)
 117 int replace_verbose = 0;
 118 #define DOLOG(code)        do{ if (replace_verbose > 1) { code; } } while(0)
 119 #define DOLOG_SHORT(code)  do{ if (replace_verbose > 0) { code; } } while(0)
 120 #else
 121 #define DOLOG(code)
 122 #define DOLOG_SHORT(code)
 123 #endif
 124
 125
 126 /*** statistics ***************************************************************/
 127
 128 #define REPLACE_STATISTICS
 129
 130 #if defined(REPLACE_STATISTICS)
 131
 132 static int stat_replacements = 0;
 133 static int stat_frames = 0;
 134 static int stat_recompile = 0;
 135 static int stat_staticpatch = 0;
 136 static int stat_unroll_inline = 0;
 137 static int stat_unroll_call = 0;
 138 static int stat_dist_frames[20] = { 0 };
 139 static int stat_dist_locals[20] = { 0 };
 140 static int stat_dist_locals_adr[10] = { 0 };
 141 static int stat_dist_locals_prim[10] = { 0 };
 142 static int stat_dist_locals_ret[10] = { 0 };
 143 static int stat_dist_locals_void[10] = { 0 };
 144 static int stat_dist_stack[10] = { 0 };
 145 static int stat_dist_stack_adr[10] = { 0 };
 146 static int stat_dist_stack_prim[10] = { 0 };
 147 static int stat_dist_stack_ret[10] = { 0 };
 148 static int stat_methods = 0;
 149 static int stat_rploints = 0;
 150 static int stat_regallocs = 0;
 151 static int stat_dist_method_rplpoints[20] = { 0 };
 152
 153 #define REPLACE_COUNT(cnt)  (cnt)++
 154 #define REPLACE_COUNT_IF(cnt, cond)  do{ if(cond) (cnt)++; } while(0)
 155 #define REPLACE_COUNT_INC(cnt, inc)  ((cnt) += (inc))
 156
 157 #define REPLACE_COUNT_DIST(array, val)                               \
 158     do {                                                             \
 159         int limit = (sizeof(array) / sizeof(int)) - 1;               \
 160         if ((val) < (limit)) (array)[val]++;                         \
 161         else (array)[limit]++;                                       \
 162     } while (0)
 163
 164 static void replace_statistics_source_frame(sourceframe_t *frame);
 165
 166 #else
 167
 168 #define REPLACE_COUNT(cnt)
 169 #define REPLACE_COUNT_IF(cnt, cond)
 170 #define REPLACE_COUNT_INC(cnt, inc)
 171 #define REPLACE_COUNT_DIST(array, val)
 172
 173 #endif /* defined(REPLACE_STATISTICS) */
 174
 175
 176 /*** constants used internally ************************************************/
 177
 178 #define TOP_IS_NORMAL    0
 179 #define TOP_IS_ON_STACK  1
 180 #define TOP_IS_IN_ITMP1  2
 181 #define TOP_IS_VOID      3
 182
 183
 184 /******************************************************************************/
 185 /* PART I: Creating / freeing replacement points                              */
 186 /******************************************************************************/
 187
 188
 189 /* replace_create_replacement_point ********************************************
 190
 191    Create a replacement point.
 192
 193    IN:
 194        jd...............current jitdata
 195            iinfo............inlining info for the current position
 196            rp...............pre-allocated (uninitialized) rplpoint
 197            type.............RPLPOINT_TYPE constant
 198            iptr.............current instruction
 199            *pra.............current rplalloc pointer
 200            javalocals.......the javalocals at the current point
 201            stackvars........the stack variables at the current point
 202            stackdepth.......the stack depth at the current point
 203            paramcount.......number of parameters at the start of stackvars
 204
 205    OUT:
 206        *rpa.............points to the next free rplalloc
 207
 208 *******************************************************************************/
 209
 210 static void replace_create_replacement_point(jitdata *jd,
 211                                                                                          insinfo_inline *iinfo,
 212                                                                                          rplpoint *rp,
 213                                                                                          s4 type,
 214                                                                                          instruction *iptr,
 215                                                                                          rplalloc **pra,
 216                                                                                          s4 *javalocals,
 217                                                                                          s4 *stackvars,
 218                                                                                          s4 stackdepth,
 219                                                                                          s4 paramcount)
 220 {
 221         rplalloc *ra;
 222         s4        i;
 223         varinfo  *v;
 224         s4        index;
 225
 226         ra = *pra;
 227
 228         REPLACE_COUNT(stat_rploints);
 229
 230         rp->method = (iinfo) ? iinfo->method : jd->m;
 231         rp->pc = NULL;        /* set by codegen */
 232         rp->callsize = 0;     /* set by codegen */
 233         rp->regalloc = ra;
 234         rp->flags = 0;
 235         rp->type = type;
 236         rp->id = iptr->flags.bits >> INS_FLAG_ID_SHIFT;
 237
 238         /* XXX unify these two fields */
 239         rp->parent = (iinfo) ? iinfo->rp : NULL;
 240
 241         /* store local allocation info of javalocals */
 242
 243         if (javalocals) {
 244                 for (i = 0; i < rp->method->maxlocals; ++i) {
 245                         index = javalocals[i];
 246                         if (index == UNUSED)
 247                                 continue;
 248
 249                         ra->index = i;
 250                         if (index < UNUSED) {
 251                                 ra->regoff = (UNUSED - index) - 1;
 252                                 ra->type = TYPE_RET;
 253                                 ra->flags = 0;
 254                         }
 255                         else {
 256                                 v = VAR(index);
 257                                 ra->flags = v->flags & (INMEMORY);
 258                                 ra->regoff = v->vv.regoff;
 259                                 ra->type = v->type;
 260                         }
 261                         ra++;
 262                 }
 263         }
 264
 265         /* store allocation info of java stack vars */
 266
 267         for (i = 0; i < stackdepth; ++i) {
 268                 v = VAR(stackvars[i]);
 269                 ra->flags = v->flags & (INMEMORY);
 270                 ra->index = (i < paramcount) ? RPLALLOC_PARAM : RPLALLOC_STACK;
 271                 ra->type  = v->type;
 272                 /* XXX how to handle locals on the stack containing returnAddresses? */
 273                 if (v->type == TYPE_RET) {
 274                         assert(stackvars[i] >= jd->localcount);
 275                         ra->regoff = v->vv.retaddr->nr;
 276                 }
 277                 else
 278                         ra->regoff = v->vv.regoff;
 279                 ra++;
 280         }
 281
 282         /* total number of allocations */
 283
 284         rp->regalloccount = ra - rp->regalloc;
 285
 286         *pra = ra;
 287 }
 288
 289
 290 /* replace_create_inline_start_replacement_point *******************************
 291
 292    Create an INLINE_START replacement point.
 293
 294    IN:
 295        jd...............current jitdata
 296            rp...............pre-allocated (uninitialized) rplpoint
 297            iptr.............current instruction
 298            *pra.............current rplalloc pointer
 299            javalocals.......the javalocals at the current point
 300
 301    OUT:
 302        *rpa.............points to the next free rplalloc
 303
 304    RETURN VALUE:
 305        the insinfo_inline * for the following inlined body
 306
 307 *******************************************************************************/
 308
 309 static insinfo_inline * replace_create_inline_start_replacement_point(
 310                                                                                          jitdata *jd,
 311                                                                                          rplpoint *rp,
 312                                                                                          instruction *iptr,
 313                                                                                          rplalloc **pra,
 314                                                                                          s4 *javalocals)
 315 {
 316         insinfo_inline *calleeinfo;
 317         rplalloc       *ra;
 318
 319         calleeinfo = iptr->sx.s23.s3.inlineinfo;
 320
 321         calleeinfo->rp = rp;
 322
 323         replace_create_replacement_point(jd, calleeinfo->parent, rp,
 324                         RPLPOINT_TYPE_INLINE, iptr, pra,
 325                         javalocals,
 326                         calleeinfo->stackvars, calleeinfo->stackvarscount,
 327                         calleeinfo->paramcount);
 328
 329         if (calleeinfo->synclocal != UNUSED) {
 330                 ra = (*pra)++;
 331                 ra->index  = RPLALLOC_SYNC;
 332                 ra->regoff = jd->var[calleeinfo->synclocal].vv.regoff;
 333                 ra->flags  = jd->var[calleeinfo->synclocal].flags & INMEMORY;
 334                 ra->type   = TYPE_ADR;
 335
 336                 rp->regalloccount++;
 337         }
 338
 339         return calleeinfo;
 340 }
 341
 342
 343 /* replace_create_replacement_points *******************************************
 344
 345    Create the replacement points for the given code.
 346
 347    IN:
 348        jd...............current jitdata, must not have any replacement points
 349
 350    OUT:
 351        code->rplpoints.......set to the list of replacement points
 352            code->rplpointcount...number of replacement points
 353            code->regalloc........list of allocation info
 354            code->regalloccount...total length of allocation info list
 355            code->globalcount.....number of global allocations at the
 356                                  start of code->regalloc
 357
 358    RETURN VALUE:
 359        true.............everything ok
 360        false............an exception has been thrown
 361
 362 *******************************************************************************/
 363
 364 #define CLEAR_javalocals(array, method)                              \
 365     do {                                                             \
 366         for (i=0; i<(method)->maxlocals; ++i)                        \
 367             (array)[i] = UNUSED;                                     \
 368     } while (0)
 369
 370 #define COPY_OR_CLEAR_javalocals(dest, array, method)                \
 371     do {                                                             \
 372         if ((array) != NULL)                                         \
 373             MCOPY((dest), (array), s4, (method)->maxlocals);         \
 374         else                                                         \
 375             CLEAR_javalocals((dest), (method));                      \
 376     } while (0)
 377
 378 #define COUNT_javalocals(array, method, counter)                     \
 379     do {                                                             \
 380         for (i=0; i<(method)->maxlocals; ++i)                        \
 381             if ((array)[i] != UNUSED)                                \
 382                                 (counter)++;                                         \
 383     } while (0)
 384
 385 bool replace_create_replacement_points(jitdata *jd)
 386 {
 387         codeinfo        *code;
 388         registerdata    *rd;
 389         basicblock      *bptr;
 390         int              count;
 391         methodinfo      *m;
 392         rplpoint        *rplpoints;
 393         rplpoint        *rp;
 394         int              alloccount;
 395         rplalloc        *regalloc;
 396         rplalloc        *ra;
 397         int              i;
 398         instruction     *iptr;
 399         instruction     *iend;
 400         s4              *javalocals;
 401         s4              *jl;
 402         methoddesc      *md;
 403         insinfo_inline  *iinfo;
 404         s4               startcount;
 405         s4               firstcount;
 406 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 407         bool             needentry;
 408 #endif
 409
 410         REPLACE_COUNT(stat_methods);
 411
 412         /* get required compiler data */
 413
 414         code = jd->code;
 415         rd   = jd->rd;
 416
 417         /* assert that we wont overwrite already allocated data */
 418
 419         assert(code);
 420         assert(code->m);
 421         assert(code->rplpoints == NULL);
 422         assert(code->rplpointcount == 0);
 423         assert(code->regalloc == NULL);
 424         assert(code->regalloccount == 0);
 425         assert(code->globalcount == 0);
 426
 427         m = code->m;
 428
 429         /* set codeinfo flags */
 430
 431         if (jd->isleafmethod)
 432                 CODE_SETFLAG_LEAFMETHOD(code);
 433
 434         /* in instance methods, we may need a rplpoint at the method entry */
 435
 436 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 437         if (!(m->flags & ACC_STATIC)) {
 438                 jd->basicblocks[0].bitflags |= BBFLAG_REPLACEMENT;
 439                 needentry = true;
 440         }
 441         else {
 442                 needentry = false;
 443         }
 444 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
 445
 446         /* iterate over the basic block list to find replacement points */
 447
 448         count = 0;
 449         alloccount = 0;
 450
 451         javalocals = DMNEW(s4, jd->maxlocals);
 452
 453         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 454
 455                 /* skip dead code */
 456
 457                 if (bptr->flags < BBFINISHED)
 458                         continue;
 459
 460                 /* get info about this block */
 461
 462                 m = bptr->method;
 463                 iinfo = bptr->inlineinfo;
 464
 465                 /* initialize javalocals at the start of this block */
 466
 467                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 468
 469                 /* iterate over the instructions */
 470
 471                 iptr = bptr->iinstr;
 472                 iend = iptr + bptr->icount;
 473                 startcount = count;
 474                 firstcount = count;
 475
 476                 for (; iptr != iend; ++iptr) {
 477                         switch (iptr->opc) {
 478                                 case ICMD_INVOKESTATIC:
 479                                 case ICMD_INVOKESPECIAL:
 480                                 case ICMD_INVOKEVIRTUAL:
 481                                 case ICMD_INVOKEINTERFACE:
 482                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 483                                         count++;
 484                                         COUNT_javalocals(javalocals, m, alloccount);
 485                                         alloccount += iptr->s1.argcount;
 486                                         if (iinfo)
 487                                                 alloccount -= iinfo->throughcount;
 488                                         break;
 489
 490                                 case ICMD_ISTORE:
 491                                 case ICMD_LSTORE:
 492                                 case ICMD_FSTORE:
 493                                 case ICMD_DSTORE:
 494                                 case ICMD_ASTORE:
 495                                         stack_javalocals_store(iptr, javalocals);
 496                                         break;
 497
 498                                 case ICMD_IRETURN:
 499                                 case ICMD_LRETURN:
 500                                 case ICMD_FRETURN:
 501                                 case ICMD_DRETURN:
 502                                 case ICMD_ARETURN:
 503                                         alloccount += 1;
 504                                         /* FALLTHROUGH! */
 505                                 case ICMD_RETURN:
 506                                         count++;
 507                                         break;
 508
 509                                 case ICMD_INLINE_START:
 510                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 511
 512                                         count++;
 513                                         COUNT_javalocals(javalocals, m, alloccount);
 514                                         alloccount += iinfo->stackvarscount;
 515                                         if (iinfo->synclocal != UNUSED)
 516                                                 alloccount++;
 517
 518                                         m = iinfo->method;
 519                                         /* javalocals may be set at next block start, or now */
 520                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 521                                         break;
 522
 523                                 case ICMD_INLINE_BODY:
 524                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 525
 526                                         jl = iinfo->javalocals_start;
 527                                         if (jl == NULL) {
 528                                                 /* get the javalocals from the following block start */
 529                                                 assert(bptr->next);
 530                                                 jl = bptr->next->javalocals;
 531                                         }
 532                                         count++;
 533                                         COUNT_javalocals(jl, m, alloccount);
 534                                         break;
 535
 536                                 case ICMD_INLINE_END:
 537                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 538                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 539                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 540                                         m = iinfo->outer;
 541                                         if (iinfo->javalocals_end)
 542                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 543                                         iinfo = iinfo->parent;
 544                                         break;
 545                         }
 546
 547                         if (iptr == bptr->iinstr)
 548                                 firstcount = count;
 549                 } /* end instruction loop */
 550
 551                 /* create replacement points at targets of backward branches */
 552                 /* We only need the replacement point there, if there is no  */
 553                 /* replacement point inside the block.                       */
 554
 555                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 556 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 557                         int test = (needentry && bptr == jd->basicblocks) ? firstcount : count;
 558 #else
 559                         int test = count;
 560 #endif
 561                         if (test > startcount) {
 562                                 /* we don't need an extra rplpoint */
 563                                 bptr->bitflags &= ~BBFLAG_REPLACEMENT;
 564                         }
 565                         else {
 566                                 count++;
 567                                 alloccount += bptr->indepth;
 568                                 if (bptr->inlineinfo)
 569                                         alloccount -= bptr->inlineinfo->throughcount;
 570
 571                                 COUNT_javalocals(bptr->javalocals, bptr->method, alloccount);
 572                         }
 573                 }
 574
 575         } /* end basicblock loop */
 576
 577         /* if no points were found, there's nothing to do */
 578
 579         if (!count)
 580                 return true;
 581
 582         /* allocate replacement point array and allocation array */
 583
 584         rplpoints = MNEW(rplpoint, count);
 585         regalloc = MNEW(rplalloc, alloccount);
 586         ra = regalloc;
 587
 588         /* initialize replacement point structs */
 589
 590         rp = rplpoints;
 591
 592         /* XXX try to share code with the counting loop! */
 593
 594         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 595                 /* skip dead code */
 596
 597                 if (bptr->flags < BBFINISHED)
 598                         continue;
 599
 600                 /* get info about this block */
 601
 602                 m = bptr->method;
 603                 iinfo = bptr->inlineinfo;
 604
 605                 /* initialize javalocals at the start of this block */
 606
 607                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 608
 609                 /* create replacement points at targets of backward branches */
 610
 611                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 612
 613                         i = (iinfo) ? iinfo->throughcount : 0;
 614                         replace_create_replacement_point(jd, iinfo, rp++,
 615                                         bptr->type, bptr->iinstr, &ra,
 616                                         bptr->javalocals, bptr->invars + i, bptr->indepth - i, 0);
 617
 618                         if (JITDATA_HAS_FLAG_COUNTDOWN(jd))
 619                                 rp[-1].flags |= RPLPOINT_FLAG_COUNTDOWN;
 620                 }
 621
 622                 /* iterate over the instructions */
 623
 624                 iptr = bptr->iinstr;
 625                 iend = iptr + bptr->icount;
 626
 627                 for (; iptr != iend; ++iptr) {
 628                         switch (iptr->opc) {
 629                                 case ICMD_INVOKESTATIC:
 630                                 case ICMD_INVOKESPECIAL:
 631                                 case ICMD_INVOKEVIRTUAL:
 632                                 case ICMD_INVOKEINTERFACE:
 633                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 634
 635                                         i = (iinfo) ? iinfo->throughcount : 0;
 636                                         replace_create_replacement_point(jd, iinfo, rp++,
 637                                                         RPLPOINT_TYPE_CALL, iptr, &ra,
 638                                                         javalocals, iptr->sx.s23.s2.args,
 639                                                         iptr->s1.argcount - i,
 640                                                         md->paramcount);
 641                                         break;
 642
 643                                 case ICMD_ISTORE:
 644                                 case ICMD_LSTORE:
 645                                 case ICMD_FSTORE:
 646                                 case ICMD_DSTORE:
 647                                 case ICMD_ASTORE:
 648                                         stack_javalocals_store(iptr, javalocals);
 649                                         break;
 650
 651                                 case ICMD_IRETURN:
 652                                 case ICMD_LRETURN:
 653                                 case ICMD_FRETURN:
 654                                 case ICMD_DRETURN:
 655                                 case ICMD_ARETURN:
 656                                         replace_create_replacement_point(jd, iinfo, rp++,
 657                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 658                                                         NULL, &(iptr->s1.varindex), 1, 0);
 659                                         break;
 660
 661                                 case ICMD_RETURN:
 662                                         replace_create_replacement_point(jd, iinfo, rp++,
 663                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 664                                                         NULL, NULL, 0, 0);
 665                                         break;
 666
 667                                 case ICMD_INLINE_START:
 668                                         iinfo = replace_create_inline_start_replacement_point(
 669                                                                 jd, rp++, iptr, &ra, javalocals);
 670                                         m = iinfo->method;
 671                                         /* javalocals may be set at next block start, or now */
 672                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 673                                         break;
 674
 675                                 case ICMD_INLINE_BODY:
 676                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 677
 678                                         jl = iinfo->javalocals_start;
 679                                         if (jl == NULL) {
 680                                                 /* get the javalocals from the following block start */
 681                                                 assert(bptr->next);
 682                                                 jl = bptr->next->javalocals;
 683                                         }
 684                                         /* create a non-trappable rplpoint */
 685                                         replace_create_replacement_point(jd, iinfo, rp++,
 686                                                         RPLPOINT_TYPE_BODY, iptr, &ra,
 687                                                         jl, NULL, 0, 0);
 688                                         rp[-1].flags |= RPLPOINT_FLAG_NOTRAP;
 689                                         break;
 690
 691                                 case ICMD_INLINE_END:
 692                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 693                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 694                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 695                                         m = iinfo->outer;
 696                                         if (iinfo->javalocals_end)
 697                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 698                                         iinfo = iinfo->parent;
 699                                         break;
 700                         }
 701                 } /* end instruction loop */
 702         } /* end basicblock loop */
 703
 704         assert((rp - rplpoints) == count);
 705         assert((ra - regalloc) == alloccount);
 706
 707         /* store the data in the codeinfo */
 708
 709         code->rplpoints     = rplpoints;
 710         code->rplpointcount = count;
 711         code->regalloc      = regalloc;
 712         code->regalloccount = alloccount;
 713         code->globalcount   = 0;
 714         code->savedintcount = INT_SAV_CNT - rd->savintreguse;
 715         code->savedfltcount = FLT_SAV_CNT - rd->savfltreguse;
 716         code->memuse        = rd->memuse;
 717         code->stackframesize = jd->cd->stackframesize;
 718
 719         REPLACE_COUNT_DIST(stat_dist_method_rplpoints, count);
 720         REPLACE_COUNT_INC(stat_regallocs, alloccount);
 721
 722         /* everything alright */
 723
 724         return true;
 725 }
 726
 727
 728 /* replace_free_replacement_points *********************************************
 729
 730    Free memory used by replacement points.
 731
 732    IN:
 733        code.............codeinfo whose replacement points should be freed.
 734
 735 *******************************************************************************/
 736
 737 void replace_free_replacement_points(codeinfo *code)
 738 {
 739         assert(code);
 740
 741         if (code->rplpoints)
 742                 MFREE(code->rplpoints,rplpoint,code->rplpointcount);
 743
 744         if (code->regalloc)
 745                 MFREE(code->regalloc,rplalloc,code->regalloccount);
 746
 747         code->rplpoints = NULL;
 748         code->rplpointcount = 0;
 749         code->regalloc = NULL;
 750         code->regalloccount = 0;
 751         code->globalcount = 0;
 752 }
 753
 754
 755 /******************************************************************************/
 756 /* PART II: Activating / deactivating replacement points                      */
 757 /******************************************************************************/
 758
 759
 760 /* replace_activate_replacement_points *****************************************
 761
 762    Activate the replacement points of the given compilation unit. When this
 763    function returns, the replacement points are "armed", so each thread
 764    reaching one of the points will enter the replacement mechanism.
 765
 766    IN:
 767        code.............codeinfo of which replacement points should be
 768                                                 activated
 769            mappable.........if true, only mappable replacement points are
 770                                                 activated
 771
 772 *******************************************************************************/
 773
 774 void replace_activate_replacement_points(codeinfo *code, bool mappable)
 775 {
 776         rplpoint *rp;
 777         s4        i;
 778         s4        count;
 779         s4        index;
 780         u1       *savedmcode;
 781
 782         assert(code->savedmcode == NULL);
 783
 784         /* count trappable replacement points */
 785
 786         count = 0;
 787         index = 0;
 788         i = code->rplpointcount;
 789         rp = code->rplpoints;
 790         for (; i--; rp++) {
 791                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 792                         continue;
 793
 794                 index++;
 795
 796                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 797                         continue;
 798
 799                 count++;
 800         }
 801
 802         /* allocate buffer for saved machine code */
 803
 804         savedmcode = MNEW(u1, count * REPLACEMENT_PATCH_SIZE);
 805         code->savedmcode = savedmcode;
 806         savedmcode += count * REPLACEMENT_PATCH_SIZE;
 807
 808         /* activate trappable replacement points */
 809         /* (in reverse order to handle overlapping points within basic blocks) */
 810
 811         i = code->rplpointcount;
 812         rp = code->rplpoints + i;
 813         while (rp--, i--) {
 814                 assert(!(rp->flags & RPLPOINT_FLAG_ACTIVE));
 815
 816                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 817                         continue;
 818
 819                 index--;
 820
 821                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 822                         continue;
 823
 824                 DOLOG( printf("activate replacement point:\n");
 825                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 826
 827                 savedmcode -= REPLACEMENT_PATCH_SIZE;
 828
 829 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__)) && defined(ENABLE_JIT)
 830                 md_patch_replacement_point(code, index, rp, savedmcode);
 831 #endif
 832                 rp->flags |= RPLPOINT_FLAG_ACTIVE;
 833         }
 834
 835         assert(savedmcode == code->savedmcode);
 836 }
 837
 838
 839 /* replace_deactivate_replacement_points ***************************************
 840
 841    Deactivate a replacement points in the given compilation unit.
 842    When this function returns, the replacement points will be "un-armed",
 843    that is a each thread reaching a point will just continue normally.
 844
 845    IN:
 846        code.............the compilation unit
 847
 848 *******************************************************************************/
 849
 850 void replace_deactivate_replacement_points(codeinfo *code)
 851 {
 852         rplpoint *rp;
 853         s4        i;
 854         s4        count;
 855         u1       *savedmcode;
 856
 857         if (code->savedmcode == NULL) {
 858                 /* disarm countdown points by patching the branches */
 859
 860                 i = code->rplpointcount;
 861                 rp = code->rplpoints;
 862                 for (; i--; rp++) {
 863                         if ((rp->flags & (RPLPOINT_FLAG_ACTIVE | RPLPOINT_FLAG_COUNTDOWN))
 864                                         == RPLPOINT_FLAG_COUNTDOWN)
 865                         {
 866 #if 0
 867                                 *(s4*) (rp->pc + 9) = 0; /* XXX machine dependent! */
 868 #endif
 869                         }
 870                 }
 871                 return;
 872         }
 873
 874         assert(code->savedmcode != NULL);
 875         savedmcode = code->savedmcode;
 876
 877         /* de-activate each trappable replacement point */
 878
 879         i = code->rplpointcount;
 880         rp = code->rplpoints;
 881         count = 0;
 882         for (; i--; rp++) {
 883                 if (!(rp->flags & RPLPOINT_FLAG_ACTIVE))
 884                         continue;
 885
 886                 count++;
 887
 888                 DOLOG( printf("deactivate replacement point:\n");
 889                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 890
 891 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__)) && defined(ENABLE_JIT)
 892                 md_patch_replacement_point(code, -1, rp, savedmcode);
 893 #endif
 894
 895                 rp->flags &= ~RPLPOINT_FLAG_ACTIVE;
 896
 897                 savedmcode += REPLACEMENT_PATCH_SIZE;
 898         }
 899
 900         assert(savedmcode == code->savedmcode + count * REPLACEMENT_PATCH_SIZE);
 901
 902         /* free saved machine code */
 903
 904         MFREE(code->savedmcode, u1, count * REPLACEMENT_PATCH_SIZE);
 905         code->savedmcode = NULL;
 906 }
 907
 908
 909 /******************************************************************************/
 910 /* PART III: The replacement mechanism                                        */
 911 /******************************************************************************/
 912
 913
 914 /* replace_read_value **********************************************************
 915
 916    Read a value with the given allocation from the execution state.
 917
 918    IN:
 919            es...............execution state
 920            sp...............stack pointer of the execution state (XXX eliminate?)
 921            ra...............allocation
 922            javaval..........where to put the value
 923
 924    OUT:
 925        *javaval.........the value
 926
 927 *******************************************************************************/
 928
 929 static void replace_read_value(executionstate_t *es,
 930                                                            stackslot_t *sp,
 931                                                            rplalloc *ra,
 932                                                            replace_val_t *javaval)
 933 {
 934         if (ra->flags & INMEMORY) {
 935                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
 936 #ifdef HAS_4BYTE_STACKSLOT
 937                 if (IS_2_WORD_TYPE(ra->type)) {
 938                         javaval->l = *(u8*)(sp + ra->regoff);
 939                 }
 940                 else {
 941 #endif
 942                         javaval->p = sp[ra->regoff];
 943 #ifdef HAS_4BYTE_STACKSLOT
 944                 }
 945 #endif
 946         }
 947         else {
 948                 /* allocated register */
 949                 if (IS_FLT_DBL_TYPE(ra->type)) {
 950                         javaval->d = es->fltregs[ra->regoff];
 951
 952                         if (ra->type == TYPE_FLT)
 953                                 javaval->f = javaval->d;
 954                 }
 955                 else {
 956 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
 957                         if (ra->type == TYPE_LNG) {
 958                                 javaval->words.lo = es->intregs[GET_LOW_REG(ra->regoff)];
 959                                 javaval->words.hi = es->intregs[GET_HIGH_REG(ra->regoff)];
 960                         }
 961                         else
 962 #endif /* defined(SUPPORT_COMBINE_INTEGER_REGISTERS) */
 963                                 javaval->p = es->intregs[ra->regoff];
 964                 }
 965         }
 966 }
 967
 968
 969 /* replace_write_value *********************************************************
 970
 971    Write a value to the given allocation in the execution state.
 972
 973    IN:
 974            es...............execution state
 975            sp...............stack pointer of the execution state (XXX eliminate?)
 976            ra...............allocation
 977            *javaval.........the value
 978
 979 *******************************************************************************/
 980
 981 static void replace_write_value(executionstate_t *es,
 982                                                             stackslot_t *sp,
 983                                                             rplalloc *ra,
 984                                                             replace_val_t *javaval)
 985 {
 986         if (ra->flags & INMEMORY) {
 987                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
 988 #ifdef HAS_4BYTE_STACKSLOT
 989                 if (IS_2_WORD_TYPE(ra->type)) {
 990                         *(u8*)(sp + ra->regoff) = javaval->l;
 991                 }
 992                 else {
 993 #endif
 994                         sp[ra->regoff] = javaval->p;
 995 #ifdef HAS_4BYTE_STACKSLOT
 996                 }
 997 #endif
 998         }
 999         else {
1000                 /* allocated register */
1001                 switch (ra->type) {
1002                         case TYPE_FLT:
1003                                 es->fltregs[ra->regoff] = (double) javaval->f;
1004                                 break;
1005                         case TYPE_DBL:
1006                                 es->fltregs[ra->regoff] = javaval->d;
1007                                 break;
1008 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
1009                         case TYPE_LNG:
1010                                 es->intregs[GET_LOW_REG(ra->regoff)] = javaval->words.lo;
1011                                 es->intregs[GET_HIGH_REG(ra->regoff)] = javaval->words.hi;
1012                                 break;
1013 #endif
1014                         default:
1015                                 es->intregs[ra->regoff] = javaval->p;
1016                 }
1017         }
1018 }
1019
1020
1021 /* replace_read_executionstate *************************************************
1022
1023    Read the given executions state and translate it to a source frame.
1024
1025    IN:
1026            ss...............the source state
1027
1028    OUT:
1029            ss->frames.......set to new frame
1030
1031    RETURN VALUE:
1032        returns the new frame
1033
1034 *******************************************************************************/
1035
1036 static sourceframe_t *replace_new_sourceframe(sourcestate_t *ss)
1037 {
1038         sourceframe_t *frame;
1039
1040         frame = DNEW(sourceframe_t);
1041         MZERO(frame, sourceframe_t, 1);
1042
1043         frame->down = ss->frames;
1044         ss->frames = frame;
1045
1046         return frame;
1047 }
1048
1049
1050 /* replace_read_executionstate *************************************************
1051
1052    Read the given executions state and translate it to a source frame.
1053
1054    IN:
1055        rp...............replacement point at which `es` was taken
1056            es...............execution state
1057            ss...............where to put the source state
1058
1059    OUT:
1060        *ss..............the source state derived from the execution state
1061
1062 *******************************************************************************/
1063
1064 static s4 replace_normalize_type_map[] = {
1065 /* RPLPOINT_TYPE_STD    |--> */ RPLPOINT_TYPE_STD,
1066 /* RPLPOINT_TYPE_EXH    |--> */ RPLPOINT_TYPE_STD,
1067 /* RPLPOINT_TYPE_SBR    |--> */ RPLPOINT_TYPE_STD,
1068 /* RPLPOINT_TYPE_CALL   |--> */ RPLPOINT_TYPE_CALL,
1069 /* RPLPOINT_TYPE_INLINE |--> */ RPLPOINT_TYPE_CALL,
1070 /* RPLPOINT_TYPE_RETURN |--> */ RPLPOINT_TYPE_RETURN,
1071 /* RPLPOINT_TYPE_BODY   |--> */ RPLPOINT_TYPE_STD
1072 };
1073
1074
1075 static void replace_read_executionstate(rplpoint *rp,
1076                                                                                 executionstate_t *es,
1077                                                                                 sourcestate_t *ss,
1078                                                                                 bool topframe)
1079 {
1080         methodinfo    *m;
1081         codeinfo      *code;
1082         int            count;
1083         int            i;
1084         rplalloc      *ra;
1085         sourceframe_t *frame;
1086         int            topslot;
1087         stackslot_t   *sp;
1088         stackslot_t   *basesp;
1089
1090         code = code_find_codeinfo_for_pc(rp->pc);
1091         m = rp->method;
1092         topslot = TOP_IS_NORMAL;
1093
1094         /* stack pointer */
1095
1096         sp = (stackslot_t *) es->sp;
1097
1098         /* in some cases the top stack slot is passed in REG_ITMP1 */
1099
1100         if (rp->type == BBTYPE_EXH) {
1101                 topslot = TOP_IS_IN_ITMP1;
1102         }
1103
1104         /* calculate base stack pointer */
1105
1106         basesp = sp + code_get_stack_frame_size(code);
1107
1108         /* create the source frame */
1109
1110         frame = replace_new_sourceframe(ss);
1111         frame->method = rp->method;
1112         frame->id = rp->id;
1113         assert(rp->type >= 0 && rp->type < sizeof(replace_normalize_type_map)/sizeof(s4));
1114         frame->type = replace_normalize_type_map[rp->type];
1115         frame->fromrp = rp;
1116         frame->fromcode = code;
1117
1118         /* read local variables */
1119
1120         count = m->maxlocals;
1121         frame->javalocalcount = count;
1122         frame->javalocals = DMNEW(replace_val_t, count);
1123         frame->javalocaltype = DMNEW(u1, count);
1124
1125         /* mark values as undefined */
1126         for (i=0; i<count; ++i) {
1127 #if !defined(NDEBUG)
1128                 frame->javalocals[i].l = (u8) 0x00dead0000dead00ULL;
1129 #endif
1130                 frame->javalocaltype[i] = TYPE_VOID;
1131         }
1132
1133         /* some entries in the intregs array are not meaningful */
1134         /*es->intregs[REG_ITMP3] = (u8) 0x11dead1111dead11ULL;*/
1135 #if !defined(NDEBUG)
1136         es->intregs[REG_SP   ] = (ptrint) 0x11dead1111dead11ULL;
1137 #ifdef REG_PV
1138         es->intregs[REG_PV   ] = (ptrint) 0x11dead1111dead11ULL;
1139 #endif
1140 #endif /* !defined(NDEBUG) */
1141
1142         /* read javalocals */
1143
1144         count = rp->regalloccount;
1145         ra = rp->regalloc;
1146
1147         while (count && (i = ra->index) >= 0) {
1148                 assert(i < m->maxlocals);
1149                 frame->javalocaltype[i] = ra->type;
1150                 if (ra->type == TYPE_RET)
1151                         frame->javalocals[i].i = ra->regoff;
1152                 else
1153                         replace_read_value(es, sp, ra, frame->javalocals + i);
1154                 ra++;
1155                 count--;
1156         }
1157
1158         /* read instance, if this is the first rplpoint */
1159
1160 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
1161         if (topframe && !(rp->method->flags & ACC_STATIC) && rp == code->rplpoints) {
1162 #if 1
1163                 /* we are at the start of the method body, so if local 0 is set, */
1164                 /* it is the instance.                                           */
1165                 if (frame->javalocaltype[0] == TYPE_ADR)
1166                         frame->instance = frame->javalocals[0];
1167 #else
1168                 rplalloc instra;
1169                 methoddesc *md;
1170
1171                 md = rp->method->parseddesc;
1172                 assert(md->params);
1173                 assert(md->paramcount >= 1);
1174                 instra.type = TYPE_ADR;
1175                 instra.regoff = md->params[0].regoff;
1176                 if (md->params[0].inmemory) {
1177                         instra.flags = INMEMORY;
1178                         instra.regoff += (1 + code->stackframesize);
1179                 }
1180                 else {
1181                         instra.flags = 0;
1182                 }
1183                 replace_read_value(es, sp, &instra, &(frame->instance));
1184 #endif
1185         }
1186 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
1187
1188         /* read stack slots */
1189
1190         frame->javastackdepth = count;
1191         frame->javastack = DMNEW(replace_val_t, count);
1192         frame->javastacktype = DMNEW(u1, count);
1193
1194 #if !defined(NDEBUG)
1195         /* mark values as undefined */
1196         for (i=0; i<count; ++i) {
1197                 frame->javastack[i].l = (u8) 0x00dead0000dead00ULL;
1198                 frame->javastacktype[i] = TYPE_VOID;
1199         }
1200 #endif /* !defined(NDEBUG) */
1201
1202         i = 0;
1203
1204         /* the first stack slot is special in SBR and EXH blocks */
1205
1206         if (topslot == TOP_IS_ON_STACK) {
1207                 assert(count);
1208
1209                 assert(ra->index == RPLALLOC_STACK);
1210                 assert(ra->type == TYPE_ADR);
1211                 frame->javastack[i].p = sp[-1];
1212                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1213                 count--;
1214                 i++;
1215                 ra++;
1216         }
1217         else if (topslot == TOP_IS_IN_ITMP1) {
1218                 assert(count);
1219
1220                 assert(ra->index == RPLALLOC_STACK);
1221                 assert(ra->type == TYPE_ADR);
1222                 frame->javastack[i].p = es->intregs[REG_ITMP1];
1223                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1224                 count--;
1225                 i++;
1226                 ra++;
1227         }
1228         else if (topslot == TOP_IS_VOID) {
1229                 assert(count);
1230
1231                 assert(ra->index == RPLALLOC_STACK);
1232                 frame->javastack[i].l = 0;
1233                 frame->javastacktype[i] = TYPE_VOID;
1234                 count--;
1235                 i++;
1236                 ra++;
1237         }
1238
1239         /* read remaining stack slots */
1240
1241         for (; count--; ra++) {
1242                 if (ra->index == RPLALLOC_SYNC) {
1243                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1244
1245                         /* only read synchronization slots when traversing an inline point */
1246
1247                         if (!topframe) {
1248                                 sourceframe_t *calleeframe = frame->down;
1249                                 assert(calleeframe);
1250                                 assert(calleeframe->syncslotcount == 0);
1251                                 assert(calleeframe->syncslots == NULL);
1252
1253                                 calleeframe->syncslotcount = 1;
1254                                 calleeframe->syncslots = DMNEW(replace_val_t, 1);
1255                                 replace_read_value(es,sp,ra,calleeframe->syncslots);
1256                         }
1257
1258                         frame->javastackdepth--;
1259                         continue;
1260                 }
1261
1262                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1263
1264                 /* do not read parameters of calls down the call chain */
1265
1266                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1267                         frame->javastackdepth--;
1268                 }
1269                 else {
1270                         if (ra->type == TYPE_RET)
1271                                 frame->javastack[i].i = ra->regoff;
1272                         else
1273                                 replace_read_value(es,sp,ra,frame->javastack + i);
1274                         frame->javastacktype[i] = ra->type;
1275                         i++;
1276                 }
1277         }
1278 }
1279
1280
1281 /* replace_write_executionstate ************************************************
1282
1283    Translate the given source state into an execution state.
1284
1285    IN:
1286        rp...............replacement point for which execution state should be
1287                             creates
1288            es...............where to put the execution state
1289            ss...............the given source state
1290
1291    OUT:
1292        *es..............the execution state derived from the source state
1293
1294 *******************************************************************************/
1295
1296 static void replace_write_executionstate(rplpoint *rp,
1297                                                                                  executionstate_t *es,
1298                                                                                  sourcestate_t *ss,
1299                                                                                  bool topframe)
1300 {
1301         methodinfo     *m;
1302         codeinfo       *code;
1303         int             count;
1304         int             i;
1305         rplalloc       *ra;
1306         sourceframe_t  *frame;
1307         int             topslot;
1308         stackslot_t    *sp;
1309         stackslot_t    *basesp;
1310
1311         code = code_find_codeinfo_for_pc(rp->pc);
1312         m = rp->method;
1313         topslot = TOP_IS_NORMAL;
1314
1315         /* pop a source frame */
1316
1317         frame = ss->frames;
1318         assert(frame);
1319         ss->frames = frame->down;
1320
1321         /* calculate stack pointer */
1322
1323         sp = (stackslot_t *) es->sp;
1324
1325         basesp = sp + code_get_stack_frame_size(code);
1326
1327         /* in some cases the top stack slot is passed in REG_ITMP1 */
1328
1329         if (rp->type == BBTYPE_EXH) {
1330                 topslot = TOP_IS_IN_ITMP1;
1331         }
1332
1333         /* write javalocals */
1334
1335         ra = rp->regalloc;
1336         count = rp->regalloccount;
1337
1338         while (count && (i = ra->index) >= 0) {
1339                 assert(i < m->maxlocals);
1340                 assert(i < frame->javalocalcount);
1341                 assert(ra->type == frame->javalocaltype[i]);
1342                 if (ra->type == TYPE_RET) {
1343                         /* XXX assert that it matches this rplpoint */
1344                 }
1345                 else
1346                         replace_write_value(es, sp, ra, frame->javalocals + i);
1347                 count--;
1348                 ra++;
1349         }
1350
1351         /* write stack slots */
1352
1353         i = 0;
1354
1355         /* the first stack slot is special in SBR and EXH blocks */
1356
1357         if (topslot == TOP_IS_ON_STACK) {
1358                 assert(count);
1359
1360                 assert(ra->index == RPLALLOC_STACK);
1361                 assert(i < frame->javastackdepth);
1362                 assert(frame->javastacktype[i] == TYPE_ADR);
1363                 sp[-1] = frame->javastack[i].p;
1364                 count--;
1365                 i++;
1366                 ra++;
1367         }
1368         else if (topslot == TOP_IS_IN_ITMP1) {
1369                 assert(count);
1370
1371                 assert(ra->index == RPLALLOC_STACK);
1372                 assert(i < frame->javastackdepth);
1373                 assert(frame->javastacktype[i] == TYPE_ADR);
1374                 es->intregs[REG_ITMP1] = frame->javastack[i].p;
1375                 count--;
1376                 i++;
1377                 ra++;
1378         }
1379         else if (topslot == TOP_IS_VOID) {
1380                 assert(count);
1381
1382                 assert(ra->index == RPLALLOC_STACK);
1383                 assert(i < frame->javastackdepth);
1384                 assert(frame->javastacktype[i] == TYPE_VOID);
1385                 count--;
1386                 i++;
1387                 ra++;
1388         }
1389
1390         /* write remaining stack slots */
1391
1392         for (; count--; ra++) {
1393                 if (ra->index == RPLALLOC_SYNC) {
1394                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1395
1396                         /* only write synchronization slots when traversing an inline point */
1397
1398                         if (!topframe) {
1399                                 assert(frame->down);
1400                                 assert(frame->down->syncslotcount == 1); /* XXX need to understand more cases */
1401                                 assert(frame->down->syncslots != NULL);
1402
1403                                 replace_write_value(es,sp,ra,frame->down->syncslots);
1404                         }
1405                         continue;
1406                 }
1407
1408                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1409
1410                 /* do not write parameters of calls down the call chain */
1411
1412                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1413                         /* skip it */
1414                 }
1415                 else {
1416                         assert(i < frame->javastackdepth);
1417                         assert(ra->type == frame->javastacktype[i]);
1418                         if (ra->type == TYPE_RET) {
1419                                 /* XXX assert that it matches this rplpoint */
1420                         }
1421                         else {
1422                                 replace_write_value(es,sp,ra,frame->javastack + i);
1423                         }
1424                         i++;
1425                 }
1426         }
1427
1428         /* set new pc */
1429
1430         es->pc = rp->pc;
1431 }
1432
1433
1434 /* replace_pop_activation_record ***********************************************
1435
1436    Peel a stack frame from the execution state.
1437
1438    *** This function imitates the effects of the method epilog ***
1439    *** and returning from the method call.                     ***
1440
1441    IN:
1442            es...............execution state
1443            frame............source frame, receives synchronization slots
1444
1445    OUT:
1446        *es..............the execution state after popping the stack frame
1447
1448 *******************************************************************************/
1449
1450 u1* replace_pop_activation_record(executionstate_t *es,
1451                                                                   sourceframe_t *frame)
1452 {
1453         u1 *ra;
1454         u1 *pv;
1455         s4 reg;
1456         s4 i;
1457         s4 count;
1458         codeinfo *code;
1459         stackslot_t *basesp;
1460         stackslot_t *sp;
1461
1462         assert(es->code);
1463         assert(frame);
1464
1465         /* read the return address */
1466
1467 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1468         if (CODE_IS_LEAFMETHOD(es->code))
1469                 ra = (u1*) (ptrint) es->intregs[REPLACE_REG_RA];
1470         else
1471 #endif
1472                 ra = md_stacktrace_get_returnaddress(es->sp,
1473                                 SIZE_OF_STACKSLOT * es->code->stackframesize);
1474
1475         DOLOG( printf("return address: %p\n", (void*)ra); );
1476
1477         assert(ra);
1478
1479         /* calculate the base of the stack frame */
1480
1481         sp = (stackslot_t *) es->sp;
1482         basesp = sp + es->code->stackframesize;
1483
1484         /* read slots used for synchronization */
1485
1486         assert(frame->syncslotcount == 0);
1487         assert(frame->syncslots == NULL);
1488         count = code_get_sync_slot_count(es->code);
1489         frame->syncslotcount = count;
1490         frame->syncslots = DMNEW(replace_val_t, count);
1491         for (i=0; i<count; ++i) {
1492                 frame->syncslots[i].p = sp[es->code->memuse + i]; /* XXX */
1493         }
1494
1495         /* restore return address, if part of frame */
1496
1497 #if defined(REPLACE_RA_TOP_OF_FRAME)
1498 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1499         if (!CODE_IS_LEAFMETHOD(es->code))
1500 #endif
1501                 es->intregs[REPLACE_REG_RA] = *--basesp;
1502 #endif /* REPLACE_RA_TOP_OF_FRAME */
1503
1504 #if defined(REPLACE_RA_LINKAGE_AREA)
1505 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1506         if (!CODE_IS_LEAFMETHOD(es->code))
1507 #endif
1508                 es->intregs[REPLACE_REG_RA] = basesp[LA_LR_OFFSET / sizeof(stackslot_t)];
1509 #endif /* REPLACE_RA_LINKAGE_AREA */
1510
1511         /* restore saved int registers */
1512
1513         reg = INT_REG_CNT;
1514         for (i=0; i<es->code->savedintcount; ++i) {
1515                 while (nregdescint[--reg] != REG_SAV)
1516                         ;
1517                 es->intregs[reg] = *--basesp;
1518         }
1519
1520         /* restore saved flt registers */
1521
1522         /* XXX align? */
1523         reg = FLT_REG_CNT;
1524         for (i=0; i<es->code->savedfltcount; ++i) {
1525                 while (nregdescfloat[--reg] != REG_SAV)
1526                         ;
1527                 basesp -= STACK_SLOTS_PER_FLOAT;
1528                 es->fltregs[reg] = *(double*)basesp;
1529         }
1530
1531         /* adjust the stackpointer */
1532
1533         es->sp += SIZE_OF_STACKSLOT * es->code->stackframesize;
1534
1535 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1536         es->sp += SIZE_OF_STACKSLOT; /* skip return address */
1537 #endif
1538
1539         /* Set the new pc. Subtract one so we do not hit the replacement point */
1540         /* of the instruction following the call, if there is one.             */
1541
1542         es->pc = ra - 1;
1543
1544         /* find the new codeinfo */
1545
1546         pv = md_codegen_get_pv_from_pc(ra);
1547
1548         DOLOG( printf("PV = %p\n", (void*) pv); );
1549
1550         if (pv == NULL) /* XXX can this really happen? */
1551                 return NULL;
1552
1553         code = *(codeinfo **)(pv + CodeinfoPointer);
1554
1555         DOLOG( printf("CODE = %p\n", (void*) code); );
1556
1557         /* return NULL if we reached native code */
1558
1559         es->pv = pv;
1560         es->code = code;
1561
1562         /* in debugging mode clobber non-saved registers */
1563
1564 #if !defined(NDEBUG)
1565         /* for debugging */
1566         for (i=0; i<INT_REG_CNT; ++i)
1567                 if ((nregdescint[i] != REG_SAV)
1568 #if defined(REG_RA)
1569                                 && (i != REPLACE_REG_RA)
1570 #endif
1571                         )
1572                         es->intregs[i] = (ptrint) 0x33dead3333dead33ULL;
1573         for (i=0; i<FLT_REG_CNT; ++i)
1574                 if (nregdescfloat[i] != REG_SAV)
1575                         *(u8*)&(es->fltregs[i]) = 0x33dead3333dead33ULL;
1576 #endif /* !defined(NDEBUG) */
1577
1578         return (code) ? ra : NULL;
1579 }
1580
1581
1582 /* replace_patch_method_pointer ************************************************
1583
1584    Patch a method pointer (may be in code, data segment, vftbl, or interface
1585    table).
1586
1587    IN:
1588            mpp..............address of the method pointer to patch
1589            entrypoint.......the new entrypoint of the method
1590            kind.............kind of call to patch, used only for debugging
1591
1592 *******************************************************************************/
1593
1594 static void replace_patch_method_pointer(methodptr *mpp,
1595                                                                                  methodptr entrypoint,
1596                                                                                  const char *kind)
1597 {
1598 #if !defined(NDEBUG)
1599         codeinfo       *oldcode;
1600         codeinfo       *newcode;
1601 #endif
1602
1603         DOLOG( printf("patch method pointer from: %p to %p\n",
1604                                   (void*) *mpp, (void*)entrypoint); );
1605
1606 #if !defined(NDEBUG)
1607         oldcode = *(codeinfo **)((u1*)(*mpp) + CodeinfoPointer);
1608         newcode = *(codeinfo **)((u1*)(entrypoint) + CodeinfoPointer);
1609
1610         DOLOG_SHORT( printf("\tpatch %s %p ", kind, (void*) oldcode);
1611                                  method_println(oldcode->m);
1612                                  printf("\t      with      %p ", (void*) newcode);
1613                                  method_println(newcode->m); );
1614
1615         assert(oldcode->m == newcode->m);
1616 #endif
1617
1618         /* write the new entrypoint */
1619
1620         *mpp = (methodptr) entrypoint;
1621 }
1622
1623
1624 /* replace_patch_class *********************************************************
1625
1626    Patch a method in the given class.
1627
1628    IN:
1629            vftbl............vftbl of the class
1630            m................the method to patch
1631            oldentrypoint....the old entrypoint to replace
1632            entrypoint.......the new entrypoint
1633
1634 *******************************************************************************/
1635
1636 void replace_patch_class(vftbl_t *vftbl,
1637                                                  methodinfo *m,
1638                                                  u1 *oldentrypoint,
1639                                                  u1 *entrypoint)
1640 {
1641         s4                 i;
1642         methodptr         *mpp;
1643         methodptr         *mppend;
1644
1645         /* patch the vftbl of the class */
1646
1647         replace_patch_method_pointer(vftbl->table + m->vftblindex,
1648                                                                  entrypoint,
1649                                                                  "virtual  ");
1650
1651         /* patch the interface tables */
1652
1653         assert(oldentrypoint);
1654
1655         for (i=0; i < vftbl->interfacetablelength; ++i) {
1656                 mpp = vftbl->interfacetable[-i];
1657                 mppend = mpp + vftbl->interfacevftbllength[i];
1658                 for (; mpp != mppend; ++mpp)
1659                         if (*mpp == oldentrypoint) {
1660                                 replace_patch_method_pointer(mpp, entrypoint, "interface");
1661                         }
1662         }
1663 }
1664
1665
1666 /* replace_patch_class_hierarchy ***********************************************
1667
1668    Patch a method in all loaded classes.
1669
1670    IN:
1671            m................the method to patch
1672            oldentrypoint....the old entrypoint to replace
1673            entrypoint.......the new entrypoint
1674
1675 *******************************************************************************/
1676
1677 struct replace_patch_data_t {
1678         methodinfo *m;
1679         u1         *oldentrypoint;
1680         u1         *entrypoint;
1681 };
1682
1683 #define CODEINFO_OF_CODE(entrypoint) \
1684         (*(codeinfo **)((u1*)(entrypoint) + CodeinfoPointer))
1685
1686 #define METHOD_OF_CODE(entrypoint) \
1687         (CODEINFO_OF_CODE(entrypoint)->m)
1688
1689 void replace_patch_callback(classinfo *c, struct replace_patch_data_t *pd)
1690 {
1691         vftbl_t *vftbl = c->vftbl;
1692
1693         if (vftbl != NULL
1694                 && vftbl->vftbllength > pd->m->vftblindex
1695                 && vftbl->table[pd->m->vftblindex] != &asm_abstractmethoderror
1696                 && METHOD_OF_CODE(vftbl->table[pd->m->vftblindex]) == pd->m)
1697         {
1698                 replace_patch_class(c->vftbl, pd->m, pd->oldentrypoint, pd->entrypoint);
1699         }
1700 }
1701
1702 void replace_patch_class_hierarchy(methodinfo *m,
1703                                                                    u1 *oldentrypoint,
1704                                                                    u1 *entrypoint)
1705 {
1706         struct replace_patch_data_t pd;
1707
1708         pd.m = m;
1709         pd.oldentrypoint = oldentrypoint;
1710         pd.entrypoint = entrypoint;
1711
1712         DOLOG_SHORT( printf("patching class hierarchy: ");
1713                              method_println(m); );
1714
1715         classcache_foreach_loaded_class(
1716                         (classcache_foreach_functionptr_t) &replace_patch_callback,
1717                         (void*) &pd);
1718 }
1719
1720
1721 /* replace_patch_future_calls **************************************************
1722
1723    Analyse a call site and depending on the kind of call patch the call, the
1724    virtual function table, or the interface table.
1725
1726    IN:
1727            ra...............return address pointing after the call site
1728            callerframe......source frame of the caller
1729            calleeframe......source frame of the callee, must have been mapped
1730
1731 *******************************************************************************/
1732
1733 void replace_patch_future_calls(u1 *ra,
1734                                                                 sourceframe_t *callerframe,
1735                                                                 sourceframe_t *calleeframe)
1736 {
1737         u1                *patchpos;
1738         methodptr          entrypoint;
1739         methodptr          oldentrypoint;
1740         bool               atentry;
1741         stackframeinfo     sfi;
1742         codeinfo          *calleecode;
1743         methodinfo        *calleem;
1744         java_objectheader *obj;
1745         vftbl_t           *vftbl;
1746
1747         assert(ra);
1748         assert(callerframe->down == calleeframe);
1749
1750         /* get the new codeinfo and the method that shall be entered */
1751
1752         calleecode = calleeframe->tocode;
1753         assert(calleecode);
1754
1755         calleem = calleeframe->method;
1756         assert(calleem == calleecode->m);
1757
1758         entrypoint = (methodptr) calleecode->entrypoint;
1759
1760         /* check if we are at an method entry rplpoint at the innermost frame */
1761
1762         atentry = (calleeframe->down == NULL)
1763                         && !(calleem->flags & ACC_STATIC)
1764                         && (calleeframe->fromrp->id == 0); /* XXX */
1765
1766         /* get the position to patch, in case it was a statically bound call   */
1767
1768         sfi.pv = callerframe->fromcode->entrypoint;
1769         patchpos = md_get_method_patch_address(ra, &sfi, NULL);
1770
1771         if (patchpos == NULL) {
1772                 /* the call was dispatched dynamically */
1773
1774                 /* we can only patch such calls if we are at the entry point */
1775
1776                 if (!atentry)
1777                         return;
1778
1779                 assert((calleem->flags & ACC_STATIC) == 0);
1780
1781                 oldentrypoint = calleeframe->fromcode->entrypoint;
1782
1783                 /* we need to know the instance */
1784
1785                 if (!calleeframe->instance.a) {
1786                         DOLOG_SHORT( printf("WARNING: object instance unknown!\n"); );
1787                         replace_patch_class_hierarchy(calleem, oldentrypoint, entrypoint);
1788                         return;
1789                 }
1790
1791                 /* get the vftbl */
1792
1793                 obj = calleeframe->instance.a;
1794                 vftbl = obj->vftbl;
1795
1796                 assert(vftbl->class->vftbl == vftbl);
1797
1798                 DOLOG_SHORT( printf("\tclass: "); class_println(vftbl->class); );
1799
1800                 replace_patch_class(vftbl, calleem, oldentrypoint, entrypoint);
1801         }
1802         else {
1803                 /* the call was statically bound */
1804
1805                 replace_patch_method_pointer((methodptr *) patchpos, entrypoint, "static   ");
1806         }
1807 }
1808
1809
1810 /* replace_push_activation_record **********************************************
1811
1812    Push a stack frame onto the execution state.
1813
1814    *** This function imitates the effects of a call and the ***
1815    *** method prolog of the callee.                         ***
1816
1817    IN:
1818            es...............execution state
1819            rpcall...........the replacement point at the call site
1820            callerframe......source frame of the caller, or NULL for creating the
1821                             first frame
1822            calleeframe......source frame of the callee, must have been mapped
1823
1824    OUT:
1825        *es..............the execution state after pushing the stack frame
1826
1827 *******************************************************************************/
1828
1829 void replace_push_activation_record(executionstate_t *es,
1830                                                                         rplpoint *rpcall,
1831                                                                         sourceframe_t *callerframe,
1832                                                                         sourceframe_t *calleeframe)
1833 {
1834         s4           reg;
1835         s4           i;
1836         s4           count;
1837         stackslot_t *basesp;
1838         stackslot_t *sp;
1839         u1          *ra;
1840         codeinfo    *calleecode;
1841
1842         assert(es);
1843         assert(!rpcall || callerframe);
1844     assert(!rpcall || rpcall->type == RPLPOINT_TYPE_CALL);
1845         assert(!rpcall || rpcall == callerframe->torp);
1846         assert(calleeframe);
1847         assert(!callerframe || calleeframe == callerframe->down);
1848
1849         /* the compilation unit we are entering */
1850
1851         calleecode = calleeframe->tocode;
1852         assert(calleecode);
1853
1854         /* calculate the return address */
1855
1856         if (rpcall)
1857                 ra = rpcall->pc + rpcall->callsize;
1858         else
1859                 ra = es->pc + 1 /* XXX this is ugly */;
1860
1861         /* write the return address */
1862
1863 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1864         es->sp -= SIZE_OF_STACKSLOT;
1865
1866         *((stackslot_t *)es->sp) = (stackslot_t) ra;
1867 #endif /* REPLACE_RA_BETWEEN_FRAMES */
1868
1869 #if defined(REPLACE_REG_RA)
1870         es->intregs[REPLACE_REG_RA] = (ptrint) ra;
1871 #endif
1872
1873         /* we move into a new code unit */
1874
1875         es->code = calleecode;
1876
1877         /* set the new pc XXX not needed? */
1878
1879         es->pc = calleecode->entrypoint;
1880
1881         /* build the stackframe */
1882
1883         DOLOG( printf("building stackframe of %d words at %p\n",
1884                                   calleecode->stackframesize, (void*)es->sp); );
1885
1886         sp = (stackslot_t *) es->sp;
1887         basesp = sp;
1888
1889         sp -= calleecode->stackframesize;
1890         es->sp = (u1*) sp;
1891
1892         /* in debug mode, invalidate stack frame first */
1893
1894         /* XXX may not invalidate linkage area used by native code! */
1895 #if !defined(NDEBUG) && 0
1896         for (i=0; i<(basesp - sp); ++i) {
1897                 sp[i] = 0xdeaddeadU;
1898         }
1899 #endif
1900
1901         /* save the return address register */
1902
1903 #if defined(REPLACE_RA_TOP_OF_FRAME)
1904 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1905         if (!CODE_IS_LEAFMETHOD(calleecode))
1906 #endif
1907                 *--basesp = (ptrint) ra;
1908 #endif /* REPLACE_RA_TOP_OF_FRAME */
1909
1910 #if defined(REPLACE_RA_LINKAGE_AREA)
1911 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1912         if (!CODE_IS_LEAFMETHOD(calleecode))
1913 #endif
1914                 basesp[LA_LR_OFFSET / sizeof(stackslot_t)] = (ptrint) ra;
1915 #endif /* REPLACE_RA_LINKAGE_AREA */
1916
1917         /* save int registers */
1918
1919         reg = INT_REG_CNT;
1920         for (i=0; i<calleecode->savedintcount; ++i) {
1921                 while (nregdescint[--reg] != REG_SAV)
1922                         ;
1923                 *--basesp = es->intregs[reg];
1924
1925                 /* XXX may not clobber saved regs used by native code! */
1926 #if !defined(NDEBUG) && 0
1927                 es->intregs[reg] = (ptrint) 0x44dead4444dead44ULL;
1928 #endif
1929         }
1930
1931         /* save flt registers */
1932
1933         /* XXX align? */
1934         reg = FLT_REG_CNT;
1935         for (i=0; i<calleecode->savedfltcount; ++i) {
1936                 while (nregdescfloat[--reg] != REG_SAV)
1937                         ;
1938                 basesp -= STACK_SLOTS_PER_FLOAT;
1939                 *(double*)basesp = es->fltregs[reg];
1940
1941                 /* XXX may not clobber saved regs used by native code! */
1942 #if !defined(NDEBUG) && 0
1943                 *(u8*)&(es->fltregs[reg]) = 0x44dead4444dead44ULL;
1944 #endif
1945         }
1946
1947         /* write slots used for synchronization */
1948
1949         count = code_get_sync_slot_count(calleecode);
1950         assert(count == calleeframe->syncslotcount);
1951         for (i=0; i<count; ++i) {
1952                 sp[calleecode->memuse + i] = calleeframe->syncslots[i].p;
1953         }
1954
1955         /* set the PV */
1956
1957         es->pv = calleecode->entrypoint;
1958
1959         /* redirect future invocations */
1960
1961         if (callerframe && rpcall) {
1962 #if defined(REPLACE_PATCH_ALL)
1963                 if (rpcall->type == callerframe->fromrp->type)
1964 #else
1965                 if (rpcall == callerframe->fromrp)
1966 #endif
1967                         replace_patch_future_calls(ra, callerframe, calleeframe);
1968         }
1969 }
1970
1971
1972 /* replace_find_replacement_point **********************************************
1973
1974    Find the replacement point in the given code corresponding to the
1975    position given in the source frame.
1976
1977    IN:
1978            code.............the codeinfo in which to search the rplpoint
1979            frame............the source frame defining the position to look for
1980            parent...........parent replacement point to match
1981
1982    RETURN VALUE:
1983        the replacement point
1984
1985 *******************************************************************************/
1986
1987 rplpoint * replace_find_replacement_point(codeinfo *code,
1988                                                                                   sourceframe_t *frame,
1989                                                                                   rplpoint *parent)
1990 {
1991         methodinfo *m;
1992         rplpoint *rp;
1993         s4        i;
1994         s4        j;
1995         s4        stacki;
1996         rplalloc *ra;
1997
1998         assert(code);
1999         assert(frame);
2000
2001         DOLOG( printf("searching replacement point for:\n");
2002                    replace_source_frame_println(frame); );
2003
2004         m = frame->method;
2005
2006         DOLOG( printf("code = %p\n", (void*)code); );
2007
2008         rp = code->rplpoints;
2009         i = code->rplpointcount;
2010         while (i--) {
2011                 if (rp->id == frame->id && rp->method == frame->method
2012                                 && rp->parent == parent
2013                                 && replace_normalize_type_map[rp->type] == frame->type)
2014                 {
2015                         /* check if returnAddresses match */
2016                         /* XXX optimize: only do this if JSRs in method */
2017                         DOLOG( printf("checking match for:");
2018                                    replace_replacement_point_println(rp, 1); fflush(stdout); );
2019                         ra = rp->regalloc;
2020                         stacki = 0;
2021                         for (j = rp->regalloccount; j--; ++ra) {
2022                                 if (ra->type == TYPE_RET) {
2023                                         if (ra->index == RPLALLOC_STACK) {
2024                                                 assert(stacki < frame->javastackdepth);
2025                                                 if (frame->javastack[stacki].i != ra->regoff)
2026                                                         goto no_match;
2027                                                 stacki++;
2028                                         }
2029                                         else {
2030                                                 assert(ra->index >= 0 && ra->index < frame->javalocalcount);
2031                                                 if (frame->javalocals[ra->index].i != ra->regoff)
2032                                                         goto no_match;
2033                                         }
2034                                 }
2035                         }
2036
2037                         /* found */
2038                         return rp;
2039                 }
2040 no_match:
2041                 rp++;
2042         }
2043
2044 #if !defined(NDEBUG)
2045         printf("candidate replacement points were:\n");
2046         rp = code->rplpoints;
2047         i = code->rplpointcount;
2048         for (; i--; ++rp) {
2049                 replace_replacement_point_println(rp, 1);
2050         }
2051 #endif
2052
2053         vm_abort("no matching replacement point found");
2054         return NULL; /* NOT REACHED */
2055 }
2056
2057
2058 /* replace_find_replacement_point_for_pc ***************************************
2059
2060    Find the nearest replacement point at or before the given PC.
2061
2062    IN:
2063        code.............compilation unit the PC is in
2064            pc...............the machine code PC
2065
2066    RETURN VALUE:
2067        the replacement point found, or
2068            NULL if no replacement point was found
2069
2070 *******************************************************************************/
2071
2072 rplpoint *replace_find_replacement_point_for_pc(codeinfo *code, u1 *pc)
2073 {
2074         rplpoint *found;
2075         rplpoint *rp;
2076         s4        i;
2077
2078         DOLOG( printf("searching for rp in %p ", (void*)code);
2079                    method_println(code->m); );
2080
2081         found = NULL;
2082
2083         rp = code->rplpoints;
2084         for (i=0; i<code->rplpointcount; ++i, ++rp) {
2085                 DOLOG( replace_replacement_point_println(rp, 2); );
2086                 if (rp->pc <= pc)
2087                         found = rp;
2088         }
2089
2090         return found;
2091 }
2092
2093
2094 /* replace_pop_native_frame ****************************************************
2095
2096    Unroll a native frame in the execution state and create a source frame
2097    for it.
2098
2099    IN:
2100            es...............current execution state
2101            ss...............the current source state
2102            sfi..............stackframeinfo for the native frame
2103
2104    OUT:
2105        es...............execution state after unrolling the native frame
2106            ss...............gets the added native source frame
2107
2108 *******************************************************************************/
2109
2110 static void replace_pop_native_frame(executionstate_t *es,
2111                                                                          sourcestate_t *ss,
2112                                                                          stackframeinfo *sfi)
2113 {
2114         sourceframe_t *frame;
2115         codeinfo      *code;
2116         s4             i,j;
2117
2118         assert(sfi);
2119
2120         frame = replace_new_sourceframe(ss);
2121
2122         frame->sfi = sfi;
2123
2124         /* remember pc and size of native frame */
2125
2126         frame->nativepc = es->pc;
2127         frame->nativeframesize = (es->sp != 0) ? (sfi->sp - es->sp) : 0;
2128         assert(frame->nativeframesize >= 0);
2129
2130         /* remember values of saved registers */
2131
2132         j = 0;
2133         for (i=0; i<INT_REG_CNT; ++i) {
2134                 if (nregdescint[i] == REG_SAV)
2135                         frame->nativesavint[j++] = es->intregs[i];
2136         }
2137
2138         j = 0;
2139         for (i=0; i<FLT_REG_CNT; ++i) {
2140                 if (nregdescfloat[i] == REG_SAV)
2141                         frame->nativesavflt[j++] = es->fltregs[i];
2142         }
2143
2144         /* restore saved registers */
2145
2146 #if defined(ENABLE_GC_CACAO)
2147         j = 0;
2148         for (i=0; i<INT_REG_CNT; ++i) {
2149                 if (nregdescint[i] == REG_SAV)
2150                         es->intregs[i] = sfi->intregs[j++];
2151         }
2152 #else
2153         /* XXX we don't have them, yet, in the sfi, so clear them */
2154
2155         for (i=0; i<INT_REG_CNT; ++i) {
2156                 if (nregdescint[i] == REG_SAV)
2157                         es->intregs[i] = 0;
2158         }
2159 #endif
2160
2161         /* XXX we don't have float registers in the sfi, so clear them */
2162
2163         for (i=0; i<FLT_REG_CNT; ++i) {
2164                 if (nregdescfloat[i] == REG_SAV)
2165                         es->fltregs[i] = 0.0;
2166         }
2167
2168         /* restore pv, pc, and sp */
2169
2170         if (sfi->pv == NULL) {
2171                 /* frame of a native function call */
2172                 es->pv = md_codegen_get_pv_from_pc(sfi->ra);
2173         }
2174         else {
2175                 es->pv = sfi->pv;
2176         }
2177         es->pc = ((sfi->xpc) ? sfi->xpc : sfi->ra) - 1;
2178         es->sp = sfi->sp;
2179
2180         /* find the new codeinfo */
2181
2182         DOLOG( printf("PV = %p\n", (void*) es->pv); );
2183
2184         assert(es->pv != NULL);
2185
2186         code = *(codeinfo **)(es->pv + CodeinfoPointer);
2187
2188         DOLOG( printf("CODE = %p\n", (void*) code); );
2189
2190         es->code = code;
2191 }
2192
2193
2194 /* replace_push_native_frame ***************************************************
2195
2196    Rebuild a native frame onto the execution state and remove its source frame.
2197
2198    Note: The native frame is "rebuild" by setting fields like PC and stack
2199          pointer in the execution state accordingly. Values in the
2200                  stackframeinfo may be modified, but the actual stack frame of the
2201                  native code is not touched.
2202
2203    IN:
2204            es...............current execution state
2205            ss...............the current source state
2206
2207    OUT:
2208        es...............execution state after re-rolling the native frame
2209            ss...............the native source frame is removed
2210
2211 *******************************************************************************/
2212
2213 static void replace_push_native_frame(executionstate_t *es, sourcestate_t *ss)
2214 {
2215         sourceframe_t *frame;
2216         s4             i,j;
2217
2218         assert(es);
2219         assert(ss);
2220
2221         DOLOG( printf("pushing native frame\n"); );
2222
2223         /* remove the frame from the source state */
2224
2225         frame = ss->frames;
2226         assert(frame);
2227         assert(REPLACE_IS_NATIVE_FRAME(frame));
2228
2229         ss->frames = frame->down;
2230
2231         /* assert that the native frame has not moved */
2232
2233         assert(es->sp == frame->sfi->sp);
2234
2235         /* update saved registers in the stackframeinfo */
2236
2237 #if defined(ENABLE_GC_CACAO)
2238         j = 0;
2239         for (i=0; i<INT_REG_CNT; ++i) {
2240                 if (nregdescint[i] == REG_SAV)
2241                         frame->sfi->intregs[j++] = es->intregs[i];
2242         }
2243
2244         /* XXX leave float registers untouched here */
2245 #endif
2246
2247         /* restore saved registers */
2248
2249         j = 0;
2250         for (i=0; i<INT_REG_CNT; ++i) {
2251                 if (nregdescint[i] == REG_SAV)
2252                         es->intregs[i] = frame->nativesavint[j++];
2253         }
2254
2255         j = 0;
2256         for (i=0; i<FLT_REG_CNT; ++i) {
2257                 if (nregdescfloat[i] == REG_SAV)
2258                         es->fltregs[i] = frame->nativesavflt[j++];
2259         }
2260
2261         /* skip the native frame on the machine stack */
2262
2263         es->sp -= frame->nativeframesize;
2264
2265         /* set the pc the next frame must return to */
2266
2267         es->pc = frame->nativepc;
2268 }
2269
2270
2271 /* replace_recover_source_state ************************************************
2272
2273    Recover the source state from the given replacement point and execution
2274    state.
2275
2276    IN:
2277        rp...............replacement point that has been reached, if any
2278            sfi..............stackframeinfo, if called from native code
2279            es...............execution state at the replacement point rp
2280
2281    RETURN VALUE:
2282        the source state
2283
2284 *******************************************************************************/
2285
2286 sourcestate_t *replace_recover_source_state(rplpoint *rp,
2287                                                                                         stackframeinfo *sfi,
2288                                                                                     executionstate_t *es)
2289 {
2290         sourcestate_t *ss;
2291         u1            *ra;
2292         bool           locked;
2293 #if defined(REPLACE_STATISTICS)
2294         s4             depth;
2295 #endif
2296
2297         /* create the source frame structure in dump memory */
2298
2299         ss = DNEW(sourcestate_t);
2300         ss->frames = NULL;
2301
2302         /* each iteration of the loop recovers one source frame */
2303
2304         depth = 0;
2305         locked = false;
2306
2307         while (rp || sfi) {
2308
2309                 DOLOG( replace_executionstate_println(es); );
2310
2311                 /* if we are not at a replacement point, it is a native frame */
2312
2313                 if (rp == NULL) {
2314                         DOLOG( printf("native frame: sfi: "); replace_stackframeinfo_println(sfi); );
2315
2316                         locked = true;
2317                         replace_pop_native_frame(es, ss, sfi);
2318                         sfi = sfi->prev;
2319
2320                         if (es->code == NULL)
2321                                 continue;
2322
2323                         goto after_machine_frame;
2324                 }
2325
2326                 /* read the values for this source frame from the execution state */
2327
2328                 DOLOG( printf("recovering source state for%s:\n",
2329                                         (ss->frames == NULL) ? " TOPFRAME" : "");
2330                            replace_replacement_point_println(rp, 1); );
2331
2332                 replace_read_executionstate(rp, es, ss, ss->frames == NULL);
2333
2334 #if defined(REPLACE_STATISTICS)
2335                 REPLACE_COUNT(stat_frames);
2336                 depth++;
2337                 replace_statistics_source_frame(ss->frames);
2338 #endif
2339
2340                 /* in locked areas (below native frames), identity map the frame */
2341
2342                 if (locked) {
2343                         ss->frames->torp = ss->frames->fromrp;
2344                         ss->frames->tocode = ss->frames->fromcode;
2345                 }
2346
2347                 /* unroll to the next (outer) frame */
2348
2349                 if (rp->parent) {
2350                         /* this frame is in inlined code */
2351
2352                         DOLOG( printf("INLINED!\n"); );
2353
2354                         rp = rp->parent;
2355
2356                         assert(rp->type == RPLPOINT_TYPE_INLINE);
2357                         REPLACE_COUNT(stat_unroll_inline);
2358                 }
2359                 else {
2360                         /* this frame had been called at machine-level. pop it. */
2361
2362                         DOLOG( printf("UNWIND\n"); );
2363
2364                         ra = replace_pop_activation_record(es, ss->frames);
2365                         if (ra == NULL) {
2366                                 DOLOG( printf("REACHED NATIVE CODE\n"); );
2367
2368                                 rp = NULL;
2369
2370                                 break; /* XXX remove to activate native frames */
2371                                 continue;
2372                         }
2373
2374                         /* find the replacement point at the call site */
2375
2376 after_machine_frame:
2377                         rp = replace_find_replacement_point_for_pc(es->code, es->pc);
2378
2379                         if (rp == NULL)
2380                                 vm_abort("could not find replacement point while unrolling call");
2381
2382                         DOLOG( printf("found replacement point.\n");
2383                                         replace_replacement_point_println(rp, 1); );
2384
2385                         assert(rp->type == RPLPOINT_TYPE_CALL);
2386                         REPLACE_COUNT(stat_unroll_call);
2387                 }
2388         } /* end loop over source frames */
2389
2390         REPLACE_COUNT_DIST(stat_dist_frames, depth);
2391
2392         return ss;
2393 }
2394
2395
2396 /* replace_map_source_state ****************************************************
2397
2398    Map each source frame in the given source state to a target replacement
2399    point and compilation unit. If no valid code is available for a source
2400    frame, it is (re)compiled.
2401
2402    IN:
2403        ss...............the source state
2404
2405    OUT:
2406        ss...............the source state, modified: The `torp` and `tocode`
2407                             fields of each source frame are set.
2408
2409    RETURN VALUE:
2410        true.............everything went ok
2411            false............an exception has been thrown
2412
2413 *******************************************************************************/
2414
2415 static bool replace_map_source_state(sourcestate_t *ss)
2416 {
2417         sourceframe_t *frame;
2418         codeinfo      *code;
2419         rplpoint      *rp;
2420         rplpoint      *parent; /* parent of inlined rplpoint */
2421 #if defined(REPLACE_STATISTICS)
2422         codeinfo      *oldcode;
2423 #endif
2424
2425         parent = NULL;
2426         code = NULL;
2427
2428         /* iterate over the source frames from outermost to innermost */
2429
2430         for (frame = ss->frames; frame != NULL; frame = frame->down) {
2431
2432                 /* XXX skip native frames */
2433
2434                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
2435                         parent = NULL;
2436                         continue;
2437                 }
2438
2439                 /* map frames which are not already mapped */
2440
2441                 if (frame->tocode) {
2442                         code = frame->tocode;
2443                         rp = frame->torp;
2444                         assert(rp);
2445                 }
2446                 else {
2447                         assert(frame->torp == NULL);
2448
2449                         if (parent == NULL) {
2450                                 /* find code for this frame */
2451
2452 #if defined(REPLACE_STATISTICS)
2453                                 oldcode = frame->method->code;
2454 #endif
2455                                 /* request optimization of hot methods and their callers */
2456
2457                                 if (frame->method->hitcountdown < 0
2458                                         || (frame->down && frame->down->method->hitcountdown < 0))
2459                                         jit_request_optimization(frame->method);
2460
2461                                 code = jit_get_current_code(frame->method);
2462
2463                                 if (code == NULL)
2464                                         return false; /* exception */
2465
2466                                 REPLACE_COUNT_IF(stat_recompile, code != oldcode);
2467                         }
2468
2469                         assert(code);
2470
2471                         /* map this frame */
2472
2473                         rp = replace_find_replacement_point(code, frame, parent);
2474
2475                         frame->tocode = code;
2476                         frame->torp = rp;
2477                 }
2478
2479                 if (rp->type == RPLPOINT_TYPE_CALL) {
2480                         parent = NULL;
2481                 }
2482                 else {
2483                         /* inlining */
2484                         parent = rp;
2485                 }
2486         }
2487
2488         return true;
2489 }
2490
2491
2492 /* replace_map_source_state_identity *******************************************
2493
2494    Map each source frame in the given source state to the same replacement
2495    point and compilation unit it was derived from. This is mainly used for
2496    garbage collection.
2497
2498    IN:
2499        ss...............the source state
2500
2501    OUT:
2502        ss...............the source state, modified: The `torp` and `tocode`
2503                             fields of each source frame are set.
2504
2505 *******************************************************************************/
2506
2507 #if defined(ENABLE_GC_CACAO)
2508 static void replace_map_source_state_identity(sourcestate_t *ss)
2509 {
2510         sourceframe_t *frame;
2511
2512         /* iterate over the source frames from outermost to innermost */
2513
2514         for (frame = ss->frames; frame != NULL; frame = frame->down) {
2515
2516                 /* skip native frames */
2517
2518                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
2519                         continue;
2520                 }
2521
2522                 /* map frames using the identity mapping */
2523
2524                 if (frame->tocode) {
2525                         assert(frame->tocode == frame->fromcode);
2526                         assert(frame->torp   == frame->fromrp);
2527                 } else {
2528                         assert(frame->tocode == NULL);
2529                         assert(frame->torp   == NULL);
2530                         frame->tocode = frame->fromcode;
2531                         frame->torp   = frame->fromrp;
2532                 }
2533         }
2534 }
2535 #endif
2536
2537
2538 /* replace_build_execution_state_intern ****************************************
2539
2540    Build an execution state for the given (mapped) source state.
2541
2542    !!! CAUTION: This function rewrites the machine stack !!!
2543
2544    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2545
2546    IN:
2547        ss...............the source state. Must have been mapped by
2548                                                 replace_map_source_state before.
2549            es...............the base execution state on which to build
2550
2551    OUT:
2552        *es..............the new execution state
2553
2554 *******************************************************************************/
2555
2556 void replace_build_execution_state_intern(sourcestate_t *ss,
2557                                                                                                  executionstate_t *es)
2558 {
2559         rplpoint      *rp;
2560         sourceframe_t *prevframe;
2561         rplpoint      *parent;
2562
2563         parent = NULL;
2564         prevframe = NULL;
2565         rp = NULL;
2566
2567         while (ss->frames) {
2568
2569                 if (REPLACE_IS_NATIVE_FRAME(ss->frames)) {
2570                         prevframe = ss->frames;
2571                         replace_push_native_frame(es, ss);
2572                         parent = NULL;
2573                         rp = NULL;
2574                         continue;
2575                 }
2576
2577                 if (parent == NULL) {
2578                         /* create a machine-level stack frame */
2579
2580                         DOLOG( printf("pushing activation record for:\n");
2581                                    if (rp) replace_replacement_point_println(rp, 1);
2582                                    else printf("\tfirst frame\n"); );
2583
2584                         replace_push_activation_record(es, rp, prevframe, ss->frames);
2585
2586                         DOLOG( replace_executionstate_println(es); );
2587                 }
2588
2589                 rp = ss->frames->torp;
2590                 assert(rp);
2591
2592                 DOLOG( printf("creating execution state for%s:\n",
2593                                 (ss->frames->down == NULL) ? " TOPFRAME" : "");
2594                            replace_replacement_point_println(ss->frames->fromrp, 1);
2595                            replace_replacement_point_println(rp, 1); );
2596
2597                 es->code = ss->frames->tocode;
2598                 prevframe = ss->frames;
2599                 replace_write_executionstate(rp, es, ss, ss->frames->down == NULL);
2600
2601                 DOLOG( replace_executionstate_println(es); );
2602
2603                 if (rp->type == RPLPOINT_TYPE_CALL) {
2604                         parent = NULL;
2605                 }
2606                 else {
2607                         /* inlining */
2608                         parent = rp;
2609                 }
2610         }
2611 }
2612
2613
2614 /* replace_build_execution_state ***********************************************
2615
2616    This function contains the final phase of replacement. It builds the new
2617    execution state, releases dump memory, and returns to the calling
2618    assembler function which finishes replacement.
2619
2620    NOTE: This function is called from asm_replacement_in, with the stack
2621          pointer at the start of the safe stack area.
2622
2623    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2624
2625    CAUTION: This function and its children must not use a lot of stack!
2626             There are only REPLACE_SAFESTACK_SIZE bytes of C stack
2627                         available.
2628
2629    IN:
2630        st...............the safestack contained the necessary data
2631
2632 *******************************************************************************/
2633
2634 void replace_build_execution_state(replace_safestack_t *st)
2635 {
2636         replace_build_execution_state_intern(st->ss, &(st->es));
2637
2638         DOLOG( replace_executionstate_println(&(st->es)); );
2639
2640         /* release dump area */
2641
2642         dump_release(st->dumpsize);
2643
2644         /* new code is entered after returning */
2645
2646         DOLOG( printf("JUMPING IN!\n"); fflush(stdout); );
2647 }
2648
2649
2650 /* replace_alloc_safestack *****************************************************
2651
2652    Allocate a safe stack area to use during the final phase of replacement.
2653    The returned area is not initialized. This must be done by the caller.
2654
2655    RETURN VALUE:
2656        a newly allocated replace_safestack_t *
2657
2658 *******************************************************************************/
2659
2660 static replace_safestack_t *replace_alloc_safestack()
2661 {
2662         u1 *mem;
2663         replace_safestack_t *st;
2664
2665         mem = MNEW(u1, sizeof(replace_safestack_t) + REPLACE_STACK_ALIGNMENT - 1);
2666
2667         st = (replace_safestack_t *) ((ptrint)(mem + REPLACE_STACK_ALIGNMENT - 1)
2668                                                                                 & ~(REPLACE_STACK_ALIGNMENT - 1));
2669
2670 #if !defined(NDEBUG)
2671         memset(st, 0xa5, sizeof(replace_safestack_t));
2672 #endif
2673
2674         st->mem = mem;
2675
2676         return st;
2677 }
2678
2679
2680 /* replace_free_safestack ******************************************************
2681
2682    Free the given safestack structure, making a copy of the contained
2683    execution state before freeing it.
2684
2685    NOTE: This function is called from asm_replacement_in.
2686
2687    IN:
2688        st...............the safestack to free
2689            tmpes............where to copy the execution state to
2690
2691    OUT:
2692            *tmpes...........receives a copy of st->es
2693
2694 *******************************************************************************/
2695
2696 void replace_free_safestack(replace_safestack_t *st, executionstate_t *tmpes)
2697 {
2698         u1 *mem;
2699
2700         /* copy the executionstate_t to the temporary location */
2701
2702         *tmpes = st->es;
2703
2704         /* get the memory address to free */
2705
2706         mem = st->mem;
2707
2708         /* destroy memory (in debug mode) */
2709
2710 #if !defined(NDEBUG)
2711         memset(st, 0xa5, sizeof(replace_safestack_t));
2712 #endif
2713
2714         /* free the safe stack struct */
2715
2716         MFREE(mem, u1, sizeof(replace_safestack_t) + REPLACE_STACK_ALIGNMENT - 1);
2717 }
2718
2719
2720 /* replace_me ******************************************************************
2721
2722    This function is called by asm_replacement_out when a thread reaches
2723    a replacement point. `replace_me` must map the execution state to the
2724    target replacement point and let execution continue there.
2725
2726    This function never returns!
2727
2728    IN:
2729        rp...............replacement point that has been reached
2730            es...............execution state read by asm_replacement_out
2731
2732 *******************************************************************************/
2733
2734 void replace_me(rplpoint *rp, executionstate_t *es)
2735 {
2736         stackframeinfo      *sfi;
2737         sourcestate_t       *ss;
2738         sourceframe_t       *frame;
2739         s4                   dumpsize;
2740         rplpoint            *origrp;
2741         replace_safestack_t *safestack;
2742 #if defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)
2743         threadobject        *thread;
2744 #endif
2745
2746         origrp = rp;
2747         es->code = code_find_codeinfo_for_pc(rp->pc);
2748
2749         DOLOG_SHORT( printf("REPLACING(%d %p): (id %d %p) ",
2750                                  stat_replacements, (void*)THREADOBJECT,
2751                                  rp->id, (void*)rp);
2752                                  method_println(es->code->m); );
2753
2754         DOLOG( replace_replacement_point_println(rp, 1);
2755                    replace_executionstate_println(es); );
2756
2757         REPLACE_COUNT(stat_replacements);
2758
2759         /* mark start of dump memory area */
2760
2761         dumpsize = dump_size();
2762
2763         /* get the stackframeinfo for the current thread */
2764
2765         sfi = STACKFRAMEINFO;
2766
2767         /* recover source state */
2768
2769         ss = replace_recover_source_state(rp, sfi, es);
2770
2771 #if defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)
2772         /* if there is a collection pending, we assume the replacement point should
2773            suspend this thread */
2774
2775         if (gc_pending) {
2776
2777                 thread = THREADOBJECT;
2778
2779                 DOLOG_SHORT( printf("REPLACEMENT: Suspending thread for GC now!\n"); );
2780
2781                 /* map the sourcestate using the identity mapping */
2782                 replace_map_source_state_identity(ss);
2783
2784                 /* since we enter the same method again, we turn off rps now */
2785                 /* XXX michi: can we really do this? what if the rp was active before
2786                    we activated it for the gc? */
2787                 frame = ss->frames;
2788                 while (frame->down)
2789                         frame = frame->down;
2790                 replace_deactivate_replacement_points(frame->tocode);
2791
2792                 /* remember executionstate and sourcestate for this thread */
2793                 GC_EXECUTIONSTATE = es;
2794                 GC_SOURCESTATE    = ss;
2795
2796                 /* really suspend this thread now (PC = 0) */
2797                 threads_suspend_ack(NULL, NULL);
2798
2799                 DOLOG_SHORT( printf("REPLACEMENT: Resuming thread after GC now!\n"); );
2800
2801         } else {
2802 #endif /*defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)*/
2803
2804         /* map the source state */
2805
2806         if (!replace_map_source_state(ss))
2807                 vm_abort("exception during method replacement");
2808
2809         DOLOG( replace_sourcestate_println(ss); );
2810
2811         DOLOG_SHORT( replace_sourcestate_println_short(ss); );
2812
2813         /* avoid infinite loops by self-replacement */
2814
2815         frame = ss->frames;
2816         while (frame->down)
2817                 frame = frame->down;
2818
2819         if (frame->torp == origrp) {
2820                 DOLOG_SHORT(
2821                         printf("WARNING: identity replacement, turning off rps to avoid infinite loop\n");
2822                 );
2823                 replace_deactivate_replacement_points(frame->tocode);
2824         }
2825
2826 #if defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)
2827         }
2828 #endif
2829
2830         /* write execution state of new code */
2831
2832         DOLOG( replace_executionstate_println(es); );
2833
2834         /* allocate a safe stack area and copy all needed data there */
2835
2836         safestack = replace_alloc_safestack();
2837
2838         safestack->es = *es;
2839         safestack->ss = ss;
2840         safestack->dumpsize = dumpsize;
2841
2842         /* call the assembler code for the last phase of replacement */
2843
2844 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__)) && defined(ENABLE_JIT)
2845         asm_replacement_in(&(safestack->es), safestack);
2846 #endif
2847
2848         abort(); /* NOT REACHED */
2849 }
2850
2851
2852 /******************************************************************************/
2853 /* NOTE: Stuff specific to the exact GC is below.                             */
2854 /******************************************************************************/
2855
2856 #if defined(ENABLE_GC_CACAO)
2857 void replace_gc_from_native(threadobject *thread, u1 *pc, u1 *sp)
2858 {
2859         stackframeinfo   *sfi;
2860         executionstate_t *es;
2861         sourcestate_t    *ss;
2862
2863         /* get the stackframeinfo of this thread */
2864         assert(thread == THREADOBJECT);
2865         sfi = STACKFRAMEINFO;
2866
2867         /* create the execution state */
2868         es = DNEW(executionstate_t);
2869         es->pc = pc;
2870         es->sp = sp;
2871         es->pv = 0;      /* since we are in a native, PV is invalid! */
2872         es->code = NULL; /* since we are in a native, we do not have a codeinfo */
2873
2874         /* we assume we are in a native (no replacement point)! */
2875         ss = replace_recover_source_state(NULL, sfi, es);
2876
2877         /* map the sourcestate using the identity mapping */
2878         replace_map_source_state_identity(ss);
2879
2880         /* remember executionstate and sourcestate for this thread */
2881         GC_EXECUTIONSTATE = es;
2882         GC_SOURCESTATE    = ss;
2883 }
2884 #endif
2885
2886
2887 /******************************************************************************/
2888 /* NOTE: No important code below.                                             */
2889 /******************************************************************************/
2890
2891
2892 /* statistics *****************************************************************/
2893
2894 #if defined(REPLACE_STATISTICS)
2895 static void print_freq(FILE *file,int *array,int limit)
2896 {
2897         int i;
2898         int sum = 0;
2899         int cum = 0;
2900         for (i=0; i<limit; ++i)
2901                 sum += array[i];
2902         sum += array[limit];
2903         for (i=0; i<limit; ++i) {
2904                 cum += array[i];
2905                 fprintf(file,"      %3d: %8d (cum %3d%%)\n",
2906                                 i, array[i], (sum) ? ((100*cum)/sum) : 0);
2907         }
2908         fprintf(file,"    >=%3d: %8d\n",limit,array[limit]);
2909 }
2910 #endif /* defined(REPLACE_STATISTICS) */
2911
2912
2913 #if defined(REPLACE_STATISTICS)
2914
2915 #define REPLACE_PRINT_DIST(name, array)                              \
2916     printf("    " name " distribution:\n");                          \
2917     print_freq(stdout, (array), sizeof(array)/sizeof(int) - 1);
2918
2919 void replace_print_statistics(void)
2920 {
2921         printf("replacement statistics:\n");
2922         printf("    # of replacements:   %d\n", stat_replacements);
2923         printf("    # of frames:         %d\n", stat_frames);
2924         printf("    # of recompilations: %d\n", stat_recompile);
2925         printf("    patched static calls:%d\n", stat_staticpatch);
2926         printf("    unrolled inlines:    %d\n", stat_unroll_inline);
2927         printf("    unrolled calls:      %d\n", stat_unroll_call);
2928         REPLACE_PRINT_DIST("frame depth", stat_dist_frames);
2929         REPLACE_PRINT_DIST("locals per frame", stat_dist_locals);
2930         REPLACE_PRINT_DIST("ADR locals per frame", stat_dist_locals_adr);
2931         REPLACE_PRINT_DIST("primitive locals per frame", stat_dist_locals_prim);
2932         REPLACE_PRINT_DIST("RET locals per frame", stat_dist_locals_ret);
2933         REPLACE_PRINT_DIST("void locals per frame", stat_dist_locals_void);
2934         REPLACE_PRINT_DIST("stack slots per frame", stat_dist_stack);
2935         REPLACE_PRINT_DIST("ADR stack slots per frame", stat_dist_stack_adr);
2936         REPLACE_PRINT_DIST("primitive stack slots per frame", stat_dist_stack_prim);
2937         REPLACE_PRINT_DIST("RET stack slots per frame", stat_dist_stack_ret);
2938         printf("\n");
2939         printf("    # of methods:            %d\n", stat_methods);
2940         printf("    # of replacement points: %d\n", stat_rploints);
2941         printf("    # of regallocs:          %d\n", stat_regallocs);
2942         printf("        per rplpoint:        %f\n", (double)stat_regallocs / stat_rploints);
2943         printf("        per method:          %f\n", (double)stat_regallocs / stat_methods);
2944         REPLACE_PRINT_DIST("replacement points per method", stat_dist_method_rplpoints);
2945         printf("\n");
2946
2947 }
2948 #endif /* defined(REPLACE_STATISTICS) */
2949
2950
2951 #if defined(REPLACE_STATISTICS)
2952 static void replace_statistics_source_frame(sourceframe_t *frame)
2953 {
2954         int adr = 0;
2955         int ret = 0;
2956         int prim = 0;
2957         int vd = 0;
2958         int n = 0;
2959         int i;
2960
2961         for (i=0; i<frame->javalocalcount; ++i) {
2962                 switch (frame->javalocaltype[i]) {
2963                         case TYPE_ADR: adr++; break;
2964                         case TYPE_RET: ret++; break;
2965                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
2966                         case TYPE_VOID: vd++; break;
2967                         default: assert(0);
2968                 }
2969                 n++;
2970         }
2971         REPLACE_COUNT_DIST(stat_dist_locals, n);
2972         REPLACE_COUNT_DIST(stat_dist_locals_adr, adr);
2973         REPLACE_COUNT_DIST(stat_dist_locals_void, vd);
2974         REPLACE_COUNT_DIST(stat_dist_locals_ret, ret);
2975         REPLACE_COUNT_DIST(stat_dist_locals_prim, prim);
2976         adr = ret = prim = n = 0;
2977         for (i=0; i<frame->javastackdepth; ++i) {
2978                 switch (frame->javastacktype[i]) {
2979                         case TYPE_ADR: adr++; break;
2980                         case TYPE_RET: ret++; break;
2981                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
2982                 }
2983                 n++;
2984         }
2985         REPLACE_COUNT_DIST(stat_dist_stack, n);
2986         REPLACE_COUNT_DIST(stat_dist_stack_adr, adr);
2987         REPLACE_COUNT_DIST(stat_dist_stack_ret, ret);
2988         REPLACE_COUNT_DIST(stat_dist_stack_prim, prim);
2989 }
2990 #endif /* defined(REPLACE_STATISTICS) */
2991
2992
2993 /* debugging helpers **********************************************************/
2994
2995 /* replace_replacement_point_println *******************************************
2996
2997    Print replacement point info.
2998
2999    IN:
3000        rp...............the replacement point to print
3001
3002 *******************************************************************************/
3003
3004 #if !defined(NDEBUG)
3005
3006 #define TYPECHAR(t)  (((t) >= 0 && (t) <= TYPE_RET) ? show_jit_type_letters[t] : '?')
3007
3008 static char *replace_type_str[] = {
3009         "STD",
3010         "EXH",
3011         "SBR",
3012         "CALL",
3013         "INLINE",
3014         "RETURN",
3015         "BODY"
3016 };
3017
3018 void replace_replacement_point_println(rplpoint *rp, int depth)
3019 {
3020         int j;
3021         int index;
3022
3023         if (!rp) {
3024                 printf("(rplpoint *)NULL\n");
3025                 return;
3026         }
3027
3028         for (j=0; j<depth; ++j)
3029                 putchar('\t');
3030
3031         printf("rplpoint (id %d) %p pc:%p+%d type:%s",
3032                         rp->id, (void*)rp,rp->pc,rp->callsize,
3033                         replace_type_str[rp->type]);
3034         if (rp->flags & RPLPOINT_FLAG_NOTRAP)
3035                 printf(" NOTRAP");
3036         if (rp->flags & RPLPOINT_FLAG_COUNTDOWN)
3037                 printf(" COUNTDOWN");
3038         if (rp->flags & RPLPOINT_FLAG_ACTIVE)
3039                 printf(" ACTIVE");
3040         printf(" parent:%p\n", (void*)rp->parent);
3041         for (j=0; j<depth; ++j)
3042                 putchar('\t');
3043         printf("ra:%d = [",     rp->regalloccount);
3044
3045         for (j=0; j<rp->regalloccount; ++j) {
3046                 if (j)
3047                         putchar(' ');
3048                 index = rp->regalloc[j].index;
3049                 switch (index) {
3050                         case RPLALLOC_STACK: printf("S"); break;
3051                         case RPLALLOC_PARAM: printf("P"); break;
3052                         case RPLALLOC_SYNC : printf("Y"); break;
3053                         default: printf("%d", index);
3054                 }
3055                 printf(":%1c:", TYPECHAR(rp->regalloc[j].type));
3056                 if (rp->regalloc[j].type == TYPE_RET) {
3057                         printf("ret(L%03d)", rp->regalloc[j].regoff);
3058                 }
3059                 else {
3060                         show_allocation(rp->regalloc[j].type, rp->regalloc[j].flags, rp->regalloc[j].regoff);
3061                 }
3062         }
3063
3064         printf("]\n");
3065         for (j=0; j<depth; ++j)
3066                 putchar('\t');
3067         printf("method: ");
3068         method_print(rp->method);
3069
3070         printf("\n");
3071 }
3072 #endif /* !defined(NDEBUG) */
3073
3074
3075 /* replace_show_replacement_points *********************************************
3076
3077    Print replacement point info.
3078
3079    IN:
3080        code.............codeinfo whose replacement points should be printed.
3081
3082 *******************************************************************************/
3083
3084 #if !defined(NDEBUG)
3085 void replace_show_replacement_points(codeinfo *code)
3086 {
3087         int i;
3088         int depth;
3089         rplpoint *rp;
3090         rplpoint *parent;
3091
3092         if (!code) {
3093                 printf("(codeinfo *)NULL\n");
3094                 return;
3095         }
3096
3097         printf("\treplacement points: %d\n",code->rplpointcount);
3098
3099         printf("\ttotal allocations : %d\n",code->regalloccount);
3100         printf("\tsaved int regs    : %d\n",code->savedintcount);
3101         printf("\tsaved flt regs    : %d\n",code->savedfltcount);
3102         printf("\tmemuse            : %d\n",code->memuse);
3103
3104         printf("\n");
3105
3106         for (i=0; i<code->rplpointcount; ++i) {
3107                 rp = code->rplpoints + i;
3108
3109                 depth = 1;
3110                 parent = rp->parent;
3111                 while (parent) {
3112                         depth++;
3113                         parent = parent->parent;
3114                 }
3115                 replace_replacement_point_println(rp, depth);
3116         }
3117 }
3118 #endif
3119
3120
3121 /* replace_executionstate_println **********************************************
3122
3123    Print execution state
3124
3125    IN:
3126        es...............the execution state to print
3127
3128 *******************************************************************************/
3129
3130 #if !defined(NDEBUG)
3131 void replace_executionstate_println(executionstate_t *es)
3132 {
3133         int i;
3134         int slots;
3135         stackslot_t *sp;
3136         int extraslots;
3137
3138         if (!es) {
3139                 printf("(executionstate_t *)NULL\n");
3140                 return;
3141         }
3142
3143         printf("executionstate_t:\n");
3144         printf("\tpc = %p",(void*)es->pc);
3145         printf("  sp = %p",(void*)es->sp);
3146         printf("  pv = %p\n",(void*)es->pv);
3147 #if defined(ENABLE_DISASSEMBLER)
3148         for (i=0; i<INT_REG_CNT; ++i) {
3149                 if (i%4 == 0)
3150                         printf("\t");
3151                 else
3152                         printf(" ");
3153 #if SIZEOF_VOID_P == 8
3154                 printf("%-3s = %016llx",abi_registers_integer_name[i],(unsigned long long)es->intregs[i]);
3155 #else
3156                 printf("%-3s = %08lx",abi_registers_integer_name[i],(unsigned long)es->intregs[i]);
3157 #endif
3158                 if (i%4 == 3)
3159                         printf("\n");
3160         }
3161         for (i=0; i<FLT_REG_CNT; ++i) {
3162                 if (i%4 == 0)
3163                         printf("\t");
3164                 else
3165                         printf(" ");
3166                 printf("F%02d = %016llx",i,(unsigned long long)es->fltregs[i]);
3167                 if (i%4 == 3)
3168                         printf("\n");
3169         }
3170 #endif
3171
3172         sp = (stackslot_t *) es->sp;
3173
3174         extraslots = 2;
3175
3176         if (es->code) {
3177                 methoddesc *md = es->code->m->parseddesc;
3178                 slots = code_get_stack_frame_size(es->code);
3179                 extraslots = 1 + md->memuse;
3180         }
3181         else
3182                 slots = 0;
3183
3184
3185         if (slots) {
3186                 printf("\tstack slots(+%d) at sp:", extraslots);
3187                 for (i=0; i<slots+extraslots; ++i) {
3188                         if (i%4 == 0)
3189                                 printf("\n\t\t");
3190                         printf("M%02d%c", i, (i >= slots) ? '(' : ' ');
3191 #ifdef HAS_4BYTE_STACKSLOT
3192                         printf("%08lx",(unsigned long)*sp++);
3193 #else
3194                         printf("%016llx",(unsigned long long)*sp++);
3195 #endif
3196                         printf("%c", (i >= slots) ? ')' : ' ');
3197                 }
3198                 printf("\n");
3199         }
3200
3201         printf("\tcode: %p", (void*)es->code);
3202         if (es->code != NULL) {
3203                 printf(" stackframesize=%d ", es->code->stackframesize);
3204                 method_print(es->code->m);
3205         }
3206         printf("\n");
3207
3208         printf("\n");
3209 }
3210 #endif
3211
3212 #if !defined(NDEBUG)
3213 static void java_value_print(s4 type, replace_val_t value)
3214 {
3215         java_objectheader *obj;
3216         utf               *u;
3217
3218         printf("%016llx",(unsigned long long) value.l);
3219
3220         if (type < 0 || type > TYPE_RET)
3221                 printf(" <INVALID TYPE:%d>", type);
3222         else
3223                 printf(" %s", show_jit_type_names[type]);
3224
3225         if (type == TYPE_ADR && value.a != NULL) {
3226                 obj = value.a;
3227                 putchar(' ');
3228                 utf_display_printable_ascii_classname(obj->vftbl->class->name);
3229
3230                 if (obj->vftbl->class == class_java_lang_String) {
3231                         printf(" \"");
3232                         u = javastring_toutf(obj, false);
3233                         utf_display_printable_ascii(u);
3234                         printf("\"");
3235                 }
3236         }
3237         else if (type == TYPE_INT) {
3238                 printf(" %ld", (long) value.i);
3239         }
3240         else if (type == TYPE_LNG) {
3241                 printf(" %lld", (long long) value.l);
3242         }
3243         else if (type == TYPE_FLT) {
3244                 printf(" %f", value.f);
3245         }
3246         else if (type == TYPE_DBL) {
3247                 printf(" %f", value.d);
3248         }
3249 }
3250 #endif /* !defined(NDEBUG) */
3251
3252
3253 #if !defined(NDEBUG)
3254 void replace_source_frame_println(sourceframe_t *frame)
3255 {
3256         s4 i,j;
3257         s4 t;
3258
3259         if (REPLACE_IS_NATIVE_FRAME(frame)) {
3260                 printf("\tNATIVE\n");
3261                 printf("\tsfi: "); replace_stackframeinfo_println(frame->sfi);
3262                 printf("\tnativepc: %p\n", frame->nativepc);
3263                 printf("\tframesize: %d\n", frame->nativeframesize);
3264
3265                 j = 0;
3266                 for (i=0; i<INT_REG_CNT; ++i) {
3267                         if (nregdescint[i] == REG_SAV)
3268                                 printf("\t%s = %p\n", abi_registers_integer_name[i], (void*)frame->nativesavint[j++]);
3269                 }
3270
3271                 j = 0;
3272                 for (i=0; i<FLT_REG_CNT; ++i) {
3273                         if (nregdescfloat[i] == REG_SAV)
3274                                 printf("\tF%02d = %f\n", i, frame->nativesavflt[j++]);
3275                 }
3276
3277                 printf("\n");
3278                 return;
3279         }
3280
3281         printf("\t");
3282         method_println(frame->method);
3283         printf("\tid: %d\n", frame->id);
3284         printf("\ttype: %s\n", replace_type_str[frame->type]);
3285         printf("\n");
3286
3287         if (frame->instance.a) {
3288                 printf("\tinstance: ");
3289                 java_value_print(TYPE_ADR, frame->instance);
3290                 printf("\n");
3291         }
3292
3293         if (frame->javalocalcount) {
3294                 printf("\tlocals (%d):\n",frame->javalocalcount);
3295                 for (i=0; i<frame->javalocalcount; ++i) {
3296                         t = frame->javalocaltype[i];
3297                         if (t == TYPE_VOID) {
3298                                 printf("\tlocal[ %2d] = void\n",i);
3299                         }
3300                         else {
3301                                 printf("\tlocal[%c%2d] = ",TYPECHAR(t),i);
3302                                 java_value_print(t, frame->javalocals[i]);
3303                                 printf("\n");
3304                         }
3305                 }
3306                 printf("\n");
3307         }
3308
3309         if (frame->javastackdepth) {
3310                 printf("\tstack (depth %d):\n",frame->javastackdepth);
3311                 for (i=0; i<frame->javastackdepth; ++i) {
3312                         t = frame->javastacktype[i];
3313                         if (t == TYPE_VOID) {
3314                                 printf("\tstack[%2d] = void", i);
3315                         }
3316                         else {
3317                                 printf("\tstack[%2d] = ",i);
3318                                 java_value_print(frame->javastacktype[i], frame->javastack[i]);
3319                                 printf("\n");
3320                         }
3321                 }
3322                 printf("\n");
3323         }
3324
3325         if (frame->syncslotcount) {
3326                 printf("\tsynchronization slots (%d):\n",frame->syncslotcount);
3327                 for (i=0; i<frame->syncslotcount; ++i) {
3328                         printf("\tslot[%2d] = ",i);
3329 #ifdef HAS_4BYTE_STACKSLOT
3330                         printf("%08lx\n",(unsigned long) frame->syncslots[i].p);
3331 #else
3332                         printf("%016llx\n",(unsigned long long) frame->syncslots[i].p);
3333 #endif
3334                 }
3335                 printf("\n");
3336         }
3337
3338         if (frame->fromcode) {
3339                 printf("\tfrom %p ", (void*)frame->fromcode);
3340                 method_println(frame->fromcode->m);
3341         }
3342         if (frame->tocode) {
3343                 printf("\tto %p ", (void*)frame->tocode);
3344                 method_println(frame->tocode->m);
3345         }
3346
3347         if (frame->fromrp) {
3348                 printf("\tfrom replacement point:\n");
3349                 replace_replacement_point_println(frame->fromrp, 2);
3350         }
3351         if (frame->torp) {
3352                 printf("\tto replacement point:\n");
3353                 replace_replacement_point_println(frame->torp, 2);
3354         }
3355
3356         printf("\n");
3357 }
3358 #endif /* !defined(NDEBUG) */
3359
3360
3361 /* replace_sourcestate_println *************************************************
3362
3363    Print source state
3364
3365    IN:
3366        ss...............the source state to print
3367
3368 *******************************************************************************/
3369
3370 #if !defined(NDEBUG)
3371 void replace_sourcestate_println(sourcestate_t *ss)
3372 {
3373         int i;
3374         sourceframe_t *frame;
3375
3376         if (!ss) {
3377                 printf("(sourcestate_t *)NULL\n");
3378                 return;
3379         }
3380
3381         printf("sourcestate_t:\n");
3382
3383         for (i=0, frame = ss->frames; frame != NULL; frame = frame->down, ++i) {
3384                 printf("    frame %d:\n", i);
3385                 replace_source_frame_println(frame);
3386         }
3387 }
3388 #endif
3389
3390
3391 /* replace_sourcestate_println_short *******************************************
3392
3393    Print a compact representation of the given source state.
3394
3395    IN:
3396        ss...............the source state to print
3397
3398 *******************************************************************************/
3399
3400 #if !defined(NDEBUG)
3401 void replace_sourcestate_println_short(sourcestate_t *ss)
3402 {
3403         sourceframe_t *frame;
3404
3405         for (frame = ss->frames; frame != NULL; frame = frame->down) {
3406                 printf("\t");
3407
3408                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
3409                         printf("NATIVE (pc %p size %d) ",
3410                                         (void*)frame->nativepc, frame->nativeframesize);
3411                         replace_stackframeinfo_println(frame->sfi);
3412                         continue;
3413                 }
3414
3415                 if (frame->torp) {
3416                         printf("%c", (frame->torp == frame->fromrp) ? '=' : '+');
3417                 }
3418
3419                 printf("%s", replace_type_str[frame->fromrp->type]);
3420
3421                 if (frame->torp && frame->torp->type != frame->fromrp->type)
3422                         printf("->%s", replace_type_str[frame->torp->type]);
3423
3424                 if (frame->tocode != frame->fromcode)
3425                         printf(" (%p->%p/%d) ",
3426                                    (void*) frame->fromcode, (void*) frame->tocode,
3427                                    frame->fromrp->id);
3428                 else
3429                         printf(" (%p/%d) ", (void*) frame->fromcode, frame->fromrp->id);
3430
3431                 method_println(frame->method);
3432         }
3433 }
3434 #endif
3435
3436 #if !defined(NDEBUG)
3437 static void replace_stackframeinfo_println(stackframeinfo *sfi)
3438 {
3439         printf("prev=%p pv=%p sp=%p ra=%p xpc=%p method=",
3440                         (void*)sfi->prev, (void*)sfi->pv, (void*)sfi->sp,
3441                         (void*)sfi->ra, (void*)sfi->xpc);
3442
3443         if (sfi->method)
3444                 method_println(sfi->method);
3445         else
3446                 printf("(nil)\n");
3447 }
3448 #endif
3449
3450 /*
3451  * These are local overrides for various environment variables in Emacs.
3452  * Please do not remove this and leave it at the end of the file, where
3453  * Emacs will automagically detect them.
3454  * ---------------------------------------------------------------------
3455  * Local variables:
3456  * mode: c
3457  * indent-tabs-mode: t
3458  * c-basic-offset: 4
3459  * tab-width: 4
3460  * End:
3461  * vim:noexpandtab:sw=4:ts=4:
3462  */