src/vm/jit/replace.c

   1 /* src/vm/jit/replace.c - on-stack replacement of methods
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id$
  26
  27 */
  28
  29 #include "config.h"
  30 #include "vm/types.h"
  31
  32 #include <assert.h>
  33 #include <stdlib.h>
  34
  35 #include "arch.h"
  36
  37 #if defined(ENABLE_GC_CACAO)
  38 # include "mm/cacao-gc/gc.h"
  39 #endif
  40
  41 #include "mm/memory.h"
  42
  43 #include "threads/threads-common.h"
  44
  45 #include "toolbox/logging.h"
  46
  47 #include "vm/stringlocal.h"
  48
  49 #include "vm/jit/abi.h"
  50 #include "vm/jit/asmpart.h"
  51 #include "vm/jit/disass.h"
  52 #include "vm/jit/jit.h"
  53 #include "vm/jit/md.h"
  54 #include "vm/jit/methodheader.h"
  55 #include "vm/jit/replace.h"
  56 #include "vm/jit/show.h"
  57 #include "vm/jit/stack.h"
  58
  59 #include "vmcore/options.h"
  60 #include "vmcore/classcache.h"
  61
  62
  63 #define REPLACE_PATCH_DYNAMIC_CALL
  64 /*#define REPLACE_PATCH_ALL*/
  65
  66 #if defined(ENABLE_VMLOG)
  67 #include <vmlog_cacao.h>
  68 #endif
  69
  70 /*** architecture-dependent configuration *************************************/
  71
  72 /* first unset the macros (default) */
  73 #undef REPLACE_RA_BETWEEN_FRAMES
  74 #undef REPLACE_RA_TOP_OF_FRAME
  75 #undef REPLACE_RA_LINKAGE_AREA
  76 #undef REPLACE_LEAFMETHODS_RA_REGISTER
  77 #undef REPLACE_REG_RA
  78
  79 /* i386 and x86_64 */
  80 #if defined(__I386__) || defined(__X86_64__)
  81 #define REPLACE_RA_BETWEEN_FRAMES
  82 /* alpha */
  83 #elif defined(__ALPHA__)
  84 #define REPLACE_RA_TOP_OF_FRAME
  85 #define REPLACE_LEAFMETHODS_RA_REGISTER
  86 #define REPLACE_REG_RA REG_RA
  87 /* powerpc */
  88 #elif defined(__POWERPC__)
  89 #define REPLACE_RA_LINKAGE_AREA
  90 #define REPLACE_LEAFMETHODS_RA_REGISTER
  91 #define REPLACE_REG_RA REG_ITMP3 /* the execution state has the LR in itmp3 */
  92 #endif
  93
  94
  95 /*** configuration of native stack slot size **********************************/
  96
  97 /* XXX this should be in md-abi.h files, probably */
  98
  99 #if defined(HAS_4BYTE_STACKSLOT)
 100 #define SIZE_OF_STACKSLOT      4
 101 #define STACK_SLOTS_PER_FLOAT  2
 102 typedef u4 stackslot_t;
 103 #else
 104 #define SIZE_OF_STACKSLOT      8
 105 #define STACK_SLOTS_PER_FLOAT  1
 106 typedef u8 stackslot_t;
 107 #endif
 108
 109
 110 /*** debugging ****************************************************************/
 111
 112 /*#define REPLACE_VERBOSE*/
 113
 114 #if !defined(NDEBUG)
 115 static void java_value_print(s4 type, replace_val_t value);
 116 static void replace_stackframeinfo_println(stackframeinfo *sfi);
 117 #endif
 118
 119 #if !defined(NDEBUG) && defined(REPLACE_VERBOSE)
 120 int replace_verbose = 0;
 121 #define DOLOG(code)        do{ if (replace_verbose > 1) { code; } } while(0)
 122 #define DOLOG_SHORT(code)  do{ if (replace_verbose > 0) { code; } } while(0)
 123 #else
 124 #define DOLOG(code)
 125 #define DOLOG_SHORT(code)
 126 #endif
 127
 128
 129 /*** statistics ***************************************************************/
 130
 131 #define REPLACE_STATISTICS
 132
 133 #if defined(REPLACE_STATISTICS)
 134
 135 static int stat_replacements = 0;
 136 static int stat_frames = 0;
 137 static int stat_recompile = 0;
 138 static int stat_staticpatch = 0;
 139 static int stat_unroll_inline = 0;
 140 static int stat_unroll_call = 0;
 141 static int stat_dist_frames[20] = { 0 };
 142 static int stat_dist_locals[20] = { 0 };
 143 static int stat_dist_locals_adr[10] = { 0 };
 144 static int stat_dist_locals_prim[10] = { 0 };
 145 static int stat_dist_locals_ret[10] = { 0 };
 146 static int stat_dist_locals_void[10] = { 0 };
 147 static int stat_dist_stack[10] = { 0 };
 148 static int stat_dist_stack_adr[10] = { 0 };
 149 static int stat_dist_stack_prim[10] = { 0 };
 150 static int stat_dist_stack_ret[10] = { 0 };
 151 static int stat_methods = 0;
 152 static int stat_rploints = 0;
 153 static int stat_regallocs = 0;
 154 static int stat_dist_method_rplpoints[20] = { 0 };
 155
 156 #define REPLACE_COUNT(cnt)  (cnt)++
 157 #define REPLACE_COUNT_IF(cnt, cond)  do{ if(cond) (cnt)++; } while(0)
 158 #define REPLACE_COUNT_INC(cnt, inc)  ((cnt) += (inc))
 159
 160 #define REPLACE_COUNT_DIST(array, val)                               \
 161     do {                                                             \
 162         int limit = (sizeof(array) / sizeof(int)) - 1;               \
 163         if ((val) < (limit)) (array)[val]++;                         \
 164         else (array)[limit]++;                                       \
 165     } while (0)
 166
 167 static void replace_statistics_source_frame(sourceframe_t *frame);
 168
 169 #else
 170
 171 #define REPLACE_COUNT(cnt)
 172 #define REPLACE_COUNT_IF(cnt, cond)
 173 #define REPLACE_COUNT_INC(cnt, inc)
 174 #define REPLACE_COUNT_DIST(array, val)
 175
 176 #endif /* defined(REPLACE_STATISTICS) */
 177
 178
 179 /*** constants used internally ************************************************/
 180
 181 #define TOP_IS_NORMAL    0
 182 #define TOP_IS_ON_STACK  1
 183 #define TOP_IS_IN_ITMP1  2
 184 #define TOP_IS_VOID      3
 185
 186
 187 /******************************************************************************/
 188 /* PART I: Creating / freeing replacement points                              */
 189 /******************************************************************************/
 190
 191
 192 /* replace_create_replacement_point ********************************************
 193
 194    Create a replacement point.
 195
 196    IN:
 197        jd...............current jitdata
 198            iinfo............inlining info for the current position
 199            rp...............pre-allocated (uninitialized) rplpoint
 200            type.............RPLPOINT_TYPE constant
 201            iptr.............current instruction
 202            *pra.............current rplalloc pointer
 203            javalocals.......the javalocals at the current point
 204            stackvars........the stack variables at the current point
 205            stackdepth.......the stack depth at the current point
 206            paramcount.......number of parameters at the start of stackvars
 207
 208    OUT:
 209        *rpa.............points to the next free rplalloc
 210
 211 *******************************************************************************/
 212
 213 static void replace_create_replacement_point(jitdata *jd,
 214                                                                                          insinfo_inline *iinfo,
 215                                                                                          rplpoint *rp,
 216                                                                                          s4 type,
 217                                                                                          instruction *iptr,
 218                                                                                          rplalloc **pra,
 219                                                                                          s4 *javalocals,
 220                                                                                          s4 *stackvars,
 221                                                                                          s4 stackdepth,
 222                                                                                          s4 paramcount)
 223 {
 224         rplalloc *ra;
 225         s4        i;
 226         varinfo  *v;
 227         s4        index;
 228
 229         ra = *pra;
 230
 231         REPLACE_COUNT(stat_rploints);
 232
 233         rp->method = (iinfo) ? iinfo->method : jd->m;
 234         rp->pc = NULL;        /* set by codegen */
 235         rp->callsize = 0;     /* set by codegen */
 236         rp->regalloc = ra;
 237         rp->flags = 0;
 238         rp->type = type;
 239         rp->id = iptr->flags.bits >> INS_FLAG_ID_SHIFT;
 240
 241         /* XXX unify these two fields */
 242         rp->parent = (iinfo) ? iinfo->rp : NULL;
 243
 244         /* store local allocation info of javalocals */
 245
 246         if (javalocals) {
 247                 for (i = 0; i < rp->method->maxlocals; ++i) {
 248                         index = javalocals[i];
 249                         if (index == UNUSED)
 250                                 continue;
 251
 252                         ra->index = i;
 253                         if (index >= 0) {
 254                                 v = VAR(index);
 255                                 ra->flags = v->flags & (INMEMORY);
 256                                 ra->regoff = v->vv.regoff;
 257                                 ra->type = v->type;
 258                         }
 259                         else {
 260                                 ra->regoff = RETADDR_FROM_JAVALOCAL(index);
 261                                 ra->type = TYPE_RET;
 262                                 ra->flags = 0;
 263                         }
 264                         ra++;
 265                 }
 266         }
 267
 268         /* store allocation info of java stack vars */
 269
 270         for (i = 0; i < stackdepth; ++i) {
 271                 v = VAR(stackvars[i]);
 272                 ra->flags = v->flags & (INMEMORY);
 273                 ra->index = (i < paramcount) ? RPLALLOC_PARAM : RPLALLOC_STACK;
 274                 ra->type  = v->type;
 275                 /* XXX how to handle locals on the stack containing returnAddresses? */
 276                 if (v->type == TYPE_RET) {
 277                         assert(stackvars[i] >= jd->localcount);
 278                         ra->regoff = v->vv.retaddr->nr;
 279                 }
 280                 else
 281                         ra->regoff = v->vv.regoff;
 282                 ra++;
 283         }
 284
 285         /* total number of allocations */
 286
 287         rp->regalloccount = ra - rp->regalloc;
 288
 289         *pra = ra;
 290 }
 291
 292
 293 /* replace_create_inline_start_replacement_point *******************************
 294
 295    Create an INLINE_START replacement point.
 296
 297    IN:
 298        jd...............current jitdata
 299            rp...............pre-allocated (uninitialized) rplpoint
 300            iptr.............current instruction
 301            *pra.............current rplalloc pointer
 302            javalocals.......the javalocals at the current point
 303
 304    OUT:
 305        *rpa.............points to the next free rplalloc
 306
 307    RETURN VALUE:
 308        the insinfo_inline * for the following inlined body
 309
 310 *******************************************************************************/
 311
 312 static insinfo_inline * replace_create_inline_start_replacement_point(
 313                                                                                          jitdata *jd,
 314                                                                                          rplpoint *rp,
 315                                                                                          instruction *iptr,
 316                                                                                          rplalloc **pra,
 317                                                                                          s4 *javalocals)
 318 {
 319         insinfo_inline *calleeinfo;
 320         rplalloc       *ra;
 321
 322         calleeinfo = iptr->sx.s23.s3.inlineinfo;
 323
 324         calleeinfo->rp = rp;
 325
 326         replace_create_replacement_point(jd, calleeinfo->parent, rp,
 327                         RPLPOINT_TYPE_INLINE, iptr, pra,
 328                         javalocals,
 329                         calleeinfo->stackvars, calleeinfo->stackvarscount,
 330                         calleeinfo->paramcount);
 331
 332         if (calleeinfo->synclocal != UNUSED) {
 333                 ra = (*pra)++;
 334                 ra->index  = RPLALLOC_SYNC;
 335                 ra->regoff = jd->var[calleeinfo->synclocal].vv.regoff;
 336                 ra->flags  = jd->var[calleeinfo->synclocal].flags & INMEMORY;
 337                 ra->type   = TYPE_ADR;
 338
 339                 rp->regalloccount++;
 340         }
 341
 342         return calleeinfo;
 343 }
 344
 345
 346 /* replace_create_replacement_points *******************************************
 347
 348    Create the replacement points for the given code.
 349
 350    IN:
 351        jd...............current jitdata, must not have any replacement points
 352
 353    OUT:
 354        code->rplpoints.......set to the list of replacement points
 355            code->rplpointcount...number of replacement points
 356            code->regalloc........list of allocation info
 357            code->regalloccount...total length of allocation info list
 358            code->globalcount.....number of global allocations at the
 359                                  start of code->regalloc
 360
 361    RETURN VALUE:
 362        true.............everything ok
 363        false............an exception has been thrown
 364
 365 *******************************************************************************/
 366
 367 #define CLEAR_javalocals(array, method)                              \
 368     do {                                                             \
 369         for (i=0; i<(method)->maxlocals; ++i)                        \
 370             (array)[i] = UNUSED;                                     \
 371     } while (0)
 372
 373 #define COPY_OR_CLEAR_javalocals(dest, array, method)                \
 374     do {                                                             \
 375         if ((array) != NULL)                                         \
 376             MCOPY((dest), (array), s4, (method)->maxlocals);         \
 377         else                                                         \
 378             CLEAR_javalocals((dest), (method));                      \
 379     } while (0)
 380
 381 #define COUNT_javalocals(array, method, counter)                     \
 382     do {                                                             \
 383         for (i=0; i<(method)->maxlocals; ++i)                        \
 384             if ((array)[i] != UNUSED)                                \
 385                                 (counter)++;                                         \
 386     } while (0)
 387
 388 bool replace_create_replacement_points(jitdata *jd)
 389 {
 390         codeinfo        *code;
 391         registerdata    *rd;
 392         basicblock      *bptr;
 393         int              count;
 394         methodinfo      *m;
 395         rplpoint        *rplpoints;
 396         rplpoint        *rp;
 397         int              alloccount;
 398         rplalloc        *regalloc;
 399         rplalloc        *ra;
 400         int              i;
 401         instruction     *iptr;
 402         instruction     *iend;
 403         s4              *javalocals;
 404         s4              *jl;
 405         methoddesc      *md;
 406         insinfo_inline  *iinfo;
 407         s4               startcount;
 408         s4               firstcount;
 409 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 410         bool             needentry;
 411 #endif
 412
 413         REPLACE_COUNT(stat_methods);
 414
 415         /* get required compiler data */
 416
 417         code = jd->code;
 418         rd   = jd->rd;
 419
 420         /* assert that we wont overwrite already allocated data */
 421
 422         assert(code);
 423         assert(code->m);
 424         assert(code->rplpoints == NULL);
 425         assert(code->rplpointcount == 0);
 426         assert(code->regalloc == NULL);
 427         assert(code->regalloccount == 0);
 428         assert(code->globalcount == 0);
 429
 430         m = code->m;
 431
 432         /* set codeinfo flags */
 433
 434         if (jd->isleafmethod)
 435                 CODE_SETFLAG_LEAFMETHOD(code);
 436
 437         /* in instance methods, we may need a rplpoint at the method entry */
 438
 439 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 440         if (!(m->flags & ACC_STATIC)) {
 441                 jd->basicblocks[0].bitflags |= BBFLAG_REPLACEMENT;
 442                 needentry = true;
 443         }
 444         else {
 445                 needentry = false;
 446         }
 447 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
 448
 449         /* iterate over the basic block list to find replacement points */
 450
 451         count = 0;
 452         alloccount = 0;
 453
 454         javalocals = DMNEW(s4, jd->maxlocals);
 455
 456         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 457
 458                 /* skip dead code */
 459
 460                 if (bptr->flags < BBFINISHED)
 461                         continue;
 462
 463                 /* get info about this block */
 464
 465                 m = bptr->method;
 466                 iinfo = bptr->inlineinfo;
 467
 468                 /* initialize javalocals at the start of this block */
 469
 470                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 471
 472                 /* iterate over the instructions */
 473
 474                 iptr = bptr->iinstr;
 475                 iend = iptr + bptr->icount;
 476                 startcount = count;
 477                 firstcount = count;
 478
 479                 for (; iptr != iend; ++iptr) {
 480                         switch (iptr->opc) {
 481                                 case ICMD_INVOKESTATIC:
 482                                 case ICMD_INVOKESPECIAL:
 483                                 case ICMD_INVOKEVIRTUAL:
 484                                 case ICMD_INVOKEINTERFACE:
 485                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 486                                         count++;
 487                                         COUNT_javalocals(javalocals, m, alloccount);
 488                                         alloccount += iptr->s1.argcount;
 489                                         if (iinfo)
 490                                                 alloccount -= iinfo->throughcount;
 491                                         break;
 492
 493                                 case ICMD_ISTORE:
 494                                 case ICMD_LSTORE:
 495                                 case ICMD_FSTORE:
 496                                 case ICMD_DSTORE:
 497                                 case ICMD_ASTORE:
 498                                         stack_javalocals_store(iptr, javalocals);
 499                                         break;
 500
 501                                 case ICMD_IRETURN:
 502                                 case ICMD_LRETURN:
 503                                 case ICMD_FRETURN:
 504                                 case ICMD_DRETURN:
 505                                 case ICMD_ARETURN:
 506                                         alloccount += 1;
 507                                         /* FALLTHROUGH! */
 508                                 case ICMD_RETURN:
 509                                         count++;
 510                                         break;
 511
 512                                 case ICMD_INLINE_START:
 513                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 514
 515                                         count++;
 516                                         COUNT_javalocals(javalocals, m, alloccount);
 517                                         alloccount += iinfo->stackvarscount;
 518                                         if (iinfo->synclocal != UNUSED)
 519                                                 alloccount++;
 520
 521                                         m = iinfo->method;
 522                                         /* javalocals may be set at next block start, or now */
 523                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 524                                         break;
 525
 526                                 case ICMD_INLINE_BODY:
 527                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 528
 529                                         jl = iinfo->javalocals_start;
 530                                         if (jl == NULL) {
 531                                                 /* get the javalocals from the following block start */
 532                                                 assert(bptr->next);
 533                                                 jl = bptr->next->javalocals;
 534                                         }
 535                                         count++;
 536                                         COUNT_javalocals(jl, m, alloccount);
 537                                         break;
 538
 539                                 case ICMD_INLINE_END:
 540                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 541                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 542                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 543                                         m = iinfo->outer;
 544                                         if (iinfo->javalocals_end)
 545                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 546                                         iinfo = iinfo->parent;
 547                                         break;
 548                         }
 549
 550                         if (iptr == bptr->iinstr)
 551                                 firstcount = count;
 552                 } /* end instruction loop */
 553
 554                 /* create replacement points at targets of backward branches */
 555                 /* We only need the replacement point there, if there is no  */
 556                 /* replacement point inside the block.                       */
 557
 558                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 559 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 560                         int test = (needentry && bptr == jd->basicblocks) ? firstcount : count;
 561 #else
 562                         int test = count;
 563 #endif
 564                         if (test > startcount) {
 565                                 /* we don't need an extra rplpoint */
 566                                 bptr->bitflags &= ~BBFLAG_REPLACEMENT;
 567                         }
 568                         else {
 569                                 count++;
 570                                 alloccount += bptr->indepth;
 571                                 if (bptr->inlineinfo)
 572                                         alloccount -= bptr->inlineinfo->throughcount;
 573
 574                                 COUNT_javalocals(bptr->javalocals, bptr->method, alloccount);
 575                         }
 576                 }
 577
 578         } /* end basicblock loop */
 579
 580         /* if no points were found, there's nothing to do */
 581
 582         if (!count)
 583                 return true;
 584
 585         /* allocate replacement point array and allocation array */
 586
 587         rplpoints = MNEW(rplpoint, count);
 588         regalloc = MNEW(rplalloc, alloccount);
 589         ra = regalloc;
 590
 591         /* initialize replacement point structs */
 592
 593         rp = rplpoints;
 594
 595         /* XXX try to share code with the counting loop! */
 596
 597         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 598                 /* skip dead code */
 599
 600                 if (bptr->flags < BBFINISHED)
 601                         continue;
 602
 603                 /* get info about this block */
 604
 605                 m = bptr->method;
 606                 iinfo = bptr->inlineinfo;
 607
 608                 /* initialize javalocals at the start of this block */
 609
 610                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 611
 612                 /* create replacement points at targets of backward branches */
 613
 614                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 615
 616                         i = (iinfo) ? iinfo->throughcount : 0;
 617                         replace_create_replacement_point(jd, iinfo, rp++,
 618                                         bptr->type, bptr->iinstr, &ra,
 619                                         bptr->javalocals, bptr->invars + i, bptr->indepth - i, 0);
 620
 621                         if (JITDATA_HAS_FLAG_COUNTDOWN(jd))
 622                                 rp[-1].flags |= RPLPOINT_FLAG_COUNTDOWN;
 623                 }
 624
 625                 /* iterate over the instructions */
 626
 627                 iptr = bptr->iinstr;
 628                 iend = iptr + bptr->icount;
 629
 630                 for (; iptr != iend; ++iptr) {
 631                         switch (iptr->opc) {
 632                                 case ICMD_INVOKESTATIC:
 633                                 case ICMD_INVOKESPECIAL:
 634                                 case ICMD_INVOKEVIRTUAL:
 635                                 case ICMD_INVOKEINTERFACE:
 636                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 637
 638                                         i = (iinfo) ? iinfo->throughcount : 0;
 639                                         replace_create_replacement_point(jd, iinfo, rp++,
 640                                                         RPLPOINT_TYPE_CALL, iptr, &ra,
 641                                                         javalocals, iptr->sx.s23.s2.args,
 642                                                         iptr->s1.argcount - i,
 643                                                         md->paramcount);
 644                                         break;
 645
 646                                 case ICMD_ISTORE:
 647                                 case ICMD_LSTORE:
 648                                 case ICMD_FSTORE:
 649                                 case ICMD_DSTORE:
 650                                 case ICMD_ASTORE:
 651                                         stack_javalocals_store(iptr, javalocals);
 652                                         break;
 653
 654                                 case ICMD_IRETURN:
 655                                 case ICMD_LRETURN:
 656                                 case ICMD_FRETURN:
 657                                 case ICMD_DRETURN:
 658                                 case ICMD_ARETURN:
 659                                         replace_create_replacement_point(jd, iinfo, rp++,
 660                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 661                                                         NULL, &(iptr->s1.varindex), 1, 0);
 662                                         break;
 663
 664                                 case ICMD_RETURN:
 665                                         replace_create_replacement_point(jd, iinfo, rp++,
 666                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 667                                                         NULL, NULL, 0, 0);
 668                                         break;
 669
 670                                 case ICMD_INLINE_START:
 671                                         iinfo = replace_create_inline_start_replacement_point(
 672                                                                 jd, rp++, iptr, &ra, javalocals);
 673                                         m = iinfo->method;
 674                                         /* javalocals may be set at next block start, or now */
 675                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 676                                         break;
 677
 678                                 case ICMD_INLINE_BODY:
 679                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 680
 681                                         jl = iinfo->javalocals_start;
 682                                         if (jl == NULL) {
 683                                                 /* get the javalocals from the following block start */
 684                                                 assert(bptr->next);
 685                                                 jl = bptr->next->javalocals;
 686                                         }
 687                                         /* create a non-trappable rplpoint */
 688                                         replace_create_replacement_point(jd, iinfo, rp++,
 689                                                         RPLPOINT_TYPE_BODY, iptr, &ra,
 690                                                         jl, NULL, 0, 0);
 691                                         rp[-1].flags |= RPLPOINT_FLAG_NOTRAP;
 692                                         break;
 693
 694                                 case ICMD_INLINE_END:
 695                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 696                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 697                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 698                                         m = iinfo->outer;
 699                                         if (iinfo->javalocals_end)
 700                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 701                                         iinfo = iinfo->parent;
 702                                         break;
 703                         }
 704                 } /* end instruction loop */
 705         } /* end basicblock loop */
 706
 707         assert((rp - rplpoints) == count);
 708         assert((ra - regalloc) == alloccount);
 709
 710         /* store the data in the codeinfo */
 711
 712         code->rplpoints     = rplpoints;
 713         code->rplpointcount = count;
 714         code->regalloc      = regalloc;
 715         code->regalloccount = alloccount;
 716         code->globalcount   = 0;
 717         code->savedintcount = INT_SAV_CNT - rd->savintreguse;
 718         code->savedfltcount = FLT_SAV_CNT - rd->savfltreguse;
 719         code->memuse        = rd->memuse;
 720         code->stackframesize = jd->cd->stackframesize;
 721
 722         REPLACE_COUNT_DIST(stat_dist_method_rplpoints, count);
 723         REPLACE_COUNT_INC(stat_regallocs, alloccount);
 724
 725         /* everything alright */
 726
 727         return true;
 728 }
 729
 730
 731 /* replace_free_replacement_points *********************************************
 732
 733    Free memory used by replacement points.
 734
 735    IN:
 736        code.............codeinfo whose replacement points should be freed.
 737
 738 *******************************************************************************/
 739
 740 void replace_free_replacement_points(codeinfo *code)
 741 {
 742         assert(code);
 743
 744         if (code->rplpoints)
 745                 MFREE(code->rplpoints,rplpoint,code->rplpointcount);
 746
 747         if (code->regalloc)
 748                 MFREE(code->regalloc,rplalloc,code->regalloccount);
 749
 750         code->rplpoints = NULL;
 751         code->rplpointcount = 0;
 752         code->regalloc = NULL;
 753         code->regalloccount = 0;
 754         code->globalcount = 0;
 755 }
 756
 757
 758 /******************************************************************************/
 759 /* PART II: Activating / deactivating replacement points                      */
 760 /******************************************************************************/
 761
 762
 763 /* replace_activate_replacement_points *****************************************
 764
 765    Activate the replacement points of the given compilation unit. When this
 766    function returns, the replacement points are "armed", so each thread
 767    reaching one of the points will enter the replacement mechanism.
 768
 769    IN:
 770        code.............codeinfo of which replacement points should be
 771                                                 activated
 772            mappable.........if true, only mappable replacement points are
 773                                                 activated
 774
 775 *******************************************************************************/
 776
 777 void replace_activate_replacement_points(codeinfo *code, bool mappable)
 778 {
 779         rplpoint *rp;
 780         s4        i;
 781         s4        count;
 782         s4        index;
 783         u1       *savedmcode;
 784
 785         assert(code->savedmcode == NULL);
 786
 787         /* count trappable replacement points */
 788
 789         count = 0;
 790         index = 0;
 791         i = code->rplpointcount;
 792         rp = code->rplpoints;
 793         for (; i--; rp++) {
 794                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 795                         continue;
 796
 797                 index++;
 798
 799                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 800                         continue;
 801
 802                 count++;
 803         }
 804
 805         /* allocate buffer for saved machine code */
 806
 807         savedmcode = MNEW(u1, count * REPLACEMENT_PATCH_SIZE);
 808         code->savedmcode = savedmcode;
 809         savedmcode += count * REPLACEMENT_PATCH_SIZE;
 810
 811         /* activate trappable replacement points */
 812         /* (in reverse order to handle overlapping points within basic blocks) */
 813
 814         i = code->rplpointcount;
 815         rp = code->rplpoints + i;
 816         while (rp--, i--) {
 817                 assert(!(rp->flags & RPLPOINT_FLAG_ACTIVE));
 818
 819                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 820                         continue;
 821
 822                 index--;
 823
 824                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 825                         continue;
 826
 827                 DOLOG( printf("activate replacement point:\n");
 828                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 829
 830                 savedmcode -= REPLACEMENT_PATCH_SIZE;
 831
 832 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__)) && defined(ENABLE_JIT)
 833                 md_patch_replacement_point(code, index, rp, savedmcode);
 834 #endif
 835                 rp->flags |= RPLPOINT_FLAG_ACTIVE;
 836         }
 837
 838         assert(savedmcode == code->savedmcode);
 839 }
 840
 841
 842 /* replace_deactivate_replacement_points ***************************************
 843
 844    Deactivate a replacement points in the given compilation unit.
 845    When this function returns, the replacement points will be "un-armed",
 846    that is a each thread reaching a point will just continue normally.
 847
 848    IN:
 849        code.............the compilation unit
 850
 851 *******************************************************************************/
 852
 853 void replace_deactivate_replacement_points(codeinfo *code)
 854 {
 855         rplpoint *rp;
 856         s4        i;
 857         s4        count;
 858         u1       *savedmcode;
 859
 860         if (code->savedmcode == NULL) {
 861                 /* disarm countdown points by patching the branches */
 862
 863                 i = code->rplpointcount;
 864                 rp = code->rplpoints;
 865                 for (; i--; rp++) {
 866                         if ((rp->flags & (RPLPOINT_FLAG_ACTIVE | RPLPOINT_FLAG_COUNTDOWN))
 867                                         == RPLPOINT_FLAG_COUNTDOWN)
 868                         {
 869 #if 0
 870                                 *(s4*) (rp->pc + 9) = 0; /* XXX machine dependent! */
 871 #endif
 872                         }
 873                 }
 874                 return;
 875         }
 876
 877         assert(code->savedmcode != NULL);
 878         savedmcode = code->savedmcode;
 879
 880         /* de-activate each trappable replacement point */
 881
 882         i = code->rplpointcount;
 883         rp = code->rplpoints;
 884         count = 0;
 885         for (; i--; rp++) {
 886                 if (!(rp->flags & RPLPOINT_FLAG_ACTIVE))
 887                         continue;
 888
 889                 count++;
 890
 891                 DOLOG( printf("deactivate replacement point:\n");
 892                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 893
 894 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__)) && defined(ENABLE_JIT)
 895                 md_patch_replacement_point(code, -1, rp, savedmcode);
 896 #endif
 897
 898                 rp->flags &= ~RPLPOINT_FLAG_ACTIVE;
 899
 900                 savedmcode += REPLACEMENT_PATCH_SIZE;
 901         }
 902
 903         assert(savedmcode == code->savedmcode + count * REPLACEMENT_PATCH_SIZE);
 904
 905         /* free saved machine code */
 906
 907         MFREE(code->savedmcode, u1, count * REPLACEMENT_PATCH_SIZE);
 908         code->savedmcode = NULL;
 909 }
 910
 911
 912 /******************************************************************************/
 913 /* PART III: The replacement mechanism                                        */
 914 /******************************************************************************/
 915
 916
 917 /* replace_read_value **********************************************************
 918
 919    Read a value with the given allocation from the execution state.
 920
 921    IN:
 922            es...............execution state
 923            sp...............stack pointer of the execution state (XXX eliminate?)
 924            ra...............allocation
 925            javaval..........where to put the value
 926
 927    OUT:
 928        *javaval.........the value
 929
 930 *******************************************************************************/
 931
 932 static void replace_read_value(executionstate_t *es,
 933                                                            stackslot_t *sp,
 934                                                            rplalloc *ra,
 935                                                            replace_val_t *javaval)
 936 {
 937         if (ra->flags & INMEMORY) {
 938                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
 939 #ifdef HAS_4BYTE_STACKSLOT
 940                 if (IS_2_WORD_TYPE(ra->type)) {
 941                         javaval->l = *(u8*)(sp + ra->regoff);
 942                 }
 943                 else {
 944 #endif
 945                         javaval->p = sp[ra->regoff];
 946 #ifdef HAS_4BYTE_STACKSLOT
 947                 }
 948 #endif
 949         }
 950         else {
 951                 /* allocated register */
 952                 if (IS_FLT_DBL_TYPE(ra->type)) {
 953                         javaval->d = es->fltregs[ra->regoff];
 954
 955                         if (ra->type == TYPE_FLT)
 956                                 javaval->f = javaval->d;
 957                 }
 958                 else {
 959 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
 960                         if (ra->type == TYPE_LNG) {
 961                                 javaval->words.lo = es->intregs[GET_LOW_REG(ra->regoff)];
 962                                 javaval->words.hi = es->intregs[GET_HIGH_REG(ra->regoff)];
 963                         }
 964                         else
 965 #endif /* defined(SUPPORT_COMBINE_INTEGER_REGISTERS) */
 966                                 javaval->p = es->intregs[ra->regoff];
 967                 }
 968         }
 969 }
 970
 971
 972 /* replace_write_value *********************************************************
 973
 974    Write a value to the given allocation in the execution state.
 975
 976    IN:
 977            es...............execution state
 978            sp...............stack pointer of the execution state (XXX eliminate?)
 979            ra...............allocation
 980            *javaval.........the value
 981
 982 *******************************************************************************/
 983
 984 static void replace_write_value(executionstate_t *es,
 985                                                             stackslot_t *sp,
 986                                                             rplalloc *ra,
 987                                                             replace_val_t *javaval)
 988 {
 989         if (ra->flags & INMEMORY) {
 990                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
 991 #ifdef HAS_4BYTE_STACKSLOT
 992                 if (IS_2_WORD_TYPE(ra->type)) {
 993                         *(u8*)(sp + ra->regoff) = javaval->l;
 994                 }
 995                 else {
 996 #endif
 997                         sp[ra->regoff] = javaval->p;
 998 #ifdef HAS_4BYTE_STACKSLOT
 999                 }
1000 #endif
1001         }
1002         else {
1003                 /* allocated register */
1004                 switch (ra->type) {
1005                         case TYPE_FLT:
1006                                 es->fltregs[ra->regoff] = (double) javaval->f;
1007                                 break;
1008                         case TYPE_DBL:
1009                                 es->fltregs[ra->regoff] = javaval->d;
1010                                 break;
1011 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
1012                         case TYPE_LNG:
1013                                 es->intregs[GET_LOW_REG(ra->regoff)] = javaval->words.lo;
1014                                 es->intregs[GET_HIGH_REG(ra->regoff)] = javaval->words.hi;
1015                                 break;
1016 #endif
1017                         default:
1018                                 es->intregs[ra->regoff] = javaval->p;
1019                 }
1020         }
1021 }
1022
1023
1024 /* replace_read_executionstate *************************************************
1025
1026    Read the given executions state and translate it to a source frame.
1027
1028    IN:
1029            ss...............the source state
1030
1031    OUT:
1032            ss->frames.......set to new frame
1033
1034    RETURN VALUE:
1035        returns the new frame
1036
1037 *******************************************************************************/
1038
1039 static sourceframe_t *replace_new_sourceframe(sourcestate_t *ss)
1040 {
1041         sourceframe_t *frame;
1042
1043         frame = DNEW(sourceframe_t);
1044         MZERO(frame, sourceframe_t, 1);
1045
1046         frame->down = ss->frames;
1047         ss->frames = frame;
1048
1049         return frame;
1050 }
1051
1052
1053 /* replace_read_executionstate *************************************************
1054
1055    Read the given executions state and translate it to a source frame.
1056
1057    IN:
1058        rp...............replacement point at which `es` was taken
1059            es...............execution state
1060            ss...............where to put the source state
1061
1062    OUT:
1063        *ss..............the source state derived from the execution state
1064
1065 *******************************************************************************/
1066
1067 static s4 replace_normalize_type_map[] = {
1068 /* RPLPOINT_TYPE_STD    |--> */ RPLPOINT_TYPE_STD,
1069 /* RPLPOINT_TYPE_EXH    |--> */ RPLPOINT_TYPE_STD,
1070 /* RPLPOINT_TYPE_SBR    |--> */ RPLPOINT_TYPE_STD,
1071 /* RPLPOINT_TYPE_CALL   |--> */ RPLPOINT_TYPE_CALL,
1072 /* RPLPOINT_TYPE_INLINE |--> */ RPLPOINT_TYPE_CALL,
1073 /* RPLPOINT_TYPE_RETURN |--> */ RPLPOINT_TYPE_RETURN,
1074 /* RPLPOINT_TYPE_BODY   |--> */ RPLPOINT_TYPE_STD
1075 };
1076
1077
1078 static void replace_read_executionstate(rplpoint *rp,
1079                                                                                 executionstate_t *es,
1080                                                                                 sourcestate_t *ss,
1081                                                                                 bool topframe)
1082 {
1083         methodinfo    *m;
1084         codeinfo      *code;
1085         int            count;
1086         int            i;
1087         rplalloc      *ra;
1088         sourceframe_t *frame;
1089         int            topslot;
1090         stackslot_t   *sp;
1091         stackslot_t   *basesp;
1092
1093         code = code_find_codeinfo_for_pc(rp->pc);
1094         m = rp->method;
1095         topslot = TOP_IS_NORMAL;
1096
1097         /* stack pointer */
1098
1099         sp = (stackslot_t *) es->sp;
1100
1101         /* in some cases the top stack slot is passed in REG_ITMP1 */
1102
1103         if (rp->type == BBTYPE_EXH) {
1104                 topslot = TOP_IS_IN_ITMP1;
1105         }
1106
1107         /* calculate base stack pointer */
1108
1109         basesp = sp + code_get_stack_frame_size(code);
1110
1111         /* create the source frame */
1112
1113         frame = replace_new_sourceframe(ss);
1114         frame->method = rp->method;
1115         frame->id = rp->id;
1116         assert(rp->type >= 0 && rp->type < sizeof(replace_normalize_type_map)/sizeof(s4));
1117         frame->type = replace_normalize_type_map[rp->type];
1118         frame->fromrp = rp;
1119         frame->fromcode = code;
1120
1121         /* read local variables */
1122
1123         count = m->maxlocals;
1124         frame->javalocalcount = count;
1125         frame->javalocals = DMNEW(replace_val_t, count);
1126         frame->javalocaltype = DMNEW(u1, count);
1127
1128         /* mark values as undefined */
1129         for (i=0; i<count; ++i) {
1130 #if !defined(NDEBUG)
1131                 frame->javalocals[i].l = (u8) 0x00dead0000dead00ULL;
1132 #endif
1133                 frame->javalocaltype[i] = TYPE_VOID;
1134         }
1135
1136         /* some entries in the intregs array are not meaningful */
1137         /*es->intregs[REG_ITMP3] = (u8) 0x11dead1111dead11ULL;*/
1138 #if !defined(NDEBUG)
1139         es->intregs[REG_SP   ] = (ptrint) 0x11dead1111dead11ULL;
1140 #ifdef REG_PV
1141         es->intregs[REG_PV   ] = (ptrint) 0x11dead1111dead11ULL;
1142 #endif
1143 #endif /* !defined(NDEBUG) */
1144
1145         /* read javalocals */
1146
1147         count = rp->regalloccount;
1148         ra = rp->regalloc;
1149
1150         while (count && (i = ra->index) >= 0) {
1151                 assert(i < m->maxlocals);
1152                 frame->javalocaltype[i] = ra->type;
1153                 if (ra->type == TYPE_RET)
1154                         frame->javalocals[i].i = ra->regoff;
1155                 else
1156                         replace_read_value(es, sp, ra, frame->javalocals + i);
1157                 ra++;
1158                 count--;
1159         }
1160
1161         /* read instance, if this is the first rplpoint */
1162
1163 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
1164         if (topframe && !(rp->method->flags & ACC_STATIC) && rp == code->rplpoints) {
1165 #if 1
1166                 /* we are at the start of the method body, so if local 0 is set, */
1167                 /* it is the instance.                                           */
1168                 if (frame->javalocaltype[0] == TYPE_ADR)
1169                         frame->instance = frame->javalocals[0];
1170 #else
1171                 rplalloc instra;
1172                 methoddesc *md;
1173
1174                 md = rp->method->parseddesc;
1175                 assert(md->params);
1176                 assert(md->paramcount >= 1);
1177                 instra.type = TYPE_ADR;
1178                 instra.regoff = md->params[0].regoff;
1179                 if (md->params[0].inmemory) {
1180                         instra.flags = INMEMORY;
1181                         instra.regoff += (1 + code->stackframesize);
1182                 }
1183                 else {
1184                         instra.flags = 0;
1185                 }
1186                 replace_read_value(es, sp, &instra, &(frame->instance));
1187 #endif
1188         }
1189 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
1190
1191         /* read stack slots */
1192
1193         frame->javastackdepth = count;
1194         frame->javastack = DMNEW(replace_val_t, count);
1195         frame->javastacktype = DMNEW(u1, count);
1196
1197 #if !defined(NDEBUG)
1198         /* mark values as undefined */
1199         for (i=0; i<count; ++i) {
1200                 frame->javastack[i].l = (u8) 0x00dead0000dead00ULL;
1201                 frame->javastacktype[i] = TYPE_VOID;
1202         }
1203 #endif /* !defined(NDEBUG) */
1204
1205         i = 0;
1206
1207         /* the first stack slot is special in SBR and EXH blocks */
1208
1209         if (topslot == TOP_IS_ON_STACK) {
1210                 assert(count);
1211
1212                 assert(ra->index == RPLALLOC_STACK);
1213                 assert(ra->type == TYPE_ADR);
1214                 frame->javastack[i].p = sp[-1];
1215                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1216                 count--;
1217                 i++;
1218                 ra++;
1219         }
1220         else if (topslot == TOP_IS_IN_ITMP1) {
1221                 assert(count);
1222
1223                 assert(ra->index == RPLALLOC_STACK);
1224                 assert(ra->type == TYPE_ADR);
1225                 frame->javastack[i].p = es->intregs[REG_ITMP1];
1226                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1227                 count--;
1228                 i++;
1229                 ra++;
1230         }
1231         else if (topslot == TOP_IS_VOID) {
1232                 assert(count);
1233
1234                 assert(ra->index == RPLALLOC_STACK);
1235                 frame->javastack[i].l = 0;
1236                 frame->javastacktype[i] = TYPE_VOID;
1237                 count--;
1238                 i++;
1239                 ra++;
1240         }
1241
1242         /* read remaining stack slots */
1243
1244         for (; count--; ra++) {
1245                 if (ra->index == RPLALLOC_SYNC) {
1246                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1247
1248                         /* only read synchronization slots when traversing an inline point */
1249
1250                         if (!topframe) {
1251                                 sourceframe_t *calleeframe = frame->down;
1252                                 assert(calleeframe);
1253                                 assert(calleeframe->syncslotcount == 0);
1254                                 assert(calleeframe->syncslots == NULL);
1255
1256                                 calleeframe->syncslotcount = 1;
1257                                 calleeframe->syncslots = DMNEW(replace_val_t, 1);
1258                                 replace_read_value(es,sp,ra,calleeframe->syncslots);
1259                         }
1260
1261                         frame->javastackdepth--;
1262                         continue;
1263                 }
1264
1265                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1266
1267                 /* do not read parameters of calls down the call chain */
1268
1269                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1270                         frame->javastackdepth--;
1271                 }
1272                 else {
1273                         if (ra->type == TYPE_RET)
1274                                 frame->javastack[i].i = ra->regoff;
1275                         else
1276                                 replace_read_value(es,sp,ra,frame->javastack + i);
1277                         frame->javastacktype[i] = ra->type;
1278                         i++;
1279                 }
1280         }
1281 }
1282
1283
1284 /* replace_write_executionstate ************************************************
1285
1286    Translate the given source state into an execution state.
1287
1288    IN:
1289        rp...............replacement point for which execution state should be
1290                             creates
1291            es...............where to put the execution state
1292            ss...............the given source state
1293
1294    OUT:
1295        *es..............the execution state derived from the source state
1296
1297 *******************************************************************************/
1298
1299 static void replace_write_executionstate(rplpoint *rp,
1300                                                                                  executionstate_t *es,
1301                                                                                  sourcestate_t *ss,
1302                                                                                  bool topframe)
1303 {
1304         methodinfo     *m;
1305         codeinfo       *code;
1306         int             count;
1307         int             i;
1308         rplalloc       *ra;
1309         sourceframe_t  *frame;
1310         int             topslot;
1311         stackslot_t    *sp;
1312         stackslot_t    *basesp;
1313
1314         code = code_find_codeinfo_for_pc(rp->pc);
1315         m = rp->method;
1316         topslot = TOP_IS_NORMAL;
1317
1318         /* pop a source frame */
1319
1320         frame = ss->frames;
1321         assert(frame);
1322         ss->frames = frame->down;
1323
1324         /* calculate stack pointer */
1325
1326         sp = (stackslot_t *) es->sp;
1327
1328         basesp = sp + code_get_stack_frame_size(code);
1329
1330         /* in some cases the top stack slot is passed in REG_ITMP1 */
1331
1332         if (rp->type == BBTYPE_EXH) {
1333                 topslot = TOP_IS_IN_ITMP1;
1334         }
1335
1336         /* write javalocals */
1337
1338         ra = rp->regalloc;
1339         count = rp->regalloccount;
1340
1341         while (count && (i = ra->index) >= 0) {
1342                 assert(i < m->maxlocals);
1343                 assert(i < frame->javalocalcount);
1344                 assert(ra->type == frame->javalocaltype[i]);
1345                 if (ra->type == TYPE_RET) {
1346                         /* XXX assert that it matches this rplpoint */
1347                 }
1348                 else
1349                         replace_write_value(es, sp, ra, frame->javalocals + i);
1350                 count--;
1351                 ra++;
1352         }
1353
1354         /* write stack slots */
1355
1356         i = 0;
1357
1358         /* the first stack slot is special in SBR and EXH blocks */
1359
1360         if (topslot == TOP_IS_ON_STACK) {
1361                 assert(count);
1362
1363                 assert(ra->index == RPLALLOC_STACK);
1364                 assert(i < frame->javastackdepth);
1365                 assert(frame->javastacktype[i] == TYPE_ADR);
1366                 sp[-1] = frame->javastack[i].p;
1367                 count--;
1368                 i++;
1369                 ra++;
1370         }
1371         else if (topslot == TOP_IS_IN_ITMP1) {
1372                 assert(count);
1373
1374                 assert(ra->index == RPLALLOC_STACK);
1375                 assert(i < frame->javastackdepth);
1376                 assert(frame->javastacktype[i] == TYPE_ADR);
1377                 es->intregs[REG_ITMP1] = frame->javastack[i].p;
1378                 count--;
1379                 i++;
1380                 ra++;
1381         }
1382         else if (topslot == TOP_IS_VOID) {
1383                 assert(count);
1384
1385                 assert(ra->index == RPLALLOC_STACK);
1386                 assert(i < frame->javastackdepth);
1387                 assert(frame->javastacktype[i] == TYPE_VOID);
1388                 count--;
1389                 i++;
1390                 ra++;
1391         }
1392
1393         /* write remaining stack slots */
1394
1395         for (; count--; ra++) {
1396                 if (ra->index == RPLALLOC_SYNC) {
1397                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1398
1399                         /* only write synchronization slots when traversing an inline point */
1400
1401                         if (!topframe) {
1402                                 assert(frame->down);
1403                                 assert(frame->down->syncslotcount == 1); /* XXX need to understand more cases */
1404                                 assert(frame->down->syncslots != NULL);
1405
1406                                 replace_write_value(es,sp,ra,frame->down->syncslots);
1407                         }
1408                         continue;
1409                 }
1410
1411                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1412
1413                 /* do not write parameters of calls down the call chain */
1414
1415                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1416                         /* skip it */
1417                 }
1418                 else {
1419                         assert(i < frame->javastackdepth);
1420                         assert(ra->type == frame->javastacktype[i]);
1421                         if (ra->type == TYPE_RET) {
1422                                 /* XXX assert that it matches this rplpoint */
1423                         }
1424                         else {
1425                                 replace_write_value(es,sp,ra,frame->javastack + i);
1426                         }
1427                         i++;
1428                 }
1429         }
1430
1431         /* set new pc */
1432
1433         es->pc = rp->pc;
1434 }
1435
1436
1437 /* replace_pop_activation_record ***********************************************
1438
1439    Peel a stack frame from the execution state.
1440
1441    *** This function imitates the effects of the method epilog ***
1442    *** and returning from the method call.                     ***
1443
1444    IN:
1445            es...............execution state
1446            frame............source frame, receives synchronization slots
1447
1448    OUT:
1449        *es..............the execution state after popping the stack frame
1450
1451 *******************************************************************************/
1452
1453 u1* replace_pop_activation_record(executionstate_t *es,
1454                                                                   sourceframe_t *frame)
1455 {
1456         u1 *ra;
1457         u1 *pv;
1458         s4 reg;
1459         s4 i;
1460         s4 count;
1461         codeinfo *code;
1462         stackslot_t *basesp;
1463         stackslot_t *sp;
1464
1465         assert(es->code);
1466         assert(frame);
1467
1468         /* read the return address */
1469
1470 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1471         if (CODE_IS_LEAFMETHOD(es->code))
1472                 ra = (u1*) (ptrint) es->intregs[REPLACE_REG_RA];
1473         else
1474 #endif
1475                 ra = md_stacktrace_get_returnaddress(es->sp,
1476                                 SIZE_OF_STACKSLOT * es->code->stackframesize);
1477
1478         DOLOG( printf("return address: %p\n", (void*)ra); );
1479
1480         assert(ra);
1481
1482         /* calculate the base of the stack frame */
1483
1484         sp = (stackslot_t *) es->sp;
1485         basesp = sp + es->code->stackframesize;
1486
1487         /* read slots used for synchronization */
1488
1489         assert(frame->syncslotcount == 0);
1490         assert(frame->syncslots == NULL);
1491         count = code_get_sync_slot_count(es->code);
1492         frame->syncslotcount = count;
1493         frame->syncslots = DMNEW(replace_val_t, count);
1494         for (i=0; i<count; ++i) {
1495                 frame->syncslots[i].p = sp[es->code->memuse + i]; /* XXX */
1496         }
1497
1498         /* restore return address, if part of frame */
1499
1500 #if defined(REPLACE_RA_TOP_OF_FRAME)
1501 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1502         if (!CODE_IS_LEAFMETHOD(es->code))
1503 #endif
1504                 es->intregs[REPLACE_REG_RA] = *--basesp;
1505 #endif /* REPLACE_RA_TOP_OF_FRAME */
1506
1507 #if defined(REPLACE_RA_LINKAGE_AREA)
1508 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1509         if (!CODE_IS_LEAFMETHOD(es->code))
1510 #endif
1511                 es->intregs[REPLACE_REG_RA] = basesp[LA_LR_OFFSET / sizeof(stackslot_t)];
1512 #endif /* REPLACE_RA_LINKAGE_AREA */
1513
1514         /* restore saved int registers */
1515
1516         reg = INT_REG_CNT;
1517         for (i=0; i<es->code->savedintcount; ++i) {
1518                 while (nregdescint[--reg] != REG_SAV)
1519                         ;
1520                 es->intregs[reg] = *--basesp;
1521         }
1522
1523         /* restore saved flt registers */
1524
1525         /* XXX align? */
1526         reg = FLT_REG_CNT;
1527         for (i=0; i<es->code->savedfltcount; ++i) {
1528                 while (nregdescfloat[--reg] != REG_SAV)
1529                         ;
1530                 basesp -= STACK_SLOTS_PER_FLOAT;
1531                 es->fltregs[reg] = *(double*)basesp;
1532         }
1533
1534         /* adjust the stackpointer */
1535
1536         es->sp += SIZE_OF_STACKSLOT * es->code->stackframesize;
1537
1538 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1539         es->sp += SIZE_OF_STACKSLOT; /* skip return address */
1540 #endif
1541
1542         /* Set the new pc. Subtract one so we do not hit the replacement point */
1543         /* of the instruction following the call, if there is one.             */
1544
1545         es->pc = ra - 1;
1546
1547         /* find the new codeinfo */
1548
1549         pv = md_codegen_get_pv_from_pc(ra);
1550
1551         DOLOG( printf("PV = %p\n", (void*) pv); );
1552
1553         if (pv == NULL) /* XXX can this really happen? */
1554                 return NULL;
1555
1556         code = *(codeinfo **)(pv + CodeinfoPointer);
1557
1558         DOLOG( printf("CODE = %p\n", (void*) code); );
1559
1560         /* return NULL if we reached native code */
1561
1562         es->pv = pv;
1563         es->code = code;
1564
1565         /* in debugging mode clobber non-saved registers */
1566
1567 #if !defined(NDEBUG)
1568         /* for debugging */
1569         for (i=0; i<INT_REG_CNT; ++i)
1570                 if ((nregdescint[i] != REG_SAV)
1571 #if defined(REG_RA)
1572                                 && (i != REPLACE_REG_RA)
1573 #endif
1574                         )
1575                         es->intregs[i] = (ptrint) 0x33dead3333dead33ULL;
1576         for (i=0; i<FLT_REG_CNT; ++i)
1577                 if (nregdescfloat[i] != REG_SAV)
1578                         *(u8*)&(es->fltregs[i]) = 0x33dead3333dead33ULL;
1579 #endif /* !defined(NDEBUG) */
1580
1581         return (code) ? ra : NULL;
1582 }
1583
1584
1585 /* replace_patch_method_pointer ************************************************
1586
1587    Patch a method pointer (may be in code, data segment, vftbl, or interface
1588    table).
1589
1590    IN:
1591            mpp..............address of the method pointer to patch
1592            entrypoint.......the new entrypoint of the method
1593            kind.............kind of call to patch, used only for debugging
1594
1595 *******************************************************************************/
1596
1597 static void replace_patch_method_pointer(methodptr *mpp,
1598                                                                                  methodptr entrypoint,
1599                                                                                  const char *kind)
1600 {
1601 #if !defined(NDEBUG)
1602         codeinfo       *oldcode;
1603         codeinfo       *newcode;
1604 #endif
1605
1606         DOLOG( printf("patch method pointer from: %p to %p\n",
1607                                   (void*) *mpp, (void*)entrypoint); );
1608
1609 #if !defined(NDEBUG)
1610         oldcode = *(codeinfo **)((u1*)(*mpp) + CodeinfoPointer);
1611         newcode = *(codeinfo **)((u1*)(entrypoint) + CodeinfoPointer);
1612
1613         DOLOG_SHORT( printf("\tpatch %s %p ", kind, (void*) oldcode);
1614                                  method_println(oldcode->m);
1615                                  printf("\t      with      %p ", (void*) newcode);
1616                                  method_println(newcode->m); );
1617
1618         assert(oldcode->m == newcode->m);
1619 #endif
1620
1621         /* write the new entrypoint */
1622
1623         *mpp = (methodptr) entrypoint;
1624 }
1625
1626
1627 /* replace_patch_class *********************************************************
1628
1629    Patch a method in the given class.
1630
1631    IN:
1632            vftbl............vftbl of the class
1633            m................the method to patch
1634            oldentrypoint....the old entrypoint to replace
1635            entrypoint.......the new entrypoint
1636
1637 *******************************************************************************/
1638
1639 void replace_patch_class(vftbl_t *vftbl,
1640                                                  methodinfo *m,
1641                                                  u1 *oldentrypoint,
1642                                                  u1 *entrypoint)
1643 {
1644         s4                 i;
1645         methodptr         *mpp;
1646         methodptr         *mppend;
1647
1648         /* patch the vftbl of the class */
1649
1650         replace_patch_method_pointer(vftbl->table + m->vftblindex,
1651                                                                  entrypoint,
1652                                                                  "virtual  ");
1653
1654         /* patch the interface tables */
1655
1656         assert(oldentrypoint);
1657
1658         for (i=0; i < vftbl->interfacetablelength; ++i) {
1659                 mpp = vftbl->interfacetable[-i];
1660                 mppend = mpp + vftbl->interfacevftbllength[i];
1661                 for (; mpp != mppend; ++mpp)
1662                         if (*mpp == oldentrypoint) {
1663                                 replace_patch_method_pointer(mpp, entrypoint, "interface");
1664                         }
1665         }
1666 }
1667
1668
1669 /* replace_patch_class_hierarchy ***********************************************
1670
1671    Patch a method in all loaded classes.
1672
1673    IN:
1674            m................the method to patch
1675            oldentrypoint....the old entrypoint to replace
1676            entrypoint.......the new entrypoint
1677
1678 *******************************************************************************/
1679
1680 struct replace_patch_data_t {
1681         methodinfo *m;
1682         u1         *oldentrypoint;
1683         u1         *entrypoint;
1684 };
1685
1686 #define CODEINFO_OF_CODE(entrypoint) \
1687         (*(codeinfo **)((u1*)(entrypoint) + CodeinfoPointer))
1688
1689 #define METHOD_OF_CODE(entrypoint) \
1690         (CODEINFO_OF_CODE(entrypoint)->m)
1691
1692 void replace_patch_callback(classinfo *c, struct replace_patch_data_t *pd)
1693 {
1694         vftbl_t *vftbl = c->vftbl;
1695
1696         if (vftbl != NULL
1697                 && vftbl->vftbllength > pd->m->vftblindex
1698                 && vftbl->table[pd->m->vftblindex] != &asm_abstractmethoderror
1699                 && METHOD_OF_CODE(vftbl->table[pd->m->vftblindex]) == pd->m)
1700         {
1701                 replace_patch_class(c->vftbl, pd->m, pd->oldentrypoint, pd->entrypoint);
1702         }
1703 }
1704
1705 void replace_patch_class_hierarchy(methodinfo *m,
1706                                                                    u1 *oldentrypoint,
1707                                                                    u1 *entrypoint)
1708 {
1709         struct replace_patch_data_t pd;
1710
1711         pd.m = m;
1712         pd.oldentrypoint = oldentrypoint;
1713         pd.entrypoint = entrypoint;
1714
1715         DOLOG_SHORT( printf("patching class hierarchy: ");
1716                              method_println(m); );
1717
1718         classcache_foreach_loaded_class(
1719                         (classcache_foreach_functionptr_t) &replace_patch_callback,
1720                         (void*) &pd);
1721 }
1722
1723
1724 /* replace_patch_future_calls **************************************************
1725
1726    Analyse a call site and depending on the kind of call patch the call, the
1727    virtual function table, or the interface table.
1728
1729    IN:
1730            ra...............return address pointing after the call site
1731            callerframe......source frame of the caller
1732            calleeframe......source frame of the callee, must have been mapped
1733
1734 *******************************************************************************/
1735
1736 void replace_patch_future_calls(u1 *ra,
1737                                                                 sourceframe_t *callerframe,
1738                                                                 sourceframe_t *calleeframe)
1739 {
1740         u1                *patchpos;
1741         methodptr          entrypoint;
1742         methodptr          oldentrypoint;
1743         bool               atentry;
1744         stackframeinfo     sfi;
1745         codeinfo          *calleecode;
1746         methodinfo        *calleem;
1747         java_objectheader *obj;
1748         vftbl_t           *vftbl;
1749
1750         assert(ra);
1751         assert(callerframe->down == calleeframe);
1752
1753         /* get the new codeinfo and the method that shall be entered */
1754
1755         calleecode = calleeframe->tocode;
1756         assert(calleecode);
1757
1758         calleem = calleeframe->method;
1759         assert(calleem == calleecode->m);
1760
1761         entrypoint = (methodptr) calleecode->entrypoint;
1762
1763         /* check if we are at an method entry rplpoint at the innermost frame */
1764
1765         atentry = (calleeframe->down == NULL)
1766                         && !(calleem->flags & ACC_STATIC)
1767                         && (calleeframe->fromrp->id == 0); /* XXX */
1768
1769         /* get the position to patch, in case it was a statically bound call   */
1770
1771         sfi.pv = callerframe->fromcode->entrypoint;
1772         patchpos = md_get_method_patch_address(ra, &sfi, NULL);
1773
1774         if (patchpos == NULL) {
1775                 /* the call was dispatched dynamically */
1776
1777                 /* we can only patch such calls if we are at the entry point */
1778
1779                 if (!atentry)
1780                         return;
1781
1782                 assert((calleem->flags & ACC_STATIC) == 0);
1783
1784                 oldentrypoint = calleeframe->fromcode->entrypoint;
1785
1786                 /* we need to know the instance */
1787
1788                 if (!calleeframe->instance.a) {
1789                         DOLOG_SHORT( printf("WARNING: object instance unknown!\n"); );
1790                         replace_patch_class_hierarchy(calleem, oldentrypoint, entrypoint);
1791                         return;
1792                 }
1793
1794                 /* get the vftbl */
1795
1796                 obj = calleeframe->instance.a;
1797                 vftbl = obj->vftbl;
1798
1799                 assert(vftbl->class->vftbl == vftbl);
1800
1801                 DOLOG_SHORT( printf("\tclass: "); class_println(vftbl->class); );
1802
1803                 replace_patch_class(vftbl, calleem, oldentrypoint, entrypoint);
1804         }
1805         else {
1806                 /* the call was statically bound */
1807
1808                 replace_patch_method_pointer((methodptr *) patchpos, entrypoint, "static   ");
1809         }
1810 }
1811
1812
1813 /* replace_push_activation_record **********************************************
1814
1815    Push a stack frame onto the execution state.
1816
1817    *** This function imitates the effects of a call and the ***
1818    *** method prolog of the callee.                         ***
1819
1820    IN:
1821            es...............execution state
1822            rpcall...........the replacement point at the call site
1823            callerframe......source frame of the caller, or NULL for creating the
1824                             first frame
1825            calleeframe......source frame of the callee, must have been mapped
1826
1827    OUT:
1828        *es..............the execution state after pushing the stack frame
1829
1830 *******************************************************************************/
1831
1832 void replace_push_activation_record(executionstate_t *es,
1833                                                                         rplpoint *rpcall,
1834                                                                         sourceframe_t *callerframe,
1835                                                                         sourceframe_t *calleeframe)
1836 {
1837         s4           reg;
1838         s4           i;
1839         s4           count;
1840         stackslot_t *basesp;
1841         stackslot_t *sp;
1842         u1          *ra;
1843         codeinfo    *calleecode;
1844
1845         assert(es);
1846         assert(!rpcall || callerframe);
1847     assert(!rpcall || rpcall->type == RPLPOINT_TYPE_CALL);
1848         assert(!rpcall || rpcall == callerframe->torp);
1849         assert(calleeframe);
1850         assert(!callerframe || calleeframe == callerframe->down);
1851
1852         /* the compilation unit we are entering */
1853
1854         calleecode = calleeframe->tocode;
1855         assert(calleecode);
1856
1857         /* calculate the return address */
1858
1859         if (rpcall)
1860                 ra = rpcall->pc + rpcall->callsize;
1861         else
1862                 ra = es->pc + 1 /* XXX this is ugly */;
1863
1864         /* write the return address */
1865
1866 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1867         es->sp -= SIZE_OF_STACKSLOT;
1868
1869         *((stackslot_t *)es->sp) = (stackslot_t) ra;
1870 #endif /* REPLACE_RA_BETWEEN_FRAMES */
1871
1872 #if defined(REPLACE_REG_RA)
1873         es->intregs[REPLACE_REG_RA] = (ptrint) ra;
1874 #endif
1875
1876         /* we move into a new code unit */
1877
1878         es->code = calleecode;
1879
1880         /* set the new pc XXX not needed? */
1881
1882         es->pc = calleecode->entrypoint;
1883
1884         /* build the stackframe */
1885
1886         DOLOG( printf("building stackframe of %d words at %p\n",
1887                                   calleecode->stackframesize, (void*)es->sp); );
1888
1889         sp = (stackslot_t *) es->sp;
1890         basesp = sp;
1891
1892         sp -= calleecode->stackframesize;
1893         es->sp = (u1*) sp;
1894
1895         /* in debug mode, invalidate stack frame first */
1896
1897         /* XXX may not invalidate linkage area used by native code! */
1898 #if !defined(NDEBUG) && 0
1899         for (i=0; i<(basesp - sp); ++i) {
1900                 sp[i] = 0xdeaddeadU;
1901         }
1902 #endif
1903
1904         /* save the return address register */
1905
1906 #if defined(REPLACE_RA_TOP_OF_FRAME)
1907 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1908         if (!CODE_IS_LEAFMETHOD(calleecode))
1909 #endif
1910                 *--basesp = (ptrint) ra;
1911 #endif /* REPLACE_RA_TOP_OF_FRAME */
1912
1913 #if defined(REPLACE_RA_LINKAGE_AREA)
1914 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1915         if (!CODE_IS_LEAFMETHOD(calleecode))
1916 #endif
1917                 basesp[LA_LR_OFFSET / sizeof(stackslot_t)] = (ptrint) ra;
1918 #endif /* REPLACE_RA_LINKAGE_AREA */
1919
1920         /* save int registers */
1921
1922         reg = INT_REG_CNT;
1923         for (i=0; i<calleecode->savedintcount; ++i) {
1924                 while (nregdescint[--reg] != REG_SAV)
1925                         ;
1926                 *--basesp = es->intregs[reg];
1927
1928                 /* XXX may not clobber saved regs used by native code! */
1929 #if !defined(NDEBUG) && 0
1930                 es->intregs[reg] = (ptrint) 0x44dead4444dead44ULL;
1931 #endif
1932         }
1933
1934         /* save flt registers */
1935
1936         /* XXX align? */
1937         reg = FLT_REG_CNT;
1938         for (i=0; i<calleecode->savedfltcount; ++i) {
1939                 while (nregdescfloat[--reg] != REG_SAV)
1940                         ;
1941                 basesp -= STACK_SLOTS_PER_FLOAT;
1942                 *(double*)basesp = es->fltregs[reg];
1943
1944                 /* XXX may not clobber saved regs used by native code! */
1945 #if !defined(NDEBUG) && 0
1946                 *(u8*)&(es->fltregs[reg]) = 0x44dead4444dead44ULL;
1947 #endif
1948         }
1949
1950         /* write slots used for synchronization */
1951
1952         count = code_get_sync_slot_count(calleecode);
1953         assert(count == calleeframe->syncslotcount);
1954         for (i=0; i<count; ++i) {
1955                 sp[calleecode->memuse + i] = calleeframe->syncslots[i].p;
1956         }
1957
1958         /* set the PV */
1959
1960         es->pv = calleecode->entrypoint;
1961
1962         /* redirect future invocations */
1963
1964         if (callerframe && rpcall) {
1965 #if defined(REPLACE_PATCH_ALL)
1966                 if (rpcall->type == callerframe->fromrp->type)
1967 #else
1968                 if (rpcall == callerframe->fromrp)
1969 #endif
1970                         replace_patch_future_calls(ra, callerframe, calleeframe);
1971         }
1972 }
1973
1974
1975 /* replace_find_replacement_point **********************************************
1976
1977    Find the replacement point in the given code corresponding to the
1978    position given in the source frame.
1979
1980    IN:
1981            code.............the codeinfo in which to search the rplpoint
1982            frame............the source frame defining the position to look for
1983            parent...........parent replacement point to match
1984
1985    RETURN VALUE:
1986        the replacement point
1987
1988 *******************************************************************************/
1989
1990 rplpoint * replace_find_replacement_point(codeinfo *code,
1991                                                                                   sourceframe_t *frame,
1992                                                                                   rplpoint *parent)
1993 {
1994         methodinfo *m;
1995         rplpoint *rp;
1996         s4        i;
1997         s4        j;
1998         s4        stacki;
1999         rplalloc *ra;
2000
2001         assert(code);
2002         assert(frame);
2003
2004         DOLOG( printf("searching replacement point for:\n");
2005                    replace_source_frame_println(frame); );
2006
2007         m = frame->method;
2008
2009         DOLOG( printf("code = %p\n", (void*)code); );
2010
2011         rp = code->rplpoints;
2012         i = code->rplpointcount;
2013         while (i--) {
2014                 if (rp->id == frame->id && rp->method == frame->method
2015                                 && rp->parent == parent
2016                                 && replace_normalize_type_map[rp->type] == frame->type)
2017                 {
2018                         /* check if returnAddresses match */
2019                         /* XXX optimize: only do this if JSRs in method */
2020                         DOLOG( printf("checking match for:");
2021                                    replace_replacement_point_println(rp, 1); fflush(stdout); );
2022                         ra = rp->regalloc;
2023                         stacki = 0;
2024                         for (j = rp->regalloccount; j--; ++ra) {
2025                                 if (ra->type == TYPE_RET) {
2026                                         if (ra->index == RPLALLOC_STACK) {
2027                                                 assert(stacki < frame->javastackdepth);
2028                                                 if (frame->javastack[stacki].i != ra->regoff)
2029                                                         goto no_match;
2030                                                 stacki++;
2031                                         }
2032                                         else {
2033                                                 assert(ra->index >= 0 && ra->index < frame->javalocalcount);
2034                                                 if (frame->javalocals[ra->index].i != ra->regoff)
2035                                                         goto no_match;
2036                                         }
2037                                 }
2038                         }
2039
2040                         /* found */
2041                         return rp;
2042                 }
2043 no_match:
2044                 rp++;
2045         }
2046
2047 #if !defined(NDEBUG)
2048         printf("candidate replacement points were:\n");
2049         rp = code->rplpoints;
2050         i = code->rplpointcount;
2051         for (; i--; ++rp) {
2052                 replace_replacement_point_println(rp, 1);
2053         }
2054 #endif
2055
2056         vm_abort("no matching replacement point found");
2057         return NULL; /* NOT REACHED */
2058 }
2059
2060
2061 /* replace_find_replacement_point_for_pc ***************************************
2062
2063    Find the nearest replacement point at or before the given PC.
2064
2065    IN:
2066        code.............compilation unit the PC is in
2067            pc...............the machine code PC
2068
2069    RETURN VALUE:
2070        the replacement point found, or
2071            NULL if no replacement point was found
2072
2073 *******************************************************************************/
2074
2075 rplpoint *replace_find_replacement_point_for_pc(codeinfo *code, u1 *pc)
2076 {
2077         rplpoint *found;
2078         rplpoint *rp;
2079         s4        i;
2080
2081         DOLOG( printf("searching for rp in %p ", (void*)code);
2082                    method_println(code->m);
2083                    printf("PC = %p\n", (void*)pc); );
2084
2085         found = NULL;
2086
2087         rp = code->rplpoints;
2088         for (i=0; i<code->rplpointcount; ++i, ++rp) {
2089                 DOLOG( replace_replacement_point_println(rp, 2); );
2090                 if (rp->pc <= pc)
2091                         found = rp;
2092         }
2093
2094         assert(found->pc + found->callsize >= pc);
2095
2096         return found;
2097 }
2098
2099
2100 /* replace_pop_native_frame ****************************************************
2101
2102    Unroll a native frame in the execution state and create a source frame
2103    for it.
2104
2105    IN:
2106            es...............current execution state
2107            ss...............the current source state
2108            sfi..............stackframeinfo for the native frame
2109
2110    OUT:
2111        es...............execution state after unrolling the native frame
2112            ss...............gets the added native source frame
2113
2114 *******************************************************************************/
2115
2116 static void replace_pop_native_frame(executionstate_t *es,
2117                                                                          sourcestate_t *ss,
2118                                                                          stackframeinfo *sfi)
2119 {
2120         sourceframe_t *frame;
2121         codeinfo      *code;
2122         s4             i,j;
2123
2124         assert(sfi);
2125
2126         frame = replace_new_sourceframe(ss);
2127
2128         frame->sfi = sfi;
2129
2130         /* remember pc and size of native frame */
2131
2132         frame->nativepc = es->pc;
2133         frame->nativeframesize = (es->sp != 0) ? (sfi->sp - es->sp) : 0;
2134         assert(frame->nativeframesize >= 0);
2135
2136         /* remember values of saved registers */
2137
2138         j = 0;
2139         for (i=0; i<INT_REG_CNT; ++i) {
2140                 if (nregdescint[i] == REG_SAV)
2141                         frame->nativesavint[j++] = es->intregs[i];
2142         }
2143
2144         j = 0;
2145         for (i=0; i<FLT_REG_CNT; ++i) {
2146                 if (nregdescfloat[i] == REG_SAV)
2147                         frame->nativesavflt[j++] = es->fltregs[i];
2148         }
2149
2150         /* restore saved registers */
2151
2152 #if defined(ENABLE_GC_CACAO)
2153         j = 0;
2154         for (i=0; i<INT_REG_CNT; ++i) {
2155                 if (nregdescint[i] == REG_SAV)
2156                         es->intregs[i] = sfi->intregs[j++];
2157         }
2158 #else
2159         /* XXX we don't have them, yet, in the sfi, so clear them */
2160
2161         for (i=0; i<INT_REG_CNT; ++i) {
2162                 if (nregdescint[i] == REG_SAV)
2163                         es->intregs[i] = 0;
2164         }
2165 #endif
2166
2167         /* XXX we don't have float registers in the sfi, so clear them */
2168
2169         for (i=0; i<FLT_REG_CNT; ++i) {
2170                 if (nregdescfloat[i] == REG_SAV)
2171                         es->fltregs[i] = 0.0;
2172         }
2173
2174         /* restore pv, pc, and sp */
2175
2176         if (sfi->pv == NULL) {
2177                 /* frame of a native function call */
2178                 es->pv = md_codegen_get_pv_from_pc(sfi->ra);
2179         }
2180         else {
2181                 es->pv = sfi->pv;
2182         }
2183         es->pc = ((sfi->xpc) ? sfi->xpc : sfi->ra) - 1;
2184         es->sp = sfi->sp;
2185
2186         /* find the new codeinfo */
2187
2188         DOLOG( printf("PV = %p\n", (void*) es->pv); );
2189
2190         assert(es->pv != NULL);
2191
2192         code = *(codeinfo **)(es->pv + CodeinfoPointer);
2193
2194         DOLOG( printf("CODE = %p\n", (void*) code); );
2195
2196         es->code = code;
2197 }
2198
2199
2200 /* replace_push_native_frame ***************************************************
2201
2202    Rebuild a native frame onto the execution state and remove its source frame.
2203
2204    Note: The native frame is "rebuild" by setting fields like PC and stack
2205          pointer in the execution state accordingly. Values in the
2206                  stackframeinfo may be modified, but the actual stack frame of the
2207                  native code is not touched.
2208
2209    IN:
2210            es...............current execution state
2211            ss...............the current source state
2212
2213    OUT:
2214        es...............execution state after re-rolling the native frame
2215            ss...............the native source frame is removed
2216
2217 *******************************************************************************/
2218
2219 static void replace_push_native_frame(executionstate_t *es, sourcestate_t *ss)
2220 {
2221         sourceframe_t *frame;
2222         s4             i,j;
2223
2224         assert(es);
2225         assert(ss);
2226
2227         DOLOG( printf("pushing native frame\n"); );
2228
2229         /* remove the frame from the source state */
2230
2231         frame = ss->frames;
2232         assert(frame);
2233         assert(REPLACE_IS_NATIVE_FRAME(frame));
2234
2235         ss->frames = frame->down;
2236
2237         /* assert that the native frame has not moved */
2238
2239         assert(es->sp == frame->sfi->sp);
2240
2241         /* update saved registers in the stackframeinfo */
2242
2243 #if defined(ENABLE_GC_CACAO)
2244         j = 0;
2245         for (i=0; i<INT_REG_CNT; ++i) {
2246                 if (nregdescint[i] == REG_SAV)
2247                         frame->sfi->intregs[j++] = es->intregs[i];
2248         }
2249
2250         /* XXX leave float registers untouched here */
2251 #endif
2252
2253         /* restore saved registers */
2254
2255         j = 0;
2256         for (i=0; i<INT_REG_CNT; ++i) {
2257                 if (nregdescint[i] == REG_SAV)
2258                         es->intregs[i] = frame->nativesavint[j++];
2259         }
2260
2261         j = 0;
2262         for (i=0; i<FLT_REG_CNT; ++i) {
2263                 if (nregdescfloat[i] == REG_SAV)
2264                         es->fltregs[i] = frame->nativesavflt[j++];
2265         }
2266
2267         /* skip the native frame on the machine stack */
2268
2269         es->sp -= frame->nativeframesize;
2270
2271         /* set the pc the next frame must return to */
2272
2273         es->pc = frame->nativepc;
2274 }
2275
2276
2277 /* replace_recover_source_state ************************************************
2278
2279    Recover the source state from the given replacement point and execution
2280    state.
2281
2282    IN:
2283        rp...............replacement point that has been reached, if any
2284            sfi..............stackframeinfo, if called from native code
2285            es...............execution state at the replacement point rp
2286
2287    RETURN VALUE:
2288        the source state
2289
2290 *******************************************************************************/
2291
2292 sourcestate_t *replace_recover_source_state(rplpoint *rp,
2293                                                                                         stackframeinfo *sfi,
2294                                                                                     executionstate_t *es)
2295 {
2296         sourcestate_t *ss;
2297         u1            *ra;
2298         bool           locked;
2299 #if defined(REPLACE_STATISTICS)
2300         s4             depth;
2301 #endif
2302
2303         /* create the source frame structure in dump memory */
2304
2305         ss = DNEW(sourcestate_t);
2306         ss->frames = NULL;
2307
2308         /* each iteration of the loop recovers one source frame */
2309
2310         depth = 0;
2311         locked = false;
2312
2313         while (rp || sfi) {
2314
2315                 DOLOG( replace_executionstate_println(es); );
2316
2317                 /* if we are not at a replacement point, it is a native frame */
2318
2319                 if (rp == NULL) {
2320                         DOLOG( printf("native frame: sfi: "); replace_stackframeinfo_println(sfi); );
2321
2322                         locked = true;
2323                         replace_pop_native_frame(es, ss, sfi);
2324                         sfi = sfi->prev;
2325
2326                         if (es->code == NULL)
2327                                 continue;
2328
2329                         goto after_machine_frame;
2330                 }
2331
2332                 /* read the values for this source frame from the execution state */
2333
2334                 DOLOG( printf("recovering source state for%s:\n",
2335                                         (ss->frames == NULL) ? " TOPFRAME" : "");
2336                            replace_replacement_point_println(rp, 1); );
2337
2338                 replace_read_executionstate(rp, es, ss, ss->frames == NULL);
2339
2340 #if defined(ENABLE_VMLOG)
2341                 vmlog_cacao_unrol_method(ss->frames->method);
2342 #endif
2343
2344 #if defined(REPLACE_STATISTICS)
2345                 REPLACE_COUNT(stat_frames);
2346                 depth++;
2347                 replace_statistics_source_frame(ss->frames);
2348 #endif
2349
2350                 /* in locked areas (below native frames), identity map the frame */
2351
2352                 if (locked) {
2353                         ss->frames->torp = ss->frames->fromrp;
2354                         ss->frames->tocode = ss->frames->fromcode;
2355                 }
2356
2357                 /* unroll to the next (outer) frame */
2358
2359                 if (rp->parent) {
2360                         /* this frame is in inlined code */
2361
2362                         DOLOG( printf("INLINED!\n"); );
2363
2364                         rp = rp->parent;
2365
2366                         assert(rp->type == RPLPOINT_TYPE_INLINE);
2367                         REPLACE_COUNT(stat_unroll_inline);
2368                 }
2369                 else {
2370                         /* this frame had been called at machine-level. pop it. */
2371
2372                         DOLOG( printf("UNWIND\n"); );
2373
2374                         ra = replace_pop_activation_record(es, ss->frames);
2375                         if (ra == NULL) {
2376                                 DOLOG( printf("REACHED NATIVE CODE\n"); );
2377
2378                                 rp = NULL;
2379
2380 #if !defined(ENABLE_GC_CACAO)
2381                                 break; /* XXX remove to activate native frames */
2382 #endif
2383                                 continue;
2384                         }
2385
2386                         /* find the replacement point at the call site */
2387
2388 after_machine_frame:
2389                         rp = replace_find_replacement_point_for_pc(es->code, es->pc);
2390
2391                         if (rp == NULL)
2392                                 vm_abort("could not find replacement point while unrolling call");
2393
2394                         DOLOG( printf("found replacement point.\n");
2395                                         replace_replacement_point_println(rp, 1); );
2396
2397                         assert(rp->type == RPLPOINT_TYPE_CALL);
2398                         REPLACE_COUNT(stat_unroll_call);
2399                 }
2400         } /* end loop over source frames */
2401
2402         REPLACE_COUNT_DIST(stat_dist_frames, depth);
2403
2404         return ss;
2405 }
2406
2407
2408 /* replace_map_source_state ****************************************************
2409
2410    Map each source frame in the given source state to a target replacement
2411    point and compilation unit. If no valid code is available for a source
2412    frame, it is (re)compiled.
2413
2414    IN:
2415        ss...............the source state
2416
2417    OUT:
2418        ss...............the source state, modified: The `torp` and `tocode`
2419                             fields of each source frame are set.
2420
2421    RETURN VALUE:
2422        true.............everything went ok
2423            false............an exception has been thrown
2424
2425 *******************************************************************************/
2426
2427 static bool replace_map_source_state(sourcestate_t *ss)
2428 {
2429         sourceframe_t *frame;
2430         codeinfo      *code;
2431         rplpoint      *rp;
2432         rplpoint      *parent; /* parent of inlined rplpoint */
2433 #if defined(REPLACE_STATISTICS)
2434         codeinfo      *oldcode;
2435 #endif
2436
2437         parent = NULL;
2438         code = NULL;
2439
2440         /* iterate over the source frames from outermost to innermost */
2441
2442         for (frame = ss->frames; frame != NULL; frame = frame->down) {
2443
2444                 /* XXX skip native frames */
2445
2446                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
2447                         parent = NULL;
2448                         continue;
2449                 }
2450
2451                 /* map frames which are not already mapped */
2452
2453                 if (frame->tocode) {
2454                         code = frame->tocode;
2455                         rp = frame->torp;
2456                         assert(rp);
2457                 }
2458                 else {
2459                         assert(frame->torp == NULL);
2460
2461                         if (parent == NULL) {
2462                                 /* find code for this frame */
2463
2464 #if defined(REPLACE_STATISTICS)
2465                                 oldcode = frame->method->code;
2466 #endif
2467                                 /* request optimization of hot methods and their callers */
2468
2469                                 if (frame->method->hitcountdown < 0
2470                                         || (frame->down && frame->down->method->hitcountdown < 0))
2471                                         jit_request_optimization(frame->method);
2472
2473                                 code = jit_get_current_code(frame->method);
2474
2475                                 if (code == NULL)
2476                                         return false; /* exception */
2477
2478                                 REPLACE_COUNT_IF(stat_recompile, code != oldcode);
2479                         }
2480
2481                         assert(code);
2482
2483                         /* map this frame */
2484
2485                         rp = replace_find_replacement_point(code, frame, parent);
2486
2487                         frame->tocode = code;
2488                         frame->torp = rp;
2489                 }
2490
2491                 if (rp->type == RPLPOINT_TYPE_CALL) {
2492                         parent = NULL;
2493                 }
2494                 else {
2495                         /* inlining */
2496                         parent = rp;
2497                 }
2498         }
2499
2500         return true;
2501 }
2502
2503
2504 /* replace_map_source_state_identity *******************************************
2505
2506    Map each source frame in the given source state to the same replacement
2507    point and compilation unit it was derived from. This is mainly used for
2508    garbage collection.
2509
2510    IN:
2511        ss...............the source state
2512
2513    OUT:
2514        ss...............the source state, modified: The `torp` and `tocode`
2515                             fields of each source frame are set.
2516
2517 *******************************************************************************/
2518
2519 #if defined(ENABLE_GC_CACAO)
2520 static void replace_map_source_state_identity(sourcestate_t *ss)
2521 {
2522         sourceframe_t *frame;
2523
2524         /* iterate over the source frames from outermost to innermost */
2525
2526         for (frame = ss->frames; frame != NULL; frame = frame->down) {
2527
2528                 /* skip native frames */
2529
2530                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
2531                         continue;
2532                 }
2533
2534                 /* map frames using the identity mapping */
2535
2536                 if (frame->tocode) {
2537                         assert(frame->tocode == frame->fromcode);
2538                         assert(frame->torp   == frame->fromrp);
2539                 } else {
2540                         assert(frame->tocode == NULL);
2541                         assert(frame->torp   == NULL);
2542                         frame->tocode = frame->fromcode;
2543                         frame->torp   = frame->fromrp;
2544                 }
2545         }
2546 }
2547 #endif
2548
2549
2550 /* replace_build_execution_state_intern ****************************************
2551
2552    Build an execution state for the given (mapped) source state.
2553
2554    !!! CAUTION: This function rewrites the machine stack !!!
2555
2556    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2557
2558    IN:
2559        ss...............the source state. Must have been mapped by
2560                                                 replace_map_source_state before.
2561            es...............the base execution state on which to build
2562
2563    OUT:
2564        *es..............the new execution state
2565
2566 *******************************************************************************/
2567
2568 void replace_build_execution_state_intern(sourcestate_t *ss,
2569                                                                                                  executionstate_t *es)
2570 {
2571         rplpoint      *rp;
2572         sourceframe_t *prevframe;
2573         rplpoint      *parent;
2574
2575         parent = NULL;
2576         prevframe = NULL;
2577         rp = NULL;
2578
2579         while (ss->frames) {
2580
2581                 if (REPLACE_IS_NATIVE_FRAME(ss->frames)) {
2582                         prevframe = ss->frames;
2583                         replace_push_native_frame(es, ss);
2584                         parent = NULL;
2585                         rp = NULL;
2586                         continue;
2587                 }
2588
2589                 if (parent == NULL) {
2590                         /* create a machine-level stack frame */
2591
2592                         DOLOG( printf("pushing activation record for:\n");
2593                                    if (rp) replace_replacement_point_println(rp, 1);
2594                                    else printf("\tfirst frame\n"); );
2595
2596                         replace_push_activation_record(es, rp, prevframe, ss->frames);
2597
2598                         DOLOG( replace_executionstate_println(es); );
2599                 }
2600
2601                 rp = ss->frames->torp;
2602                 assert(rp);
2603
2604                 DOLOG( printf("creating execution state for%s:\n",
2605                                 (ss->frames->down == NULL) ? " TOPFRAME" : "");
2606                            replace_replacement_point_println(ss->frames->fromrp, 1);
2607                            replace_replacement_point_println(rp, 1); );
2608
2609                 es->code = ss->frames->tocode;
2610                 prevframe = ss->frames;
2611
2612 #if defined(ENABLE_VMLOG)
2613                 vmlog_cacao_rerol_method(ss->frames->method);
2614 #endif
2615
2616                 replace_write_executionstate(rp, es, ss, ss->frames->down == NULL);
2617
2618                 DOLOG( replace_executionstate_println(es); );
2619
2620                 if (rp->type == RPLPOINT_TYPE_CALL) {
2621                         parent = NULL;
2622                 }
2623                 else {
2624                         /* inlining */
2625                         parent = rp;
2626                 }
2627         }
2628 }
2629
2630
2631 /* replace_build_execution_state ***********************************************
2632
2633    This function contains the final phase of replacement. It builds the new
2634    execution state, releases dump memory, and returns to the calling
2635    assembler function which finishes replacement.
2636
2637    NOTE: This function is called from asm_replacement_in, with the stack
2638          pointer at the start of the safe stack area.
2639
2640    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2641
2642    CAUTION: This function and its children must not use a lot of stack!
2643             There are only REPLACE_SAFESTACK_SIZE bytes of C stack
2644                         available.
2645
2646    IN:
2647        st...............the safestack contained the necessary data
2648
2649 *******************************************************************************/
2650
2651 void replace_build_execution_state(replace_safestack_t *st)
2652 {
2653         replace_build_execution_state_intern(st->ss, &(st->es));
2654
2655         DOLOG( replace_executionstate_println(&(st->es)); );
2656
2657         /* release dump area */
2658
2659         dump_release(st->dumpsize);
2660
2661         /* new code is entered after returning */
2662
2663         DOLOG( printf("JUMPING IN!\n"); fflush(stdout); );
2664 }
2665
2666
2667 /* replace_alloc_safestack *****************************************************
2668
2669    Allocate a safe stack area to use during the final phase of replacement.
2670    The returned area is not initialized. This must be done by the caller.
2671
2672    RETURN VALUE:
2673        a newly allocated replace_safestack_t *
2674
2675 *******************************************************************************/
2676
2677 static replace_safestack_t *replace_alloc_safestack()
2678 {
2679         u1 *mem;
2680         replace_safestack_t *st;
2681
2682         mem = MNEW(u1, sizeof(replace_safestack_t) + REPLACE_STACK_ALIGNMENT - 1);
2683
2684         st = (replace_safestack_t *) ((ptrint)(mem + REPLACE_STACK_ALIGNMENT - 1)
2685                                                                                 & ~(REPLACE_STACK_ALIGNMENT - 1));
2686
2687 #if !defined(NDEBUG)
2688         memset(st, 0xa5, sizeof(replace_safestack_t));
2689 #endif
2690
2691         st->mem = mem;
2692
2693         return st;
2694 }
2695
2696
2697 /* replace_free_safestack ******************************************************
2698
2699    Free the given safestack structure, making a copy of the contained
2700    execution state before freeing it.
2701
2702    NOTE: This function is called from asm_replacement_in.
2703
2704    IN:
2705        st...............the safestack to free
2706            tmpes............where to copy the execution state to
2707
2708    OUT:
2709            *tmpes...........receives a copy of st->es
2710
2711 *******************************************************************************/
2712
2713 void replace_free_safestack(replace_safestack_t *st, executionstate_t *tmpes)
2714 {
2715         u1 *mem;
2716
2717         /* copy the executionstate_t to the temporary location */
2718
2719         *tmpes = st->es;
2720
2721         /* get the memory address to free */
2722
2723         mem = st->mem;
2724
2725         /* destroy memory (in debug mode) */
2726
2727 #if !defined(NDEBUG)
2728         memset(st, 0xa5, sizeof(replace_safestack_t));
2729 #endif
2730
2731         /* free the safe stack struct */
2732
2733         MFREE(mem, u1, sizeof(replace_safestack_t) + REPLACE_STACK_ALIGNMENT - 1);
2734 }
2735
2736
2737 /* replace_me ******************************************************************
2738
2739    This function is called by asm_replacement_out when a thread reaches
2740    a replacement point. `replace_me` must map the execution state to the
2741    target replacement point and let execution continue there.
2742
2743    This function never returns!
2744
2745    IN:
2746        rp...............replacement point that has been reached
2747            es...............execution state read by asm_replacement_out
2748
2749 *******************************************************************************/
2750
2751 void replace_me(rplpoint *rp, executionstate_t *es)
2752 {
2753         stackframeinfo      *sfi;
2754         sourcestate_t       *ss;
2755         sourceframe_t       *frame;
2756         s4                   dumpsize;
2757         rplpoint            *origrp;
2758         replace_safestack_t *safestack;
2759 #if defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)
2760         threadobject        *thread;
2761 #endif
2762
2763         origrp = rp;
2764         es->code = code_find_codeinfo_for_pc(rp->pc);
2765
2766         DOLOG_SHORT( printf("REPLACING(%d %p): (id %d %p) ",
2767                                  stat_replacements, (void*)THREADOBJECT,
2768                                  rp->id, (void*)rp);
2769                                  method_println(es->code->m); );
2770
2771         DOLOG( replace_replacement_point_println(rp, 1);
2772                    replace_executionstate_println(es); );
2773
2774         REPLACE_COUNT(stat_replacements);
2775
2776         /* mark start of dump memory area */
2777
2778         dumpsize = dump_size();
2779
2780         /* get the stackframeinfo for the current thread */
2781
2782         sfi = STACKFRAMEINFO;
2783
2784         /* recover source state */
2785
2786         ss = replace_recover_source_state(rp, sfi, es);
2787
2788 #if defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)
2789         /* if there is a collection pending, we assume the replacement point should
2790            suspend this thread */
2791
2792         if (gc_pending) {
2793
2794                 thread = THREADOBJECT;
2795
2796                 DOLOG_SHORT( printf("REPLACEMENT: Suspending thread for GC now!\n"); );
2797
2798                 /* map the sourcestate using the identity mapping */
2799                 replace_map_source_state_identity(ss);
2800
2801                 /* since we enter the same method again, we turn off rps now */
2802                 /* XXX michi: can we really do this? what if the rp was active before
2803                    we activated it for the gc? */
2804                 frame = ss->frames;
2805                 while (frame->down)
2806                         frame = frame->down;
2807                 replace_deactivate_replacement_points(frame->tocode);
2808
2809                 /* remember executionstate and sourcestate for this thread */
2810                 GC_EXECUTIONSTATE = es;
2811                 GC_SOURCESTATE    = ss;
2812
2813                 /* really suspend this thread now (PC = 0) */
2814                 threads_suspend_ack(NULL, NULL);
2815
2816                 DOLOG_SHORT( printf("REPLACEMENT: Resuming thread after GC now!\n"); );
2817
2818         } else {
2819 #endif /*defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)*/
2820
2821         /* map the source state */
2822
2823         if (!replace_map_source_state(ss))
2824                 vm_abort("exception during method replacement");
2825
2826         DOLOG( replace_sourcestate_println(ss); );
2827
2828         DOLOG_SHORT( replace_sourcestate_println_short(ss); );
2829
2830         /* avoid infinite loops by self-replacement */
2831
2832         frame = ss->frames;
2833         while (frame->down)
2834                 frame = frame->down;
2835
2836         if (frame->torp == origrp) {
2837                 DOLOG_SHORT(
2838                         printf("WARNING: identity replacement, turning off rps to avoid infinite loop\n");
2839                 );
2840                 replace_deactivate_replacement_points(frame->tocode);
2841         }
2842
2843 #if defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)
2844         }
2845 #endif
2846
2847         /* write execution state of new code */
2848
2849         DOLOG( replace_executionstate_println(es); );
2850
2851         /* allocate a safe stack area and copy all needed data there */
2852
2853         safestack = replace_alloc_safestack();
2854
2855         safestack->es = *es;
2856         safestack->ss = ss;
2857         safestack->dumpsize = dumpsize;
2858
2859         /* call the assembler code for the last phase of replacement */
2860
2861 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__)) && defined(ENABLE_JIT)
2862         asm_replacement_in(&(safestack->es), safestack);
2863 #endif
2864
2865         abort(); /* NOT REACHED */
2866 }
2867
2868
2869 /******************************************************************************/
2870 /* NOTE: Stuff specific to the exact GC is below.                             */
2871 /******************************************************************************/
2872
2873 #if defined(ENABLE_GC_CACAO)
2874 void replace_gc_from_native(threadobject *thread, u1 *pc, u1 *sp)
2875 {
2876         stackframeinfo   *sfi;
2877         executionstate_t *es;
2878         sourcestate_t    *ss;
2879
2880         /* get the stackframeinfo of this thread */
2881         assert(thread == THREADOBJECT);
2882         sfi = STACKFRAMEINFO;
2883
2884         /* create the execution state */
2885         es = DNEW(executionstate_t);
2886         es->pc = pc;
2887         es->sp = sp;
2888         es->pv = 0;      /* since we are in a native, PV is invalid! */
2889         es->code = NULL; /* since we are in a native, we do not have a codeinfo */
2890
2891         /* we assume we are in a native (no replacement point)! */
2892         ss = replace_recover_source_state(NULL, sfi, es);
2893
2894         /* map the sourcestate using the identity mapping */
2895         replace_map_source_state_identity(ss);
2896
2897         /* remember executionstate and sourcestate for this thread */
2898         GC_EXECUTIONSTATE = es;
2899         GC_SOURCESTATE    = ss;
2900 }
2901 #endif
2902
2903
2904 /******************************************************************************/
2905 /* NOTE: No important code below.                                             */
2906 /******************************************************************************/
2907
2908
2909 /* statistics *****************************************************************/
2910
2911 #if defined(REPLACE_STATISTICS)
2912 static void print_freq(FILE *file,int *array,int limit)
2913 {
2914         int i;
2915         int sum = 0;
2916         int cum = 0;
2917         for (i=0; i<limit; ++i)
2918                 sum += array[i];
2919         sum += array[limit];
2920         for (i=0; i<limit; ++i) {
2921                 cum += array[i];
2922                 fprintf(file,"      %3d: %8d (cum %3d%%)\n",
2923                                 i, array[i], (sum) ? ((100*cum)/sum) : 0);
2924         }
2925         fprintf(file,"    >=%3d: %8d\n",limit,array[limit]);
2926 }
2927 #endif /* defined(REPLACE_STATISTICS) */
2928
2929
2930 #if defined(REPLACE_STATISTICS)
2931
2932 #define REPLACE_PRINT_DIST(name, array)                              \
2933     printf("    " name " distribution:\n");                          \
2934     print_freq(stdout, (array), sizeof(array)/sizeof(int) - 1);
2935
2936 void replace_print_statistics(void)
2937 {
2938         printf("replacement statistics:\n");
2939         printf("    # of replacements:   %d\n", stat_replacements);
2940         printf("    # of frames:         %d\n", stat_frames);
2941         printf("    # of recompilations: %d\n", stat_recompile);
2942         printf("    patched static calls:%d\n", stat_staticpatch);
2943         printf("    unrolled inlines:    %d\n", stat_unroll_inline);
2944         printf("    unrolled calls:      %d\n", stat_unroll_call);
2945         REPLACE_PRINT_DIST("frame depth", stat_dist_frames);
2946         REPLACE_PRINT_DIST("locals per frame", stat_dist_locals);
2947         REPLACE_PRINT_DIST("ADR locals per frame", stat_dist_locals_adr);
2948         REPLACE_PRINT_DIST("primitive locals per frame", stat_dist_locals_prim);
2949         REPLACE_PRINT_DIST("RET locals per frame", stat_dist_locals_ret);
2950         REPLACE_PRINT_DIST("void locals per frame", stat_dist_locals_void);
2951         REPLACE_PRINT_DIST("stack slots per frame", stat_dist_stack);
2952         REPLACE_PRINT_DIST("ADR stack slots per frame", stat_dist_stack_adr);
2953         REPLACE_PRINT_DIST("primitive stack slots per frame", stat_dist_stack_prim);
2954         REPLACE_PRINT_DIST("RET stack slots per frame", stat_dist_stack_ret);
2955         printf("\n");
2956         printf("    # of methods:            %d\n", stat_methods);
2957         printf("    # of replacement points: %d\n", stat_rploints);
2958         printf("    # of regallocs:          %d\n", stat_regallocs);
2959         printf("        per rplpoint:        %f\n", (double)stat_regallocs / stat_rploints);
2960         printf("        per method:          %f\n", (double)stat_regallocs / stat_methods);
2961         REPLACE_PRINT_DIST("replacement points per method", stat_dist_method_rplpoints);
2962         printf("\n");
2963
2964 }
2965 #endif /* defined(REPLACE_STATISTICS) */
2966
2967
2968 #if defined(REPLACE_STATISTICS)
2969 static void replace_statistics_source_frame(sourceframe_t *frame)
2970 {
2971         int adr = 0;
2972         int ret = 0;
2973         int prim = 0;
2974         int vd = 0;
2975         int n = 0;
2976         int i;
2977
2978         for (i=0; i<frame->javalocalcount; ++i) {
2979                 switch (frame->javalocaltype[i]) {
2980                         case TYPE_ADR: adr++; break;
2981                         case TYPE_RET: ret++; break;
2982                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
2983                         case TYPE_VOID: vd++; break;
2984                         default: assert(0);
2985                 }
2986                 n++;
2987         }
2988         REPLACE_COUNT_DIST(stat_dist_locals, n);
2989         REPLACE_COUNT_DIST(stat_dist_locals_adr, adr);
2990         REPLACE_COUNT_DIST(stat_dist_locals_void, vd);
2991         REPLACE_COUNT_DIST(stat_dist_locals_ret, ret);
2992         REPLACE_COUNT_DIST(stat_dist_locals_prim, prim);
2993         adr = ret = prim = n = 0;
2994         for (i=0; i<frame->javastackdepth; ++i) {
2995                 switch (frame->javastacktype[i]) {
2996                         case TYPE_ADR: adr++; break;
2997                         case TYPE_RET: ret++; break;
2998                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
2999                 }
3000                 n++;
3001         }
3002         REPLACE_COUNT_DIST(stat_dist_stack, n);
3003         REPLACE_COUNT_DIST(stat_dist_stack_adr, adr);
3004         REPLACE_COUNT_DIST(stat_dist_stack_ret, ret);
3005         REPLACE_COUNT_DIST(stat_dist_stack_prim, prim);
3006 }
3007 #endif /* defined(REPLACE_STATISTICS) */
3008
3009
3010 /* debugging helpers **********************************************************/
3011
3012 /* replace_replacement_point_println *******************************************
3013
3014    Print replacement point info.
3015
3016    IN:
3017        rp...............the replacement point to print
3018
3019 *******************************************************************************/
3020
3021 #if !defined(NDEBUG)
3022
3023 #define TYPECHAR(t)  (((t) >= 0 && (t) <= TYPE_RET) ? show_jit_type_letters[t] : '?')
3024
3025 static char *replace_type_str[] = {
3026         "STD",
3027         "EXH",
3028         "SBR",
3029         "CALL",
3030         "INLINE",
3031         "RETURN",
3032         "BODY"
3033 };
3034
3035 void replace_replacement_point_println(rplpoint *rp, int depth)
3036 {
3037         int j;
3038         int index;
3039
3040         if (!rp) {
3041                 printf("(rplpoint *)NULL\n");
3042                 return;
3043         }
3044
3045         for (j=0; j<depth; ++j)
3046                 putchar('\t');
3047
3048         printf("rplpoint (id %d) %p pc:%p+%d type:%s",
3049                         rp->id, (void*)rp,rp->pc,rp->callsize,
3050                         replace_type_str[rp->type]);
3051         if (rp->flags & RPLPOINT_FLAG_NOTRAP)
3052                 printf(" NOTRAP");
3053         if (rp->flags & RPLPOINT_FLAG_COUNTDOWN)
3054                 printf(" COUNTDOWN");
3055         if (rp->flags & RPLPOINT_FLAG_ACTIVE)
3056                 printf(" ACTIVE");
3057         printf(" parent:%p\n", (void*)rp->parent);
3058         for (j=0; j<depth; ++j)
3059                 putchar('\t');
3060         printf("ra:%d = [",     rp->regalloccount);
3061
3062         for (j=0; j<rp->regalloccount; ++j) {
3063                 if (j)
3064                         putchar(' ');
3065                 index = rp->regalloc[j].index;
3066                 switch (index) {
3067                         case RPLALLOC_STACK: printf("S"); break;
3068                         case RPLALLOC_PARAM: printf("P"); break;
3069                         case RPLALLOC_SYNC : printf("Y"); break;
3070                         default: printf("%d", index);
3071                 }
3072                 printf(":%1c:", TYPECHAR(rp->regalloc[j].type));
3073                 if (rp->regalloc[j].type == TYPE_RET) {
3074                         printf("ret(L%03d)", rp->regalloc[j].regoff);
3075                 }
3076                 else {
3077                         show_allocation(rp->regalloc[j].type, rp->regalloc[j].flags, rp->regalloc[j].regoff);
3078                 }
3079         }
3080
3081         printf("]\n");
3082         for (j=0; j<depth; ++j)
3083                 putchar('\t');
3084         printf("method: ");
3085         method_print(rp->method);
3086
3087         printf("\n");
3088 }
3089 #endif /* !defined(NDEBUG) */
3090
3091
3092 /* replace_show_replacement_points *********************************************
3093
3094    Print replacement point info.
3095
3096    IN:
3097        code.............codeinfo whose replacement points should be printed.
3098
3099 *******************************************************************************/
3100
3101 #if !defined(NDEBUG)
3102 void replace_show_replacement_points(codeinfo *code)
3103 {
3104         int i;
3105         int depth;
3106         rplpoint *rp;
3107         rplpoint *parent;
3108
3109         if (!code) {
3110                 printf("(codeinfo *)NULL\n");
3111                 return;
3112         }
3113
3114         printf("\treplacement points: %d\n",code->rplpointcount);
3115
3116         printf("\ttotal allocations : %d\n",code->regalloccount);
3117         printf("\tsaved int regs    : %d\n",code->savedintcount);
3118         printf("\tsaved flt regs    : %d\n",code->savedfltcount);
3119         printf("\tmemuse            : %d\n",code->memuse);
3120
3121         printf("\n");
3122
3123         for (i=0; i<code->rplpointcount; ++i) {
3124                 rp = code->rplpoints + i;
3125
3126                 depth = 1;
3127                 parent = rp->parent;
3128                 while (parent) {
3129                         depth++;
3130                         parent = parent->parent;
3131                 }
3132                 replace_replacement_point_println(rp, depth);
3133         }
3134 }
3135 #endif
3136
3137
3138 /* replace_executionstate_println **********************************************
3139
3140    Print execution state
3141
3142    IN:
3143        es...............the execution state to print
3144
3145 *******************************************************************************/
3146
3147 #if !defined(NDEBUG)
3148 void replace_executionstate_println(executionstate_t *es)
3149 {
3150         int i;
3151         int slots;
3152         stackslot_t *sp;
3153         int extraslots;
3154
3155         if (!es) {
3156                 printf("(executionstate_t *)NULL\n");
3157                 return;
3158         }
3159
3160         printf("executionstate_t:\n");
3161         printf("\tpc = %p",(void*)es->pc);
3162         printf("  sp = %p",(void*)es->sp);
3163         printf("  pv = %p\n",(void*)es->pv);
3164 #if defined(ENABLE_DISASSEMBLER)
3165         for (i=0; i<INT_REG_CNT; ++i) {
3166                 if (i%4 == 0)
3167                         printf("\t");
3168                 else
3169                         printf(" ");
3170 #if SIZEOF_VOID_P == 8
3171                 printf("%-3s = %016llx",abi_registers_integer_name[i],(unsigned long long)es->intregs[i]);
3172 #else
3173                 printf("%-3s = %08lx",abi_registers_integer_name[i],(unsigned long)es->intregs[i]);
3174 #endif
3175                 if (i%4 == 3)
3176                         printf("\n");
3177         }
3178         for (i=0; i<FLT_REG_CNT; ++i) {
3179                 if (i%4 == 0)
3180                         printf("\t");
3181                 else
3182                         printf(" ");
3183                 printf("F%02d = %016llx",i,(unsigned long long)es->fltregs[i]);
3184                 if (i%4 == 3)
3185                         printf("\n");
3186         }
3187 #endif
3188
3189         sp = (stackslot_t *) es->sp;
3190
3191         extraslots = 2;
3192
3193         if (es->code) {
3194                 methoddesc *md = es->code->m->parseddesc;
3195                 slots = code_get_stack_frame_size(es->code);
3196                 extraslots = 1 + md->memuse;
3197         }
3198         else
3199                 slots = 0;
3200
3201
3202         if (slots) {
3203                 printf("\tstack slots(+%d) at sp:", extraslots);
3204                 for (i=0; i<slots+extraslots; ++i) {
3205                         if (i%4 == 0)
3206                                 printf("\n\t\t");
3207                         printf("M%02d%c", i, (i >= slots) ? '(' : ' ');
3208 #ifdef HAS_4BYTE_STACKSLOT
3209                         printf("%08lx",(unsigned long)*sp++);
3210 #else
3211                         printf("%016llx",(unsigned long long)*sp++);
3212 #endif
3213                         printf("%c", (i >= slots) ? ')' : ' ');
3214                 }
3215                 printf("\n");
3216         }
3217
3218         printf("\tcode: %p", (void*)es->code);
3219         if (es->code != NULL) {
3220                 printf(" stackframesize=%d ", es->code->stackframesize);
3221                 method_print(es->code->m);
3222         }
3223         printf("\n");
3224
3225         printf("\n");
3226 }
3227 #endif
3228
3229 #if !defined(NDEBUG)
3230 static void java_value_print(s4 type, replace_val_t value)
3231 {
3232         java_objectheader *obj;
3233         utf               *u;
3234
3235         printf("%016llx",(unsigned long long) value.l);
3236
3237         if (type < 0 || type > TYPE_RET)
3238                 printf(" <INVALID TYPE:%d>", type);
3239         else
3240                 printf(" %s", show_jit_type_names[type]);
3241
3242         if (type == TYPE_ADR && value.a != NULL) {
3243                 obj = value.a;
3244                 putchar(' ');
3245                 utf_display_printable_ascii_classname(obj->vftbl->class->name);
3246
3247                 if (obj->vftbl->class == class_java_lang_String) {
3248                         printf(" \"");
3249                         u = javastring_toutf(obj, false);
3250                         utf_display_printable_ascii(u);
3251                         printf("\"");
3252                 }
3253         }
3254         else if (type == TYPE_INT) {
3255                 printf(" %ld", (long) value.i);
3256         }
3257         else if (type == TYPE_LNG) {
3258                 printf(" %lld", (long long) value.l);
3259         }
3260         else if (type == TYPE_FLT) {
3261                 printf(" %f", value.f);
3262         }
3263         else if (type == TYPE_DBL) {
3264                 printf(" %f", value.d);
3265         }
3266 }
3267 #endif /* !defined(NDEBUG) */
3268
3269
3270 #if !defined(NDEBUG)
3271 void replace_source_frame_println(sourceframe_t *frame)
3272 {
3273         s4 i,j;
3274         s4 t;
3275
3276         if (REPLACE_IS_NATIVE_FRAME(frame)) {
3277                 printf("\tNATIVE\n");
3278                 printf("\tsfi: "); replace_stackframeinfo_println(frame->sfi);
3279                 printf("\tnativepc: %p\n", frame->nativepc);
3280                 printf("\tframesize: %d\n", frame->nativeframesize);
3281
3282                 j = 0;
3283                 for (i=0; i<INT_REG_CNT; ++i) {
3284                         if (nregdescint[i] == REG_SAV)
3285                                 printf("\t%s = %p\n", abi_registers_integer_name[i], (void*)frame->nativesavint[j++]);
3286                 }
3287
3288                 j = 0;
3289                 for (i=0; i<FLT_REG_CNT; ++i) {
3290                         if (nregdescfloat[i] == REG_SAV)
3291                                 printf("\tF%02d = %f\n", i, frame->nativesavflt[j++]);
3292                 }
3293
3294                 printf("\n");
3295                 return;
3296         }
3297
3298         printf("\t");
3299         method_println(frame->method);
3300         printf("\tid: %d\n", frame->id);
3301         printf("\ttype: %s\n", replace_type_str[frame->type]);
3302         printf("\n");
3303
3304         if (frame->instance.a) {
3305                 printf("\tinstance: ");
3306                 java_value_print(TYPE_ADR, frame->instance);
3307                 printf("\n");
3308         }
3309
3310         if (frame->javalocalcount) {
3311                 printf("\tlocals (%d):\n",frame->javalocalcount);
3312                 for (i=0; i<frame->javalocalcount; ++i) {
3313                         t = frame->javalocaltype[i];
3314                         if (t == TYPE_VOID) {
3315                                 printf("\tlocal[ %2d] = void\n",i);
3316                         }
3317                         else {
3318                                 printf("\tlocal[%c%2d] = ",TYPECHAR(t),i);
3319                                 java_value_print(t, frame->javalocals[i]);
3320                                 printf("\n");
3321                         }
3322                 }
3323                 printf("\n");
3324         }
3325
3326         if (frame->javastackdepth) {
3327                 printf("\tstack (depth %d):\n",frame->javastackdepth);
3328                 for (i=0; i<frame->javastackdepth; ++i) {
3329                         t = frame->javastacktype[i];
3330                         if (t == TYPE_VOID) {
3331                                 printf("\tstack[%2d] = void", i);
3332                         }
3333                         else {
3334                                 printf("\tstack[%2d] = ",i);
3335                                 java_value_print(frame->javastacktype[i], frame->javastack[i]);
3336                                 printf("\n");
3337                         }
3338                 }
3339                 printf("\n");
3340         }
3341
3342         if (frame->syncslotcount) {
3343                 printf("\tsynchronization slots (%d):\n",frame->syncslotcount);
3344                 for (i=0; i<frame->syncslotcount; ++i) {
3345                         printf("\tslot[%2d] = ",i);
3346 #ifdef HAS_4BYTE_STACKSLOT
3347                         printf("%08lx\n",(unsigned long) frame->syncslots[i].p);
3348 #else
3349                         printf("%016llx\n",(unsigned long long) frame->syncslots[i].p);
3350 #endif
3351                 }
3352                 printf("\n");
3353         }
3354
3355         if (frame->fromcode) {
3356                 printf("\tfrom %p ", (void*)frame->fromcode);
3357                 method_println(frame->fromcode->m);
3358         }
3359         if (frame->tocode) {
3360                 printf("\tto %p ", (void*)frame->tocode);
3361                 method_println(frame->tocode->m);
3362         }
3363
3364         if (frame->fromrp) {
3365                 printf("\tfrom replacement point:\n");
3366                 replace_replacement_point_println(frame->fromrp, 2);
3367         }
3368         if (frame->torp) {
3369                 printf("\tto replacement point:\n");
3370                 replace_replacement_point_println(frame->torp, 2);
3371         }
3372
3373         printf("\n");
3374 }
3375 #endif /* !defined(NDEBUG) */
3376
3377
3378 /* replace_sourcestate_println *************************************************
3379
3380    Print source state
3381
3382    IN:
3383        ss...............the source state to print
3384
3385 *******************************************************************************/
3386
3387 #if !defined(NDEBUG)
3388 void replace_sourcestate_println(sourcestate_t *ss)
3389 {
3390         int i;
3391         sourceframe_t *frame;
3392
3393         if (!ss) {
3394                 printf("(sourcestate_t *)NULL\n");
3395                 return;
3396         }
3397
3398         printf("sourcestate_t:\n");
3399
3400         for (i=0, frame = ss->frames; frame != NULL; frame = frame->down, ++i) {
3401                 printf("    frame %d:\n", i);
3402                 replace_source_frame_println(frame);
3403         }
3404 }
3405 #endif
3406
3407
3408 /* replace_sourcestate_println_short *******************************************
3409
3410    Print a compact representation of the given source state.
3411
3412    IN:
3413        ss...............the source state to print
3414
3415 *******************************************************************************/
3416
3417 #if !defined(NDEBUG)
3418 void replace_sourcestate_println_short(sourcestate_t *ss)
3419 {
3420         sourceframe_t *frame;
3421
3422         for (frame = ss->frames; frame != NULL; frame = frame->down) {
3423                 printf("\t");
3424
3425                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
3426                         printf("NATIVE (pc %p size %d) ",
3427                                         (void*)frame->nativepc, frame->nativeframesize);
3428                         replace_stackframeinfo_println(frame->sfi);
3429                         continue;
3430                 }
3431
3432                 if (frame->torp) {
3433                         printf("%c", (frame->torp == frame->fromrp) ? '=' : '+');
3434                 }
3435
3436                 printf("%s", replace_type_str[frame->fromrp->type]);
3437
3438                 if (frame->torp && frame->torp->type != frame->fromrp->type)
3439                         printf("->%s", replace_type_str[frame->torp->type]);
3440
3441                 if (frame->tocode != frame->fromcode)
3442                         printf(" (%p->%p/%d) ",
3443                                    (void*) frame->fromcode, (void*) frame->tocode,
3444                                    frame->fromrp->id);
3445                 else
3446                         printf(" (%p/%d) ", (void*) frame->fromcode, frame->fromrp->id);
3447
3448                 method_println(frame->method);
3449         }
3450 }
3451 #endif
3452
3453 #if !defined(NDEBUG)
3454 static void replace_stackframeinfo_println(stackframeinfo *sfi)
3455 {
3456         printf("prev=%p pv=%p sp=%p ra=%p xpc=%p method=",
3457                         (void*)sfi->prev, (void*)sfi->pv, (void*)sfi->sp,
3458                         (void*)sfi->ra, (void*)sfi->xpc);
3459
3460         if (sfi->method)
3461                 method_println(sfi->method);
3462         else
3463                 printf("(nil)\n");
3464 }
3465 #endif
3466
3467 /*
3468  * These are local overrides for various environment variables in Emacs.
3469  * Please do not remove this and leave it at the end of the file, where
3470  * Emacs will automagically detect them.
3471  * ---------------------------------------------------------------------
3472  * Local variables:
3473  * mode: c
3474  * indent-tabs-mode: t
3475  * c-basic-offset: 4
3476  * tab-width: 4
3477  * End:
3478  * vim:noexpandtab:sw=4:ts=4:
3479  */