src/vm/jit/replace.c

   1 /* src/vm/jit/replace.c - on-stack replacement of methods
   2
   3    Copyright (C) 1996-2005, 2006, 2007, 2008
   4    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
   5
   6    This file is part of CACAO.
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2, or (at
  11    your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful, but
  14    WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16    General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, write to the Free Software
  20    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  21    02110-1301, USA.
  22
  23 */
  24
  25 #include "config.h"
  26 #include "vm/types.h"
  27
  28 #include <assert.h>
  29 #include <stdint.h>
  30 #include <stdlib.h>
  31
  32 #include "arch.h"
  33 #include "md.h"
  34
  35 #if defined(ENABLE_GC_CACAO)
  36 # include "mm/cacao-gc/gc.h"
  37 #endif
  38
  39 #include "mm/memory.h"
  40
  41 #include "threads/thread.h"
  42
  43 #include "toolbox/logging.h"
  44
  45 #include "vm/stringlocal.h"
  46
  47 #include "vm/jit/abi.h"
  48 #include "vm/jit/asmpart.h"
  49 #include "vm/jit/disass.h"
  50 #include "vm/jit/executionstate.h"
  51 #include "vm/jit/jit.h"
  52 #include "vm/jit/methodheader.h"
  53 #include "vm/jit/replace.h"
  54 #include "vm/jit/show.h"
  55 #include "vm/jit/stack.h"
  56
  57 #include "vmcore/options.h"
  58 #include "vmcore/classcache.h"
  59
  60
  61 #define REPLACE_PATCH_DYNAMIC_CALL
  62 /*#define REPLACE_PATCH_ALL*/
  63
  64 #if defined(ENABLE_VMLOG)
  65 #include <vmlog_cacao.h>
  66 #endif
  67
  68 /*** architecture-dependent configuration *************************************/
  69
  70 /* first unset the macros (default) */
  71 #undef REPLACE_RA_BETWEEN_FRAMES
  72 #undef REPLACE_RA_TOP_OF_FRAME
  73 #undef REPLACE_RA_LINKAGE_AREA
  74 #undef REPLACE_LEAFMETHODS_RA_REGISTER
  75 #undef REPLACE_REG_RA
  76
  77 /* i386, x86_64 and m68k */
  78 #if defined(__I386__) || defined(__X86_64__) || defined(__M68K__)
  79 #define REPLACE_RA_BETWEEN_FRAMES
  80 /* alpha */
  81 #elif defined(__ALPHA__)
  82 #define REPLACE_RA_TOP_OF_FRAME
  83 #define REPLACE_LEAFMETHODS_RA_REGISTER
  84 #define REPLACE_REG_RA REG_RA
  85 /* powerpc */
  86 #elif defined(__POWERPC__)
  87 #define REPLACE_RA_LINKAGE_AREA
  88 #define REPLACE_LEAFMETHODS_RA_REGISTER
  89 #define REPLACE_REG_RA REG_ITMP3 /* the execution state has the LR in itmp3 */
  90 /* s390 */
  91 #elif defined(__S390__)
  92 #define REPLACE_RA_TOP_OF_FRAME
  93 #define REPLACE_REG_RA REG_ITMP3
  94 #endif
  95
  96
  97 /*** configuration of native stack slot size **********************************/
  98
  99 /* XXX this should be in md-abi.h files, probably */
 100
 101 #if defined(HAS_4BYTE_STACKSLOT)
 102 #define SIZE_OF_STACKSLOT      4
 103 #define STACK_SLOTS_PER_FLOAT  2
 104 typedef u4 stackslot_t;
 105 #else
 106 #define SIZE_OF_STACKSLOT      8
 107 #define STACK_SLOTS_PER_FLOAT  1
 108 typedef u8 stackslot_t;
 109 #endif
 110
 111
 112 /*** debugging ****************************************************************/
 113
 114 #if !defined(NDEBUG)
 115 static void java_value_print(s4 type, replace_val_t value);
 116 static void replace_stackframeinfo_println(stackframeinfo_t *sfi);
 117 #endif
 118
 119 #if !defined(NDEBUG)
 120 #define DOLOG(code)        do{ if (opt_TraceReplacement > 1) { code; } } while(0)
 121 #define DOLOG_SHORT(code)  do{ if (opt_TraceReplacement > 0) { code; } } while(0)
 122 #else
 123 #define DOLOG(code)
 124 #define DOLOG_SHORT(code)
 125 #endif
 126
 127
 128 /*** statistics ***************************************************************/
 129
 130 #define REPLACE_STATISTICS
 131
 132 #if defined(REPLACE_STATISTICS)
 133
 134 static int stat_replacements = 0;
 135 static int stat_frames = 0;
 136 static int stat_recompile = 0;
 137 static int stat_staticpatch = 0;
 138 static int stat_unroll_inline = 0;
 139 static int stat_unroll_call = 0;
 140 static int stat_dist_frames[20] = { 0 };
 141 static int stat_dist_locals[20] = { 0 };
 142 static int stat_dist_locals_adr[10] = { 0 };
 143 static int stat_dist_locals_prim[10] = { 0 };
 144 static int stat_dist_locals_ret[10] = { 0 };
 145 static int stat_dist_locals_void[10] = { 0 };
 146 static int stat_dist_stack[10] = { 0 };
 147 static int stat_dist_stack_adr[10] = { 0 };
 148 static int stat_dist_stack_prim[10] = { 0 };
 149 static int stat_dist_stack_ret[10] = { 0 };
 150 static int stat_methods = 0;
 151 static int stat_rploints = 0;
 152 static int stat_regallocs = 0;
 153 static int stat_dist_method_rplpoints[20] = { 0 };
 154
 155 #define REPLACE_COUNT(cnt)  (cnt)++
 156 #define REPLACE_COUNT_IF(cnt, cond)  do{ if(cond) (cnt)++; } while(0)
 157 #define REPLACE_COUNT_INC(cnt, inc)  ((cnt) += (inc))
 158
 159 #define REPLACE_COUNT_DIST(array, val)                               \
 160     do {                                                             \
 161         int limit = (sizeof(array) / sizeof(int)) - 1;               \
 162         if ((val) < (limit)) (array)[val]++;                         \
 163         else (array)[limit]++;                                       \
 164     } while (0)
 165
 166 static void replace_statistics_source_frame(sourceframe_t *frame);
 167
 168 #else
 169
 170 #define REPLACE_COUNT(cnt)
 171 #define REPLACE_COUNT_IF(cnt, cond)
 172 #define REPLACE_COUNT_INC(cnt, inc)
 173 #define REPLACE_COUNT_DIST(array, val)
 174
 175 #endif /* defined(REPLACE_STATISTICS) */
 176
 177
 178 /*** constants used internally ************************************************/
 179
 180 #define TOP_IS_NORMAL    0
 181 #define TOP_IS_ON_STACK  1
 182 #define TOP_IS_IN_ITMP1  2
 183 #define TOP_IS_VOID      3
 184
 185
 186 /******************************************************************************/
 187 /* PART I: Creating / freeing replacement points                              */
 188 /******************************************************************************/
 189
 190
 191 /* replace_create_replacement_point ********************************************
 192
 193    Create a replacement point.
 194
 195    IN:
 196        jd...............current jitdata
 197            iinfo............inlining info for the current position
 198            rp...............pre-allocated (uninitialized) rplpoint
 199            type.............RPLPOINT_TYPE constant
 200            iptr.............current instruction
 201            *pra.............current rplalloc pointer
 202            javalocals.......the javalocals at the current point
 203            stackvars........the stack variables at the current point
 204            stackdepth.......the stack depth at the current point
 205            paramcount.......number of parameters at the start of stackvars
 206
 207    OUT:
 208        *rpa.............points to the next free rplalloc
 209
 210 *******************************************************************************/
 211
 212 static void replace_create_replacement_point(jitdata *jd,
 213                                                                                          insinfo_inline *iinfo,
 214                                                                                          rplpoint *rp,
 215                                                                                          s4 type,
 216                                                                                          instruction *iptr,
 217                                                                                          rplalloc **pra,
 218                                                                                          s4 *javalocals,
 219                                                                                          s4 *stackvars,
 220                                                                                          s4 stackdepth,
 221                                                                                          s4 paramcount)
 222 {
 223         rplalloc *ra;
 224         s4        i;
 225         varinfo  *v;
 226         s4        index;
 227
 228         ra = *pra;
 229
 230         REPLACE_COUNT(stat_rploints);
 231
 232         rp->method = (iinfo) ? iinfo->method : jd->m;
 233         rp->pc = NULL;        /* set by codegen */
 234         rp->callsize = 0;     /* set by codegen */
 235         rp->regalloc = ra;
 236         rp->flags = 0;
 237         rp->type = type;
 238         rp->id = iptr->flags.bits >> INS_FLAG_ID_SHIFT;
 239
 240         /* XXX unify these two fields */
 241         rp->parent = (iinfo) ? iinfo->rp : NULL;
 242
 243         /* store local allocation info of javalocals */
 244
 245         if (javalocals) {
 246                 for (i = 0; i < rp->method->maxlocals; ++i) {
 247                         index = javalocals[i];
 248                         if (index == UNUSED)
 249                                 continue;
 250
 251                         ra->index = i;
 252                         if (index >= 0) {
 253                                 v = VAR(index);
 254                                 ra->flags = v->flags & (INMEMORY);
 255                                 ra->regoff = v->vv.regoff;
 256                                 ra->type = v->type;
 257                         }
 258                         else {
 259                                 ra->regoff = RETADDR_FROM_JAVALOCAL(index);
 260                                 ra->type = TYPE_RET;
 261                                 ra->flags = 0;
 262                         }
 263                         ra++;
 264                 }
 265         }
 266
 267         /* store allocation info of java stack vars */
 268
 269         for (i = 0; i < stackdepth; ++i) {
 270                 v = VAR(stackvars[i]);
 271                 ra->flags = v->flags & (INMEMORY);
 272                 ra->index = (i < paramcount) ? RPLALLOC_PARAM : RPLALLOC_STACK;
 273                 ra->type  = v->type;
 274                 /* XXX how to handle locals on the stack containing returnAddresses? */
 275                 if (v->type == TYPE_RET) {
 276                         assert(stackvars[i] >= jd->localcount);
 277                         ra->regoff = v->vv.retaddr->nr;
 278                 }
 279                 else
 280                         ra->regoff = v->vv.regoff;
 281                 ra++;
 282         }
 283
 284         /* total number of allocations */
 285
 286         rp->regalloccount = ra - rp->regalloc;
 287
 288         *pra = ra;
 289 }
 290
 291
 292 /* replace_create_inline_start_replacement_point *******************************
 293
 294    Create an INLINE_START replacement point.
 295
 296    IN:
 297        jd...............current jitdata
 298            rp...............pre-allocated (uninitialized) rplpoint
 299            iptr.............current instruction
 300            *pra.............current rplalloc pointer
 301            javalocals.......the javalocals at the current point
 302
 303    OUT:
 304        *rpa.............points to the next free rplalloc
 305
 306    RETURN VALUE:
 307        the insinfo_inline * for the following inlined body
 308
 309 *******************************************************************************/
 310
 311 static insinfo_inline * replace_create_inline_start_replacement_point(
 312                                                                                          jitdata *jd,
 313                                                                                          rplpoint *rp,
 314                                                                                          instruction *iptr,
 315                                                                                          rplalloc **pra,
 316                                                                                          s4 *javalocals)
 317 {
 318         insinfo_inline *calleeinfo;
 319         rplalloc       *ra;
 320
 321         calleeinfo = iptr->sx.s23.s3.inlineinfo;
 322
 323         calleeinfo->rp = rp;
 324
 325         replace_create_replacement_point(jd, calleeinfo->parent, rp,
 326                         RPLPOINT_TYPE_INLINE, iptr, pra,
 327                         javalocals,
 328                         calleeinfo->stackvars, calleeinfo->stackvarscount,
 329                         calleeinfo->paramcount);
 330
 331         if (calleeinfo->synclocal != UNUSED) {
 332                 ra = (*pra)++;
 333                 ra->index  = RPLALLOC_SYNC;
 334                 ra->regoff = jd->var[calleeinfo->synclocal].vv.regoff;
 335                 ra->flags  = jd->var[calleeinfo->synclocal].flags & INMEMORY;
 336                 ra->type   = TYPE_ADR;
 337
 338                 rp->regalloccount++;
 339         }
 340
 341         return calleeinfo;
 342 }
 343
 344
 345 /* replace_create_replacement_points *******************************************
 346
 347    Create the replacement points for the given code.
 348
 349    IN:
 350        jd...............current jitdata, must not have any replacement points
 351
 352    OUT:
 353        code->rplpoints.......set to the list of replacement points
 354            code->rplpointcount...number of replacement points
 355            code->regalloc........list of allocation info
 356            code->regalloccount...total length of allocation info list
 357            code->globalcount.....number of global allocations at the
 358                                  start of code->regalloc
 359
 360    RETURN VALUE:
 361        true.............everything ok
 362        false............an exception has been thrown
 363
 364 *******************************************************************************/
 365
 366 #define CLEAR_javalocals(array, method)                              \
 367     do {                                                             \
 368         for (i=0; i<(method)->maxlocals; ++i)                        \
 369             (array)[i] = UNUSED;                                     \
 370     } while (0)
 371
 372 #define COPY_OR_CLEAR_javalocals(dest, array, method)                \
 373     do {                                                             \
 374         if ((array) != NULL)                                         \
 375             MCOPY((dest), (array), s4, (method)->maxlocals);         \
 376         else                                                         \
 377             CLEAR_javalocals((dest), (method));                      \
 378     } while (0)
 379
 380 #define COUNT_javalocals(array, method, counter)                     \
 381     do {                                                             \
 382         for (i=0; i<(method)->maxlocals; ++i)                        \
 383             if ((array)[i] != UNUSED)                                \
 384                                 (counter)++;                                         \
 385     } while (0)
 386
 387 bool replace_create_replacement_points(jitdata *jd)
 388 {
 389         codeinfo        *code;
 390         registerdata    *rd;
 391         basicblock      *bptr;
 392         int              count;
 393         methodinfo      *m;
 394         rplpoint        *rplpoints;
 395         rplpoint        *rp;
 396         int              alloccount;
 397         rplalloc        *regalloc;
 398         rplalloc        *ra;
 399         int              i;
 400         instruction     *iptr;
 401         instruction     *iend;
 402         s4              *javalocals;
 403         s4              *jl;
 404         methoddesc      *md;
 405         insinfo_inline  *iinfo;
 406         s4               startcount;
 407         s4               firstcount;
 408 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 409         bool             needentry;
 410 #endif
 411
 412         REPLACE_COUNT(stat_methods);
 413
 414         /* get required compiler data */
 415
 416         code = jd->code;
 417         rd   = jd->rd;
 418
 419         /* assert that we wont overwrite already allocated data */
 420
 421         assert(code);
 422         assert(code->m);
 423         assert(code->rplpoints == NULL);
 424         assert(code->rplpointcount == 0);
 425         assert(code->regalloc == NULL);
 426         assert(code->regalloccount == 0);
 427         assert(code->globalcount == 0);
 428
 429         m = code->m;
 430
 431         /* in instance methods, we may need a rplpoint at the method entry */
 432
 433 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 434         if (!(m->flags & ACC_STATIC)) {
 435                 jd->basicblocks[0].bitflags |= BBFLAG_REPLACEMENT;
 436                 needentry = true;
 437         }
 438         else {
 439                 needentry = false;
 440         }
 441 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
 442
 443         /* iterate over the basic block list to find replacement points */
 444
 445         count = 0;
 446         alloccount = 0;
 447
 448         javalocals = DMNEW(s4, jd->maxlocals);
 449
 450         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 451
 452                 /* skip dead code */
 453
 454                 if (bptr->flags < BBFINISHED)
 455                         continue;
 456
 457                 /* get info about this block */
 458
 459                 m = bptr->method;
 460                 iinfo = bptr->inlineinfo;
 461
 462                 /* initialize javalocals at the start of this block */
 463
 464                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 465
 466                 /* iterate over the instructions */
 467
 468                 iptr = bptr->iinstr;
 469                 iend = iptr + bptr->icount;
 470                 startcount = count;
 471                 firstcount = count;
 472
 473                 for (; iptr != iend; ++iptr) {
 474                         switch (iptr->opc) {
 475 #if defined(ENABLE_GC_CACAO)
 476                                 case ICMD_BUILTIN:
 477                                         md = iptr->sx.s23.s3.bte->md;
 478                                         count++;
 479                                         COUNT_javalocals(javalocals, m, alloccount);
 480                                         alloccount += iptr->s1.argcount;
 481                                         if (iinfo)
 482                                                 alloccount -= iinfo->throughcount;
 483                                         break;
 484 #endif
 485
 486                                 case ICMD_INVOKESTATIC:
 487                                 case ICMD_INVOKESPECIAL:
 488                                 case ICMD_INVOKEVIRTUAL:
 489                                 case ICMD_INVOKEINTERFACE:
 490                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 491                                         count++;
 492                                         COUNT_javalocals(javalocals, m, alloccount);
 493                                         alloccount += iptr->s1.argcount;
 494                                         if (iinfo)
 495                                                 alloccount -= iinfo->throughcount;
 496                                         break;
 497
 498                                 case ICMD_ISTORE:
 499                                 case ICMD_LSTORE:
 500                                 case ICMD_FSTORE:
 501                                 case ICMD_DSTORE:
 502                                 case ICMD_ASTORE:
 503                                         stack_javalocals_store(iptr, javalocals);
 504                                         break;
 505
 506                                 case ICMD_IRETURN:
 507                                 case ICMD_LRETURN:
 508                                 case ICMD_FRETURN:
 509                                 case ICMD_DRETURN:
 510                                 case ICMD_ARETURN:
 511                                         alloccount += 1;
 512                                         /* FALLTHROUGH! */
 513                                 case ICMD_RETURN:
 514                                         count++;
 515                                         break;
 516
 517                                 case ICMD_INLINE_START:
 518                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 519
 520                                         count++;
 521                                         COUNT_javalocals(javalocals, m, alloccount);
 522                                         alloccount += iinfo->stackvarscount;
 523                                         if (iinfo->synclocal != UNUSED)
 524                                                 alloccount++;
 525
 526                                         m = iinfo->method;
 527                                         /* javalocals may be set at next block start, or now */
 528                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 529                                         break;
 530
 531                                 case ICMD_INLINE_BODY:
 532                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 533
 534                                         jl = iinfo->javalocals_start;
 535                                         if (jl == NULL) {
 536                                                 /* get the javalocals from the following block start */
 537                                                 assert(bptr->next);
 538                                                 jl = bptr->next->javalocals;
 539                                         }
 540                                         count++;
 541                                         COUNT_javalocals(jl, m, alloccount);
 542                                         break;
 543
 544                                 case ICMD_INLINE_END:
 545                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 546                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 547                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 548                                         m = iinfo->outer;
 549                                         if (iinfo->javalocals_end)
 550                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 551                                         iinfo = iinfo->parent;
 552                                         break;
 553                         }
 554
 555                         if (iptr == bptr->iinstr)
 556                                 firstcount = count;
 557                 } /* end instruction loop */
 558
 559                 /* create replacement points at targets of backward branches */
 560                 /* We only need the replacement point there, if there is no  */
 561                 /* replacement point inside the block.                       */
 562
 563                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 564 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 565                         int test = (needentry && bptr == jd->basicblocks) ? firstcount : count;
 566 #else
 567                         int test = count;
 568 #endif
 569                         if (test > startcount) {
 570                                 /* we don't need an extra rplpoint */
 571                                 bptr->bitflags &= ~BBFLAG_REPLACEMENT;
 572                         }
 573                         else {
 574                                 count++;
 575                                 alloccount += bptr->indepth;
 576                                 if (bptr->inlineinfo)
 577                                         alloccount -= bptr->inlineinfo->throughcount;
 578
 579                                 COUNT_javalocals(bptr->javalocals, bptr->method, alloccount);
 580                         }
 581                 }
 582
 583         } /* end basicblock loop */
 584
 585         /* if no points were found, there's nothing to do */
 586
 587         if (!count)
 588                 return true;
 589
 590         /* allocate replacement point array and allocation array */
 591
 592         rplpoints = MNEW(rplpoint, count);
 593         regalloc = MNEW(rplalloc, alloccount);
 594         ra = regalloc;
 595
 596         /* initialize replacement point structs */
 597
 598         rp = rplpoints;
 599
 600         /* XXX try to share code with the counting loop! */
 601
 602         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 603                 /* skip dead code */
 604
 605                 if (bptr->flags < BBFINISHED)
 606                         continue;
 607
 608                 /* get info about this block */
 609
 610                 m = bptr->method;
 611                 iinfo = bptr->inlineinfo;
 612
 613                 /* initialize javalocals at the start of this block */
 614
 615                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 616
 617                 /* create replacement points at targets of backward branches */
 618
 619                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 620
 621                         i = (iinfo) ? iinfo->throughcount : 0;
 622                         replace_create_replacement_point(jd, iinfo, rp++,
 623                                         bptr->type, bptr->iinstr, &ra,
 624                                         bptr->javalocals, bptr->invars + i, bptr->indepth - i, 0);
 625
 626                         if (JITDATA_HAS_FLAG_COUNTDOWN(jd))
 627                                 rp[-1].flags |= RPLPOINT_FLAG_COUNTDOWN;
 628                 }
 629
 630                 /* iterate over the instructions */
 631
 632                 iptr = bptr->iinstr;
 633                 iend = iptr + bptr->icount;
 634
 635                 for (; iptr != iend; ++iptr) {
 636                         switch (iptr->opc) {
 637 #if defined(ENABLE_GC_CACAO)
 638                                 case ICMD_BUILTIN:
 639                                         md = iptr->sx.s23.s3.bte->md;
 640
 641                                         i = (iinfo) ? iinfo->throughcount : 0;
 642                                         replace_create_replacement_point(jd, iinfo, rp++,
 643                                                         RPLPOINT_TYPE_CALL, iptr, &ra,
 644                                                         javalocals, iptr->sx.s23.s2.args,
 645                                                         iptr->s1.argcount - i,
 646                                                         md->paramcount);
 647                                         break;
 648 #endif
 649
 650                                 case ICMD_INVOKESTATIC:
 651                                 case ICMD_INVOKESPECIAL:
 652                                 case ICMD_INVOKEVIRTUAL:
 653                                 case ICMD_INVOKEINTERFACE:
 654                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 655
 656                                         i = (iinfo) ? iinfo->throughcount : 0;
 657                                         replace_create_replacement_point(jd, iinfo, rp++,
 658                                                         RPLPOINT_TYPE_CALL, iptr, &ra,
 659                                                         javalocals, iptr->sx.s23.s2.args,
 660                                                         iptr->s1.argcount - i,
 661                                                         md->paramcount);
 662                                         break;
 663
 664                                 case ICMD_ISTORE:
 665                                 case ICMD_LSTORE:
 666                                 case ICMD_FSTORE:
 667                                 case ICMD_DSTORE:
 668                                 case ICMD_ASTORE:
 669                                         stack_javalocals_store(iptr, javalocals);
 670                                         break;
 671
 672                                 case ICMD_IRETURN:
 673                                 case ICMD_LRETURN:
 674                                 case ICMD_FRETURN:
 675                                 case ICMD_DRETURN:
 676                                 case ICMD_ARETURN:
 677                                         replace_create_replacement_point(jd, iinfo, rp++,
 678                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 679                                                         NULL, &(iptr->s1.varindex), 1, 0);
 680                                         break;
 681
 682                                 case ICMD_RETURN:
 683                                         replace_create_replacement_point(jd, iinfo, rp++,
 684                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 685                                                         NULL, NULL, 0, 0);
 686                                         break;
 687
 688                                 case ICMD_INLINE_START:
 689                                         iinfo = replace_create_inline_start_replacement_point(
 690                                                                 jd, rp++, iptr, &ra, javalocals);
 691                                         m = iinfo->method;
 692                                         /* javalocals may be set at next block start, or now */
 693                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 694                                         break;
 695
 696                                 case ICMD_INLINE_BODY:
 697                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 698
 699                                         jl = iinfo->javalocals_start;
 700                                         if (jl == NULL) {
 701                                                 /* get the javalocals from the following block start */
 702                                                 assert(bptr->next);
 703                                                 jl = bptr->next->javalocals;
 704                                         }
 705                                         /* create a non-trappable rplpoint */
 706                                         replace_create_replacement_point(jd, iinfo, rp++,
 707                                                         RPLPOINT_TYPE_BODY, iptr, &ra,
 708                                                         jl, NULL, 0, 0);
 709                                         rp[-1].flags |= RPLPOINT_FLAG_NOTRAP;
 710                                         break;
 711
 712                                 case ICMD_INLINE_END:
 713                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 714                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 715                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 716                                         m = iinfo->outer;
 717                                         if (iinfo->javalocals_end)
 718                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 719                                         iinfo = iinfo->parent;
 720                                         break;
 721                         }
 722                 } /* end instruction loop */
 723         } /* end basicblock loop */
 724
 725         assert((rp - rplpoints) == count);
 726         assert((ra - regalloc) == alloccount);
 727
 728         /* store the data in the codeinfo */
 729
 730         code->rplpoints     = rplpoints;
 731         code->rplpointcount = count;
 732         code->regalloc      = regalloc;
 733         code->regalloccount = alloccount;
 734         code->globalcount   = 0;
 735         code->savedintcount = INT_SAV_CNT - rd->savintreguse;
 736         code->savedfltcount = FLT_SAV_CNT - rd->savfltreguse;
 737 #if defined(HAS_ADDRESS_REGISTER_FILE)
 738         code->savedadrcount = ADR_SAV_CNT - rd->savadrreguse;
 739 #endif
 740         code->memuse        = rd->memuse;
 741         code->stackframesize = jd->cd->stackframesize;
 742
 743         REPLACE_COUNT_DIST(stat_dist_method_rplpoints, count);
 744         REPLACE_COUNT_INC(stat_regallocs, alloccount);
 745
 746         /* everything alright */
 747
 748         return true;
 749 }
 750
 751
 752 /* replace_free_replacement_points *********************************************
 753
 754    Free memory used by replacement points.
 755
 756    IN:
 757        code.............codeinfo whose replacement points should be freed.
 758
 759 *******************************************************************************/
 760
 761 void replace_free_replacement_points(codeinfo *code)
 762 {
 763         assert(code);
 764
 765         if (code->rplpoints)
 766                 MFREE(code->rplpoints,rplpoint,code->rplpointcount);
 767
 768         if (code->regalloc)
 769                 MFREE(code->regalloc,rplalloc,code->regalloccount);
 770
 771         code->rplpoints = NULL;
 772         code->rplpointcount = 0;
 773         code->regalloc = NULL;
 774         code->regalloccount = 0;
 775         code->globalcount = 0;
 776 }
 777
 778
 779 /******************************************************************************/
 780 /* PART II: Activating / deactivating replacement points                      */
 781 /******************************************************************************/
 782
 783
 784 /* replace_activate_replacement_points *****************************************
 785
 786    Activate the replacement points of the given compilation unit. When this
 787    function returns, the replacement points are "armed", so each thread
 788    reaching one of the points will enter the replacement mechanism.
 789
 790    IN:
 791        code.............codeinfo of which replacement points should be
 792                                                 activated
 793            mappable.........if true, only mappable replacement points are
 794                                                 activated
 795
 796 *******************************************************************************/
 797
 798 void replace_activate_replacement_points(codeinfo *code, bool mappable)
 799 {
 800         rplpoint *rp;
 801         s4        i;
 802         s4        count;
 803         u1       *savedmcode;
 804
 805         assert(code->savedmcode == NULL);
 806
 807         /* count trappable replacement points */
 808
 809         count = 0;
 810         i = code->rplpointcount;
 811         rp = code->rplpoints;
 812         for (; i--; rp++) {
 813                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 814                         continue;
 815
 816                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 817                         continue;
 818
 819                 count++;
 820         }
 821
 822         /* allocate buffer for saved machine code */
 823
 824         savedmcode = MNEW(u1, count * REPLACEMENT_PATCH_SIZE);
 825         code->savedmcode = savedmcode;
 826         savedmcode += count * REPLACEMENT_PATCH_SIZE;
 827
 828         /* activate trappable replacement points */
 829         /* (in reverse order to handle overlapping points within basic blocks) */
 830
 831         i = code->rplpointcount;
 832         rp = code->rplpoints + i;
 833         while (rp--, i--) {
 834                 assert(!(rp->flags & RPLPOINT_FLAG_ACTIVE));
 835
 836                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 837                         continue;
 838
 839                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 840                         continue;
 841
 842                 DOLOG( printf("activate replacement point:\n");
 843                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 844
 845                 savedmcode -= REPLACEMENT_PATCH_SIZE;
 846
 847 #if defined(ENABLE_JIT)
 848 # if defined(ENABLE_DISASSEMBLER)
 849                 DOLOG( printf("\tinstruction before: ");
 850                            disassinstr(rp->pc); fflush(stdout); );
 851 # endif
 852
 853                 md_patch_replacement_point(rp->pc, savedmcode, false);
 854
 855 # if defined(ENABLE_DISASSEMBLER)
 856                 DOLOG( printf("\tinstruction after : ");
 857                            disassinstr(rp->pc); fflush(stdout); );
 858 # endif
 859 #endif
 860
 861                 rp->flags |= RPLPOINT_FLAG_ACTIVE;
 862         }
 863
 864         assert(savedmcode == code->savedmcode);
 865 }
 866
 867
 868 /* replace_deactivate_replacement_points ***************************************
 869
 870    Deactivate a replacement points in the given compilation unit.
 871    When this function returns, the replacement points will be "un-armed",
 872    that is a each thread reaching a point will just continue normally.
 873
 874    IN:
 875        code.............the compilation unit
 876
 877 *******************************************************************************/
 878
 879 void replace_deactivate_replacement_points(codeinfo *code)
 880 {
 881         rplpoint *rp;
 882         s4        i;
 883         s4        count;
 884         u1       *savedmcode;
 885
 886         if (code->savedmcode == NULL) {
 887                 /* disarm countdown points by patching the branches */
 888
 889                 i = code->rplpointcount;
 890                 rp = code->rplpoints;
 891                 for (; i--; rp++) {
 892                         if ((rp->flags & (RPLPOINT_FLAG_ACTIVE | RPLPOINT_FLAG_COUNTDOWN))
 893                                         == RPLPOINT_FLAG_COUNTDOWN)
 894                         {
 895 #if 0
 896                                 *(s4*) (rp->pc + 9) = 0; /* XXX machine dependent! */
 897 #endif
 898                         }
 899                 }
 900                 return;
 901         }
 902
 903         assert(code->savedmcode != NULL);
 904         savedmcode = code->savedmcode;
 905
 906         /* de-activate each trappable replacement point */
 907
 908         i = code->rplpointcount;
 909         rp = code->rplpoints;
 910         count = 0;
 911         for (; i--; rp++) {
 912                 if (!(rp->flags & RPLPOINT_FLAG_ACTIVE))
 913                         continue;
 914
 915                 count++;
 916
 917                 DOLOG( printf("deactivate replacement point:\n");
 918                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 919
 920 #if defined(ENABLE_JIT)
 921 # if defined(ENABLE_DISASSEMBLER)
 922                 DOLOG( printf("\tinstruction before: ");
 923                            disassinstr(rp->pc); fflush(stdout); );
 924 # endif
 925
 926                 md_patch_replacement_point(rp->pc, savedmcode, true);
 927
 928 # if defined(ENABLE_DISASSEMBLER)
 929                 DOLOG( printf("\tinstruction before: ");
 930                            disassinstr(rp->pc); fflush(stdout); );
 931 # endif
 932 #endif
 933
 934                 rp->flags &= ~RPLPOINT_FLAG_ACTIVE;
 935
 936                 savedmcode += REPLACEMENT_PATCH_SIZE;
 937         }
 938
 939         assert(savedmcode == code->savedmcode + count * REPLACEMENT_PATCH_SIZE);
 940
 941         /* free saved machine code */
 942
 943         MFREE(code->savedmcode, u1, count * REPLACEMENT_PATCH_SIZE);
 944         code->savedmcode = NULL;
 945 }
 946
 947
 948 /******************************************************************************/
 949 /* PART III: The replacement mechanism                                        */
 950 /******************************************************************************/
 951
 952
 953 /* replace_read_value **********************************************************
 954
 955    Read a value with the given allocation from the execution state.
 956
 957    IN:
 958            es...............execution state
 959            ra...............allocation
 960            javaval..........where to put the value
 961
 962    OUT:
 963        *javaval.........the value
 964
 965 *******************************************************************************/
 966
 967 static void replace_read_value(executionstate_t *es,
 968                                                            rplalloc *ra,
 969                                                            replace_val_t *javaval)
 970 {
 971         if (ra->flags & INMEMORY) {
 972                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
 973 #ifdef HAS_4BYTE_STACKSLOT
 974                 if (IS_2_WORD_TYPE(ra->type)) {
 975                         javaval->l = *(u8*)(es->sp + ra->regoff);
 976                 }
 977                 else {
 978 #endif
 979                         javaval->p = *(ptrint*)(es->sp + ra->regoff);
 980 #ifdef HAS_4BYTE_STACKSLOT
 981                 }
 982 #endif
 983         }
 984         else {
 985                 /* allocated register */
 986                 if (IS_FLT_DBL_TYPE(ra->type)) {
 987                         javaval->d = es->fltregs[ra->regoff];
 988
 989                         if (ra->type == TYPE_FLT)
 990                                 javaval->f = javaval->d;
 991                 }
 992 #if defined(HAS_ADDRESS_REGISTER_FILE)
 993                 else if (IS_ADR_TYPE(ra->type)) {
 994                         javaval->p = es->adrregs[ra->regoff];
 995                 }
 996 #endif
 997                 else {
 998 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
 999                         if (ra->type == TYPE_LNG) {
1000                                 javaval->words.lo = es->intregs[GET_LOW_REG(ra->regoff)];
1001                                 javaval->words.hi = es->intregs[GET_HIGH_REG(ra->regoff)];
1002                         }
1003                         else
1004 #endif /* defined(SUPPORT_COMBINE_INTEGER_REGISTERS) */
1005                                 javaval->p = es->intregs[ra->regoff];
1006                 }
1007         }
1008 }
1009
1010
1011 /* replace_write_value *********************************************************
1012
1013    Write a value to the given allocation in the execution state.
1014
1015    IN:
1016            es...............execution state
1017            ra...............allocation
1018            *javaval.........the value
1019
1020 *******************************************************************************/
1021
1022 static void replace_write_value(executionstate_t *es,
1023                                                             rplalloc *ra,
1024                                                             replace_val_t *javaval)
1025 {
1026         if (ra->flags & INMEMORY) {
1027                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
1028 #ifdef HAS_4BYTE_STACKSLOT
1029                 if (IS_2_WORD_TYPE(ra->type)) {
1030                         *(u8*)(es->sp + ra->regoff) = javaval->l;
1031                 }
1032                 else {
1033 #endif
1034                         *(ptrint*)(es->sp + ra->regoff) = javaval->p;
1035 #ifdef HAS_4BYTE_STACKSLOT
1036                 }
1037 #endif
1038         }
1039         else {
1040                 /* allocated register */
1041                 switch (ra->type) {
1042                         case TYPE_FLT:
1043                                 es->fltregs[ra->regoff] = (double) javaval->f;
1044                                 break;
1045                         case TYPE_DBL:
1046                                 es->fltregs[ra->regoff] = javaval->d;
1047                                 break;
1048 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
1049                         case TYPE_LNG:
1050                                 es->intregs[GET_LOW_REG(ra->regoff)] = javaval->words.lo;
1051                                 es->intregs[GET_HIGH_REG(ra->regoff)] = javaval->words.hi;
1052                                 break;
1053 #endif
1054 #if defined(HAS_ADDRESS_REGISTER_FILE)
1055                         case TYPE_ADR:
1056                                 es->adrregs[ra->regoff] = javaval->p;
1057 #endif
1058                         default:
1059                                 es->intregs[ra->regoff] = javaval->p;
1060                 }
1061         }
1062 }
1063
1064
1065 /* replace_new_sourceframe *****************************************************
1066
1067    Allocate a new source frame and insert it at the front of the frame list.
1068
1069    IN:
1070            ss...............the source state
1071
1072    OUT:
1073            ss->frames.......set to new frame (the new head of the frame list).
1074
1075    RETURN VALUE:
1076        returns the new frame
1077
1078 *******************************************************************************/
1079
1080 static sourceframe_t *replace_new_sourceframe(sourcestate_t *ss)
1081 {
1082         sourceframe_t *frame;
1083
1084         frame = DNEW(sourceframe_t);
1085         MZERO(frame, sourceframe_t, 1);
1086
1087         frame->down = ss->frames;
1088         ss->frames = frame;
1089
1090         return frame;
1091 }
1092
1093
1094 /* replace_read_executionstate *************************************************
1095
1096    Read a source frame from the given executions state.
1097    The new source frame is pushed to the front of the frame list of the
1098    source state.
1099
1100    IN:
1101        rp...............replacement point at which `es` was taken
1102            es...............execution state
1103            ss...............the source state to add the source frame to
1104            topframe.........true, if the first (top-most) source frame on the
1105                             stack is to be read
1106
1107    OUT:
1108        *ss..............the source state with the newly created source frame
1109                             added
1110
1111 *******************************************************************************/
1112
1113 static s4 replace_normalize_type_map[] = {
1114 /* RPLPOINT_TYPE_STD    |--> */ RPLPOINT_TYPE_STD,
1115 /* RPLPOINT_TYPE_EXH    |--> */ RPLPOINT_TYPE_STD,
1116 /* RPLPOINT_TYPE_SBR    |--> */ RPLPOINT_TYPE_STD,
1117 /* RPLPOINT_TYPE_CALL   |--> */ RPLPOINT_TYPE_CALL,
1118 /* RPLPOINT_TYPE_INLINE |--> */ RPLPOINT_TYPE_CALL,
1119 /* RPLPOINT_TYPE_RETURN |--> */ RPLPOINT_TYPE_RETURN,
1120 /* RPLPOINT_TYPE_BODY   |--> */ RPLPOINT_TYPE_STD
1121 };
1122
1123
1124 static void replace_read_executionstate(rplpoint *rp,
1125                                                                                 executionstate_t *es,
1126                                                                                 sourcestate_t *ss,
1127                                                                                 bool topframe)
1128 {
1129         methodinfo    *m;
1130         codeinfo      *code;
1131         int            count;
1132         int            i;
1133         rplalloc      *ra;
1134         sourceframe_t *frame;
1135         int            topslot;
1136         stackslot_t   *sp;
1137         stackslot_t   *basesp;
1138
1139         code = code_find_codeinfo_for_pc(rp->pc);
1140         m = rp->method;
1141         topslot = TOP_IS_NORMAL;
1142
1143         /* stack pointer */
1144
1145         sp = (stackslot_t *) es->sp;
1146
1147         /* in some cases the top stack slot is passed in REG_ITMP1 */
1148
1149         if (rp->type == BBTYPE_EXH) {
1150                 topslot = TOP_IS_IN_ITMP1;
1151         }
1152
1153         /* calculate base stack pointer */
1154
1155         basesp = sp + code->stackframesize;
1156
1157         /* create the source frame */
1158
1159         frame = replace_new_sourceframe(ss);
1160         frame->method = rp->method;
1161         frame->id = rp->id;
1162         assert(rp->type >= 0 && rp->type < sizeof(replace_normalize_type_map)/sizeof(s4));
1163         frame->type = replace_normalize_type_map[rp->type];
1164         frame->fromrp = rp;
1165         frame->fromcode = code;
1166
1167         /* read local variables */
1168
1169         count = m->maxlocals;
1170         frame->javalocalcount = count;
1171         frame->javalocals = DMNEW(replace_val_t, count);
1172         frame->javalocaltype = DMNEW(u1, count);
1173
1174         /* mark values as undefined */
1175         for (i=0; i<count; ++i) {
1176 #if !defined(NDEBUG)
1177                 frame->javalocals[i].l = (u8) 0x00dead0000dead00ULL;
1178 #endif
1179                 frame->javalocaltype[i] = TYPE_VOID;
1180         }
1181
1182         /* some entries in the intregs array are not meaningful */
1183         /*es->intregs[REG_ITMP3] = (u8) 0x11dead1111dead11ULL;*/
1184 #if !defined(NDEBUG)
1185         es->intregs[REG_SP   ] = (ptrint) 0x11dead1111dead11ULL;
1186 #ifdef REG_PV
1187         es->intregs[REG_PV   ] = (ptrint) 0x11dead1111dead11ULL;
1188 #endif
1189 #endif /* !defined(NDEBUG) */
1190
1191         /* read javalocals */
1192
1193         count = rp->regalloccount;
1194         ra = rp->regalloc;
1195
1196         while (count && (i = ra->index) >= 0) {
1197                 assert(i < m->maxlocals);
1198                 frame->javalocaltype[i] = ra->type;
1199                 if (ra->type == TYPE_RET)
1200                         frame->javalocals[i].i = ra->regoff;
1201                 else
1202                         replace_read_value(es, ra, frame->javalocals + i);
1203                 ra++;
1204                 count--;
1205         }
1206
1207         /* read instance, if this is the first rplpoint */
1208
1209 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
1210         if (topframe && !(rp->method->flags & ACC_STATIC) && rp == code->rplpoints) {
1211 #if 1
1212                 /* we are at the start of the method body, so if local 0 is set, */
1213                 /* it is the instance.                                           */
1214                 if (frame->javalocaltype[0] == TYPE_ADR)
1215                         frame->instance = frame->javalocals[0];
1216 #else
1217                 rplalloc instra;
1218                 methoddesc *md;
1219
1220                 md = rp->method->parseddesc;
1221                 assert(md->params);
1222                 assert(md->paramcount >= 1);
1223                 instra.type = TYPE_ADR;
1224                 instra.regoff = md->params[0].regoff;
1225                 if (md->params[0].inmemory) {
1226                         instra.flags = INMEMORY;
1227                         instra.regoff += (1 + code->stackframesize) * SIZE_OF_STACKSLOT;
1228                 }
1229                 else {
1230                         instra.flags = 0;
1231                 }
1232                 replace_read_value(es, &instra, &(frame->instance));
1233 #endif
1234         }
1235 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
1236
1237         /* read stack slots */
1238
1239         frame->javastackdepth = count;
1240         frame->javastack = DMNEW(replace_val_t, count);
1241         frame->javastacktype = DMNEW(u1, count);
1242
1243 #if !defined(NDEBUG)
1244         /* mark values as undefined */
1245         for (i=0; i<count; ++i) {
1246                 frame->javastack[i].l = (u8) 0x00dead0000dead00ULL;
1247                 frame->javastacktype[i] = TYPE_VOID;
1248         }
1249 #endif /* !defined(NDEBUG) */
1250
1251         i = 0;
1252
1253         /* the first stack slot is special in SBR and EXH blocks */
1254
1255         if (topslot == TOP_IS_ON_STACK) {
1256                 assert(count);
1257
1258                 assert(ra->index == RPLALLOC_STACK);
1259                 assert(ra->type == TYPE_ADR);
1260                 frame->javastack[i].p = sp[-1];
1261                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1262                 count--;
1263                 i++;
1264                 ra++;
1265         }
1266         else if (topslot == TOP_IS_IN_ITMP1) {
1267                 assert(count);
1268
1269                 assert(ra->index == RPLALLOC_STACK);
1270                 assert(ra->type == TYPE_ADR);
1271                 frame->javastack[i].p = es->intregs[REG_ITMP1];
1272                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1273                 count--;
1274                 i++;
1275                 ra++;
1276         }
1277         else if (topslot == TOP_IS_VOID) {
1278                 assert(count);
1279
1280                 assert(ra->index == RPLALLOC_STACK);
1281                 frame->javastack[i].l = 0;
1282                 frame->javastacktype[i] = TYPE_VOID;
1283                 count--;
1284                 i++;
1285                 ra++;
1286         }
1287
1288         /* read remaining stack slots */
1289
1290         for (; count--; ra++) {
1291                 if (ra->index == RPLALLOC_SYNC) {
1292                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1293
1294                         /* only read synchronization slots when traversing an inline point */
1295
1296                         if (!topframe) {
1297                                 sourceframe_t *calleeframe = frame->down;
1298                                 assert(calleeframe);
1299                                 assert(calleeframe->syncslotcount == 0);
1300                                 assert(calleeframe->syncslots == NULL);
1301
1302                                 calleeframe->syncslotcount = 1;
1303                                 calleeframe->syncslots = DMNEW(replace_val_t, 1);
1304                                 replace_read_value(es,ra,calleeframe->syncslots);
1305                         }
1306
1307                         frame->javastackdepth--;
1308                         continue;
1309                 }
1310
1311                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1312
1313                 /* do not read parameters of calls down the call chain */
1314
1315                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1316                         frame->javastackdepth--;
1317                 }
1318                 else {
1319                         if (ra->type == TYPE_RET)
1320                                 frame->javastack[i].i = ra->regoff;
1321                         else
1322                                 replace_read_value(es,ra,frame->javastack + i);
1323                         frame->javastacktype[i] = ra->type;
1324                         i++;
1325                 }
1326         }
1327 }
1328
1329
1330 /* replace_write_executionstate ************************************************
1331
1332    Pop a source frame from the front of the frame list of the given source state
1333    and write its values into the execution state.
1334
1335    IN:
1336        rp...............replacement point for which execution state should be
1337                             created
1338            es...............the execution state to modify
1339            ss...............the given source state
1340            topframe.........true, if this is the last (top-most) source frame to be
1341                             translated
1342
1343    OUT:
1344        *es..............the execution state derived from the source state
1345
1346 *******************************************************************************/
1347
1348 static void replace_write_executionstate(rplpoint *rp,
1349                                                                                  executionstate_t *es,
1350                                                                                  sourcestate_t *ss,
1351                                                                                  bool topframe)
1352 {
1353         methodinfo     *m;
1354         codeinfo       *code;
1355         int             count;
1356         int             i;
1357         rplalloc       *ra;
1358         sourceframe_t  *frame;
1359         int             topslot;
1360         stackslot_t    *sp;
1361         stackslot_t    *basesp;
1362
1363         code = code_find_codeinfo_for_pc(rp->pc);
1364         m = rp->method;
1365         topslot = TOP_IS_NORMAL;
1366
1367         /* pop a source frame */
1368
1369         frame = ss->frames;
1370         assert(frame);
1371         ss->frames = frame->down;
1372
1373         /* calculate stack pointer */
1374
1375         sp = (stackslot_t *) es->sp;
1376
1377         basesp = sp + code->stackframesize;
1378
1379         /* in some cases the top stack slot is passed in REG_ITMP1 */
1380
1381         if (rp->type == BBTYPE_EXH) {
1382                 topslot = TOP_IS_IN_ITMP1;
1383         }
1384
1385         /* write javalocals */
1386
1387         ra = rp->regalloc;
1388         count = rp->regalloccount;
1389
1390         while (count && (i = ra->index) >= 0) {
1391                 assert(i < m->maxlocals);
1392                 assert(i < frame->javalocalcount);
1393                 assert(ra->type == frame->javalocaltype[i]);
1394                 if (ra->type == TYPE_RET) {
1395                         /* XXX assert that it matches this rplpoint */
1396                 }
1397                 else
1398                         replace_write_value(es, ra, frame->javalocals + i);
1399                 count--;
1400                 ra++;
1401         }
1402
1403         /* write stack slots */
1404
1405         i = 0;
1406
1407         /* the first stack slot is special in SBR and EXH blocks */
1408
1409         if (topslot == TOP_IS_ON_STACK) {
1410                 assert(count);
1411
1412                 assert(ra->index == RPLALLOC_STACK);
1413                 assert(i < frame->javastackdepth);
1414                 assert(frame->javastacktype[i] == TYPE_ADR);
1415                 sp[-1] = frame->javastack[i].p;
1416                 count--;
1417                 i++;
1418                 ra++;
1419         }
1420         else if (topslot == TOP_IS_IN_ITMP1) {
1421                 assert(count);
1422
1423                 assert(ra->index == RPLALLOC_STACK);
1424                 assert(i < frame->javastackdepth);
1425                 assert(frame->javastacktype[i] == TYPE_ADR);
1426                 es->intregs[REG_ITMP1] = frame->javastack[i].p;
1427                 count--;
1428                 i++;
1429                 ra++;
1430         }
1431         else if (topslot == TOP_IS_VOID) {
1432                 assert(count);
1433
1434                 assert(ra->index == RPLALLOC_STACK);
1435                 assert(i < frame->javastackdepth);
1436                 assert(frame->javastacktype[i] == TYPE_VOID);
1437                 count--;
1438                 i++;
1439                 ra++;
1440         }
1441
1442         /* write remaining stack slots */
1443
1444         for (; count--; ra++) {
1445                 if (ra->index == RPLALLOC_SYNC) {
1446                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1447
1448                         /* only write synchronization slots when traversing an inline point */
1449
1450                         if (!topframe) {
1451                                 assert(frame->down);
1452                                 assert(frame->down->syncslotcount == 1); /* XXX need to understand more cases */
1453                                 assert(frame->down->syncslots != NULL);
1454
1455                                 replace_write_value(es,ra,frame->down->syncslots);
1456                         }
1457                         continue;
1458                 }
1459
1460                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1461
1462                 /* do not write parameters of calls down the call chain */
1463
1464                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1465                         /* skip it */
1466                 }
1467                 else {
1468                         assert(i < frame->javastackdepth);
1469                         assert(ra->type == frame->javastacktype[i]);
1470                         if (ra->type == TYPE_RET) {
1471                                 /* XXX assert that it matches this rplpoint */
1472                         }
1473                         else {
1474                                 replace_write_value(es,ra,frame->javastack + i);
1475                         }
1476                         i++;
1477                 }
1478         }
1479
1480         /* set new pc */
1481
1482         es->pc = rp->pc;
1483 }
1484
1485
1486 /* replace_pop_activation_record ***********************************************
1487
1488    Peel a stack frame from the execution state.
1489
1490    *** This function imitates the effects of the method epilog ***
1491    *** and returning from the method call.                     ***
1492
1493    IN:
1494            es...............execution state
1495            frame............source frame, receives synchronization slots
1496
1497    OUT:
1498        *es..............the execution state after popping the stack frame
1499
1500 *******************************************************************************/
1501
1502 u1* replace_pop_activation_record(executionstate_t *es,
1503                                                                   sourceframe_t *frame)
1504 {
1505         u1 *ra;
1506         u1 *pv;
1507         s4 reg;
1508         s4 i;
1509         s4 count;
1510         codeinfo *code;
1511         stackslot_t *basesp;
1512         stackslot_t *sp;
1513
1514         assert(es->code);
1515         assert(frame);
1516
1517         /* read the return address */
1518
1519 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1520         if (code_is_leafmethod(es->code))
1521                 ra = (u1*) (ptrint) es->intregs[REPLACE_REG_RA];
1522         else
1523 #endif
1524                 ra = md_stacktrace_get_returnaddress(es->sp,
1525                                 SIZE_OF_STACKSLOT * es->code->stackframesize);
1526
1527         DOLOG( printf("RA = %p\n", (void*)ra); );
1528
1529         assert(ra);
1530
1531         /* calculate the base of the stack frame */
1532
1533         sp = (stackslot_t *) es->sp;
1534         basesp = sp + es->code->stackframesize;
1535
1536         /* read slots used for synchronization */
1537
1538         assert(frame->syncslotcount == 0);
1539         assert(frame->syncslots == NULL);
1540         count = code_get_sync_slot_count(es->code);
1541         frame->syncslotcount = count;
1542         frame->syncslots = DMNEW(replace_val_t, count);
1543         for (i=0; i<count; ++i) {
1544                 frame->syncslots[i].p = sp[es->code->memuse + i]; /* XXX */
1545         }
1546
1547         /* restore return address, if part of frame */
1548
1549 #if defined(REPLACE_RA_TOP_OF_FRAME)
1550 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1551         if (!code_is_leafmethod(es->code))
1552 #endif
1553                 es->intregs[REPLACE_REG_RA] = *--basesp;
1554 #endif /* REPLACE_RA_TOP_OF_FRAME */
1555
1556 #if defined(REPLACE_RA_LINKAGE_AREA)
1557 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1558         if (!code_is_leafmethod(es->code))
1559 #endif
1560                 es->intregs[REPLACE_REG_RA] = basesp[LA_LR_OFFSET / sizeof(stackslot_t)];
1561 #endif /* REPLACE_RA_LINKAGE_AREA */
1562
1563         /* restore saved int registers */
1564
1565         reg = INT_REG_CNT;
1566         for (i=0; i<es->code->savedintcount; ++i) {
1567                 while (nregdescint[--reg] != REG_SAV)
1568                         ;
1569                 es->intregs[reg] = *--basesp;
1570         }
1571
1572         /* restore saved flt registers */
1573
1574         /* XXX align? */
1575         reg = FLT_REG_CNT;
1576         for (i=0; i<es->code->savedfltcount; ++i) {
1577                 while (nregdescfloat[--reg] != REG_SAV)
1578                         ;
1579                 basesp -= STACK_SLOTS_PER_FLOAT;
1580                 es->fltregs[reg] = *(double*)basesp;
1581         }
1582
1583 #if defined(HAS_ADDRESS_REGISTER_FILE)
1584         /* restore saved adr registers */
1585
1586         reg = ADR_REG_CNT;
1587         for (i=0; i<es->code->savedadrcount; ++i) {
1588                 while (nregdescadr[--reg] != REG_SAV)
1589                         ;
1590                 es->adrregs[reg] = *--basesp;
1591         }
1592 #endif
1593
1594         /* adjust the stackpointer */
1595
1596         es->sp += SIZE_OF_STACKSLOT * es->code->stackframesize;
1597
1598 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1599         es->sp += SIZE_OF_STACKSLOT; /* skip return address */
1600 #endif
1601
1602         /* Set the new pc. Subtract one so we do not hit the replacement point */
1603         /* of the instruction following the call, if there is one.             */
1604
1605         es->pc = ra - 1;
1606
1607         /* find the new codeinfo */
1608
1609         pv = md_codegen_get_pv_from_pc(ra);
1610         DOLOG( printf("PV = %p\n", (void*) pv); );
1611
1612         code = code_get_codeinfo_for_pv(pv);
1613         DOLOG( printf("CODE = %p\n", (void*) code); );
1614
1615         /* return NULL if we reached native code */
1616
1617         es->pv = pv;
1618         es->code = code;
1619
1620         /* in debugging mode clobber non-saved registers */
1621
1622 #if !defined(NDEBUG)
1623         /* for debugging */
1624         for (i=0; i<INT_REG_CNT; ++i)
1625                 if ((nregdescint[i] != REG_SAV)
1626 #if defined(REG_RA)
1627                                 && (i != REPLACE_REG_RA)
1628 #endif
1629                         )
1630                         es->intregs[i] = (ptrint) 0x33dead3333dead33ULL;
1631         for (i=0; i<FLT_REG_CNT; ++i)
1632                 if (nregdescfloat[i] != REG_SAV)
1633                         *(u8*)&(es->fltregs[i]) = 0x33dead3333dead33ULL;
1634 # if defined(HAS_ADDRESS_REGISTER_FILE)
1635         for (i=0; i<ADR_REG_CNT; ++i)
1636                 if (nregdescadr[i] != REG_SAV)
1637                         es->adrregs[i] = (ptrint) 0x33dead3333dead33ULL;
1638 # endif
1639 #endif /* !defined(NDEBUG) */
1640
1641         return (code) ? ra : NULL;
1642 }
1643
1644
1645 /* replace_patch_method_pointer ************************************************
1646
1647    Patch a method pointer (may be in code, data segment, vftbl, or interface
1648    table).
1649
1650    IN:
1651            mpp..............address of the method pointer to patch
1652            entrypoint.......the new entrypoint of the method
1653            kind.............kind of call to patch, used only for debugging
1654
1655 *******************************************************************************/
1656
1657 static void replace_patch_method_pointer(methodptr *mpp,
1658                                                                                  methodptr entrypoint,
1659                                                                                  const char *kind)
1660 {
1661 #if !defined(NDEBUG)
1662         codeinfo       *oldcode;
1663         codeinfo       *newcode;
1664 #endif
1665
1666         DOLOG( printf("patch method pointer from: %p to %p\n",
1667                                   (void*) *mpp, (void*)entrypoint); );
1668
1669 #if !defined(NDEBUG)
1670         oldcode = code_get_codeinfo_for_pv(*mpp);
1671         newcode = code_get_codeinfo_for_pv(entrypoint);
1672
1673         DOLOG_SHORT( printf("\tpatch %s %p ", kind, (void*) oldcode);
1674                                  method_println(oldcode->m);
1675                                  printf("\t      with      %p ", (void*) newcode);
1676                                  method_println(newcode->m); );
1677
1678         assert(oldcode->m == newcode->m);
1679 #endif
1680
1681         /* write the new entrypoint */
1682
1683         *mpp = (methodptr) entrypoint;
1684 }
1685
1686
1687 /* replace_patch_class *********************************************************
1688
1689    Patch a method in the given class.
1690
1691    IN:
1692            vftbl............vftbl of the class
1693            m................the method to patch
1694            oldentrypoint....the old entrypoint to replace
1695            entrypoint.......the new entrypoint
1696
1697 *******************************************************************************/
1698
1699 void replace_patch_class(vftbl_t *vftbl,
1700                                                  methodinfo *m,
1701                                                  u1 *oldentrypoint,
1702                                                  u1 *entrypoint)
1703 {
1704         s4                 i;
1705         methodptr         *mpp;
1706         methodptr         *mppend;
1707
1708         /* patch the vftbl of the class */
1709
1710         replace_patch_method_pointer(vftbl->table + m->vftblindex,
1711                                                                  entrypoint,
1712                                                                  "virtual  ");
1713
1714         /* patch the interface tables */
1715
1716         assert(oldentrypoint);
1717
1718         for (i=0; i < vftbl->interfacetablelength; ++i) {
1719                 mpp = vftbl->interfacetable[-i];
1720                 mppend = mpp + vftbl->interfacevftbllength[i];
1721                 for (; mpp != mppend; ++mpp)
1722                         if (*mpp == oldentrypoint) {
1723                                 replace_patch_method_pointer(mpp, entrypoint, "interface");
1724                         }
1725         }
1726 }
1727
1728
1729 /* replace_patch_class_hierarchy ***********************************************
1730
1731    Patch a method in all loaded classes.
1732
1733    IN:
1734            m................the method to patch
1735            oldentrypoint....the old entrypoint to replace
1736            entrypoint.......the new entrypoint
1737
1738 *******************************************************************************/
1739
1740 struct replace_patch_data_t {
1741         methodinfo *m;
1742         u1         *oldentrypoint;
1743         u1         *entrypoint;
1744 };
1745
1746 void replace_patch_callback(classinfo *c, struct replace_patch_data_t *pd)
1747 {
1748         vftbl_t *vftbl = c->vftbl;
1749
1750         if (vftbl != NULL
1751                 && vftbl->vftbllength > pd->m->vftblindex
1752                 && vftbl->table[pd->m->vftblindex] != &asm_abstractmethoderror
1753                 && code_get_methodinfo_for_pv(vftbl->table[pd->m->vftblindex]) == pd->m)
1754         {
1755                 replace_patch_class(c->vftbl, pd->m, pd->oldentrypoint, pd->entrypoint);
1756         }
1757 }
1758
1759 void replace_patch_class_hierarchy(methodinfo *m,
1760                                                                    u1 *oldentrypoint,
1761                                                                    u1 *entrypoint)
1762 {
1763         struct replace_patch_data_t pd;
1764
1765         pd.m = m;
1766         pd.oldentrypoint = oldentrypoint;
1767         pd.entrypoint = entrypoint;
1768
1769         DOLOG_SHORT( printf("patching class hierarchy: ");
1770                              method_println(m); );
1771
1772         classcache_foreach_loaded_class(
1773                         (classcache_foreach_functionptr_t) &replace_patch_callback,
1774                         (void*) &pd);
1775 }
1776
1777
1778 /* replace_patch_future_calls **************************************************
1779
1780    Analyse a call site and depending on the kind of call patch the call, the
1781    virtual function table, or the interface table.
1782
1783    IN:
1784            ra...............return address pointing after the call site
1785            callerframe......source frame of the caller
1786            calleeframe......source frame of the callee, must have been mapped
1787
1788 *******************************************************************************/
1789
1790 void replace_patch_future_calls(u1 *ra,
1791                                                                 sourceframe_t *callerframe,
1792                                                                 sourceframe_t *calleeframe)
1793 {
1794         u1            *patchpos;
1795         methodptr      entrypoint;
1796         methodptr      oldentrypoint;
1797         bool           atentry;
1798         void          *pv;
1799         codeinfo      *calleecode;
1800         methodinfo    *calleem;
1801         java_object_t *obj;
1802         vftbl_t       *vftbl;
1803
1804         assert(ra);
1805         assert(callerframe->down == calleeframe);
1806
1807         /* get the new codeinfo and the method that shall be entered */
1808
1809         calleecode = calleeframe->tocode;
1810         assert(calleecode);
1811
1812         calleem = calleeframe->method;
1813         assert(calleem == calleecode->m);
1814
1815         entrypoint = (methodptr) calleecode->entrypoint;
1816
1817         /* check if we are at an method entry rplpoint at the innermost frame */
1818
1819         atentry = (calleeframe->down == NULL)
1820                         && !(calleem->flags & ACC_STATIC)
1821                         && (calleeframe->fromrp->id == 0); /* XXX */
1822
1823         /* get the position to patch, in case it was a statically bound call   */
1824
1825         pv = callerframe->fromcode->entrypoint;
1826         patchpos = md_jit_method_patch_address(pv, ra, NULL);
1827
1828         if (patchpos == NULL) {
1829                 /* the call was dispatched dynamically */
1830
1831                 /* we can only patch such calls if we are at the entry point */
1832
1833                 if (!atentry)
1834                         return;
1835
1836                 assert((calleem->flags & ACC_STATIC) == 0);
1837
1838                 oldentrypoint = calleeframe->fromcode->entrypoint;
1839
1840                 /* we need to know the instance */
1841
1842                 if (!calleeframe->instance.a) {
1843                         DOLOG_SHORT( printf("WARNING: object instance unknown!\n"); );
1844                         replace_patch_class_hierarchy(calleem, oldentrypoint, entrypoint);
1845                         return;
1846                 }
1847
1848                 /* get the vftbl */
1849
1850                 obj = calleeframe->instance.a;
1851                 vftbl = obj->vftbl;
1852
1853                 assert(vftbl->clazz->vftbl == vftbl);
1854
1855                 DOLOG_SHORT( printf("\tclass: "); class_println(vftbl->clazz); );
1856
1857                 replace_patch_class(vftbl, calleem, oldentrypoint, entrypoint);
1858         }
1859         else {
1860                 /* the call was statically bound */
1861
1862                 replace_patch_method_pointer((methodptr *) patchpos, entrypoint, "static   ");
1863         }
1864 }
1865
1866
1867 /* replace_push_activation_record **********************************************
1868
1869    Push a stack frame onto the execution state.
1870
1871    *** This function imitates the effects of a call and the ***
1872    *** method prolog of the callee.                         ***
1873
1874    IN:
1875            es...............execution state
1876            rpcall...........the replacement point at the call site
1877            callerframe......source frame of the caller, or NULL for creating the
1878                             first frame
1879            calleeframe......source frame of the callee, must have been mapped
1880
1881    OUT:
1882        *es..............the execution state after pushing the stack frame
1883
1884 *******************************************************************************/
1885
1886 void replace_push_activation_record(executionstate_t *es,
1887                                                                         rplpoint *rpcall,
1888                                                                         sourceframe_t *callerframe,
1889                                                                         sourceframe_t *calleeframe)
1890 {
1891         s4           reg;
1892         s4           i;
1893         s4           count;
1894         stackslot_t *basesp;
1895         stackslot_t *sp;
1896         u1          *ra;
1897         codeinfo    *calleecode;
1898
1899         assert(es);
1900         assert(!rpcall || callerframe);
1901     assert(!rpcall || rpcall->type == RPLPOINT_TYPE_CALL);
1902         assert(!rpcall || rpcall == callerframe->torp);
1903         assert(calleeframe);
1904         assert(!callerframe || calleeframe == callerframe->down);
1905
1906         /* the compilation unit we are entering */
1907
1908         calleecode = calleeframe->tocode;
1909         assert(calleecode);
1910
1911         /* calculate the return address */
1912
1913         if (rpcall)
1914                 ra = rpcall->pc + rpcall->callsize;
1915         else
1916                 ra = es->pc + 1 /* XXX this is ugly */;
1917
1918         /* write the return address */
1919
1920 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1921         es->sp -= SIZE_OF_STACKSLOT;
1922
1923         *((stackslot_t *)es->sp) = (stackslot_t) ra;
1924 #endif /* REPLACE_RA_BETWEEN_FRAMES */
1925
1926 #if defined(REPLACE_REG_RA)
1927         es->intregs[REPLACE_REG_RA] = (ptrint) ra;
1928 #endif
1929
1930         /* we move into a new code unit */
1931
1932         es->code = calleecode;
1933
1934         /* set the new pc XXX not needed? */
1935
1936         es->pc = calleecode->entrypoint;
1937
1938         /* build the stackframe */
1939
1940         DOLOG( printf("building stackframe of %d words at %p\n",
1941                                   calleecode->stackframesize, (void*)es->sp); );
1942
1943         sp = (stackslot_t *) es->sp;
1944         basesp = sp;
1945
1946         sp -= calleecode->stackframesize;
1947         es->sp = (u1*) sp;
1948
1949         /* in debug mode, invalidate stack frame first */
1950
1951         /* XXX may not invalidate linkage area used by native code! */
1952 #if !defined(NDEBUG) && 0
1953         for (i=0; i<(basesp - sp); ++i) {
1954                 sp[i] = 0xdeaddeadU;
1955         }
1956 #endif
1957
1958         /* save the return address register */
1959
1960 #if defined(REPLACE_RA_TOP_OF_FRAME)
1961 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1962         if (!code_is_leafmethod(calleecode))
1963 #endif
1964                 *--basesp = (ptrint) ra;
1965 #endif /* REPLACE_RA_TOP_OF_FRAME */
1966
1967 #if defined(REPLACE_RA_LINKAGE_AREA)
1968 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1969         if (!code_is_leafmethod(calleecode))
1970 #endif
1971                 basesp[LA_LR_OFFSET / sizeof(stackslot_t)] = (ptrint) ra;
1972 #endif /* REPLACE_RA_LINKAGE_AREA */
1973
1974         /* save int registers */
1975
1976         reg = INT_REG_CNT;
1977         for (i=0; i<calleecode->savedintcount; ++i) {
1978                 while (nregdescint[--reg] != REG_SAV)
1979                         ;
1980                 *--basesp = es->intregs[reg];
1981
1982                 /* XXX may not clobber saved regs used by native code! */
1983 #if !defined(NDEBUG) && 0
1984                 es->intregs[reg] = (ptrint) 0x44dead4444dead44ULL;
1985 #endif
1986         }
1987
1988         /* save flt registers */
1989
1990         /* XXX align? */
1991         reg = FLT_REG_CNT;
1992         for (i=0; i<calleecode->savedfltcount; ++i) {
1993                 while (nregdescfloat[--reg] != REG_SAV)
1994                         ;
1995                 basesp -= STACK_SLOTS_PER_FLOAT;
1996                 *(double*)basesp = es->fltregs[reg];
1997
1998                 /* XXX may not clobber saved regs used by native code! */
1999 #if !defined(NDEBUG) && 0
2000                 *(u8*)&(es->fltregs[reg]) = 0x44dead4444dead44ULL;
2001 #endif
2002         }
2003
2004 #if defined(HAS_ADDRESS_REGISTER_FILE)
2005         /* save adr registers */
2006
2007         reg = ADR_REG_CNT;
2008         for (i=0; i<calleecode->savedadrcount; ++i) {
2009                 while (nregdescadr[--reg] != REG_SAV)
2010                         ;
2011                 *--basesp = es->adrregs[reg];
2012
2013                 /* XXX may not clobber saved regs used by native code! */
2014 #if !defined(NDEBUG) && 0
2015                 es->adrregs[reg] = (ptrint) 0x44dead4444dead44ULL;
2016 #endif
2017         }
2018 #endif
2019
2020         /* write slots used for synchronization */
2021
2022         count = code_get_sync_slot_count(calleecode);
2023         assert(count == calleeframe->syncslotcount);
2024         for (i=0; i<count; ++i) {
2025                 sp[calleecode->memuse + i] = calleeframe->syncslots[i].p;
2026         }
2027
2028         /* set the PV */
2029
2030         es->pv = calleecode->entrypoint;
2031
2032         /* redirect future invocations */
2033
2034         if (callerframe && rpcall) {
2035 #if defined(REPLACE_PATCH_ALL)
2036                 if (rpcall->type == callerframe->fromrp->type)
2037 #else
2038                 if (rpcall == callerframe->fromrp)
2039 #endif
2040                         replace_patch_future_calls(ra, callerframe, calleeframe);
2041         }
2042 }
2043
2044
2045 /* replace_find_replacement_point **********************************************
2046
2047    Find the replacement point in the given code corresponding to the
2048    position given in the source frame.
2049
2050    IN:
2051            code.............the codeinfo in which to search the rplpoint
2052            frame............the source frame defining the position to look for
2053            parent...........parent replacement point to match
2054
2055    RETURN VALUE:
2056        the replacement point
2057
2058 *******************************************************************************/
2059
2060 rplpoint * replace_find_replacement_point(codeinfo *code,
2061                                                                                   sourceframe_t *frame,
2062                                                                                   rplpoint *parent)
2063 {
2064         methodinfo *m;
2065         rplpoint *rp;
2066         s4        i;
2067         s4        j;
2068         s4        stacki;
2069         rplalloc *ra;
2070
2071         assert(code);
2072         assert(frame);
2073
2074         DOLOG( printf("searching replacement point for:\n");
2075                    replace_source_frame_println(frame); );
2076
2077         m = frame->method;
2078
2079         DOLOG( printf("code = %p\n", (void*)code); );
2080
2081         rp = code->rplpoints;
2082         i = code->rplpointcount;
2083         while (i--) {
2084                 if (rp->id == frame->id && rp->method == frame->method
2085                                 && rp->parent == parent
2086                                 && replace_normalize_type_map[rp->type] == frame->type)
2087                 {
2088                         /* check if returnAddresses match */
2089                         /* XXX optimize: only do this if JSRs in method */
2090                         DOLOG( printf("checking match for:");
2091                                    replace_replacement_point_println(rp, 1); fflush(stdout); );
2092                         ra = rp->regalloc;
2093                         stacki = 0;
2094                         for (j = rp->regalloccount; j--; ++ra) {
2095                                 if (ra->type == TYPE_RET) {
2096                                         if (ra->index == RPLALLOC_STACK) {
2097                                                 assert(stacki < frame->javastackdepth);
2098                                                 if (frame->javastack[stacki].i != ra->regoff)
2099                                                         goto no_match;
2100                                                 stacki++;
2101                                         }
2102                                         else {
2103                                                 assert(ra->index >= 0 && ra->index < frame->javalocalcount);
2104                                                 if (frame->javalocals[ra->index].i != ra->regoff)
2105                                                         goto no_match;
2106                                         }
2107                                 }
2108                         }
2109
2110                         /* found */
2111                         return rp;
2112                 }
2113 no_match:
2114                 rp++;
2115         }
2116
2117 #if !defined(NDEBUG)
2118         printf("candidate replacement points were:\n");
2119         rp = code->rplpoints;
2120         i = code->rplpointcount;
2121         for (; i--; ++rp) {
2122                 replace_replacement_point_println(rp, 1);
2123         }
2124 #endif
2125
2126         vm_abort("no matching replacement point found");
2127         return NULL; /* NOT REACHED */
2128 }
2129
2130
2131 /* replace_find_replacement_point_for_pc ***************************************
2132
2133    Find the nearest replacement point at or before the given PC. The
2134    given PC has to be between (rp->pc) and (rp->pc+rp->callsize) for
2135    the replacement point to be found.
2136
2137    IN:
2138        code.............compilation unit the PC is in
2139            pc...............the machine code PC
2140
2141    RETURN VALUE:
2142        the replacement point found, or
2143            NULL if no replacement point was found
2144
2145 *******************************************************************************/
2146
2147 rplpoint *replace_find_replacement_point_for_pc(codeinfo *code, u1 *pc)
2148 {
2149         rplpoint *found;
2150         rplpoint *rp;
2151         s4        i;
2152
2153         DOLOG( printf("searching for rp at pc:%p in %p ", (void*)pc, (void*)code);
2154                    method_println(code->m); );
2155
2156         found = NULL;
2157
2158         rp = code->rplpoints;
2159         for (i=0; i<code->rplpointcount; ++i, ++rp) {
2160                 DOLOG( replace_replacement_point_println(rp, 2); );
2161                 if (rp->pc <= pc && rp->pc + rp->callsize >= pc)
2162                         found = rp;
2163         }
2164
2165         return found;
2166 }
2167
2168
2169 /* replace_pop_native_frame ****************************************************
2170
2171    Unroll a native frame in the execution state and create a source frame
2172    for it.
2173
2174    IN:
2175            es...............current execution state
2176            ss...............the current source state
2177            sfi..............stackframeinfo for the native frame
2178
2179    OUT:
2180        es...............execution state after unrolling the native frame
2181            ss...............gets the added native source frame
2182
2183 *******************************************************************************/
2184
2185 static void replace_pop_native_frame(executionstate_t *es,
2186                                                                          sourcestate_t *ss,
2187                                                                          stackframeinfo_t *sfi)
2188 {
2189         sourceframe_t *frame;
2190         codeinfo      *code;
2191         s4             i,j;
2192
2193         assert(sfi);
2194
2195         frame = replace_new_sourceframe(ss);
2196
2197         frame->sfi = sfi;
2198
2199         /* remember pc and size of native frame */
2200
2201         frame->nativepc = es->pc;
2202         frame->nativeframesize = (es->sp != 0) ? (sfi->sp - es->sp) : 0;
2203         assert(frame->nativeframesize >= 0);
2204
2205         /* remember values of saved registers */
2206
2207         j = 0;
2208         for (i=0; i<INT_REG_CNT; ++i) {
2209                 if (nregdescint[i] == REG_SAV)
2210                         frame->nativesavint[j++] = es->intregs[i];
2211         }
2212
2213         j = 0;
2214         for (i=0; i<FLT_REG_CNT; ++i) {
2215                 if (nregdescfloat[i] == REG_SAV)
2216                         frame->nativesavflt[j++] = es->fltregs[i];
2217         }
2218
2219 #if defined(HAS_ADDRESS_REGISTER_FILE)
2220         j = 0;
2221         for (i=0; i<ADR_REG_CNT; ++i) {
2222                 if (nregdescadr[i] == REG_SAV)
2223                         frame->nativesavadr[j++] = es->adrregs[i];
2224         }
2225 #endif
2226
2227         /* restore saved registers */
2228
2229 #if defined(ENABLE_GC_CACAO) && !defined(HAS_ADDRESS_REGISTER_FILE)
2230         j = 0;
2231         for (i=0; i<INT_REG_CNT; ++i) {
2232                 if (nregdescint[i] == REG_SAV)
2233                         es->intregs[i] = sfi->intregs[j++];
2234         }
2235 #else
2236         /* XXX we don't have them, yet, in the sfi, so clear them */
2237
2238         for (i=0; i<INT_REG_CNT; ++i) {
2239                 if (nregdescint[i] == REG_SAV)
2240                         es->intregs[i] = 0;
2241         }
2242 #endif
2243
2244         /* XXX we don't have float registers in the sfi, so clear them */
2245
2246         for (i=0; i<FLT_REG_CNT; ++i) {
2247                 if (nregdescfloat[i] == REG_SAV)
2248                         es->fltregs[i] = 0.0;
2249         }
2250
2251 #if defined(HAS_ADDRESS_REGISTER_FILE)
2252 # if defined(ENABLE_GC_CACAO)
2253         j = 0;
2254         for (i=0; i<ADR_REG_CNT; ++i) {
2255                 if (nregdescadr[i] == REG_SAV)
2256                         es->adrregs[i] = sfi->adrregs[j++];
2257         }
2258 # else
2259         for (i=0; i<ADR_REG_CNT; ++i) {
2260                 if (nregdescadr[i] == REG_SAV)
2261                         es->adrregs[i] = 0;
2262         }
2263 # endif
2264 #endif
2265
2266         /* restore codeinfo of the native stub */
2267
2268         code = code_get_codeinfo_for_pv(sfi->pv);
2269
2270         /* restore sp, pv, pc and codeinfo of the parent method */
2271
2272         /* XXX michi: use this instead:
2273         es->sp = sfi->sp + code->stackframesize; */
2274         es->sp   = sfi->sp + (*(s4 *) (sfi->pv + FrameSize));
2275 #if defined(REPLACE_RA_BETWEEN_FRAMES)
2276         es->sp  += SIZE_OF_STACKSLOT; /* skip return address */
2277 #endif
2278         es->pv   = md_codegen_get_pv_from_pc(sfi->ra);
2279         es->pc   = ((sfi->xpc) ? sfi->xpc : sfi->ra) - 1;
2280         es->code = code_get_codeinfo_for_pv(es->pv);
2281 }
2282
2283
2284 /* replace_push_native_frame ***************************************************
2285
2286    Rebuild a native frame onto the execution state and remove its source frame.
2287
2288    Note: The native frame is "rebuild" by setting fields like PC and stack
2289          pointer in the execution state accordingly. Values in the
2290                  stackframeinfo may be modified, but the actual stack frame of the
2291                  native code is not touched.
2292
2293    IN:
2294            es...............current execution state
2295            ss...............the current source state
2296
2297    OUT:
2298        es...............execution state after re-rolling the native frame
2299            ss...............the native source frame is removed
2300
2301 *******************************************************************************/
2302
2303 static void replace_push_native_frame(executionstate_t *es, sourcestate_t *ss)
2304 {
2305         sourceframe_t *frame;
2306         s4             i,j;
2307
2308         assert(es);
2309         assert(ss);
2310
2311         DOLOG( printf("pushing native frame\n"); );
2312
2313         /* remove the frame from the source state */
2314
2315         frame = ss->frames;
2316         assert(frame);
2317         assert(REPLACE_IS_NATIVE_FRAME(frame));
2318
2319         ss->frames = frame->down;
2320
2321         /* skip sp for the native stub */
2322
2323         es->sp -= (*(s4 *) (frame->sfi->pv + FrameSize));
2324 #if defined(REPLACE_RA_BETWEEN_FRAMES)
2325         es->sp -= SIZE_OF_STACKSLOT; /* skip return address */
2326 #endif
2327
2328         /* assert that the native frame has not moved */
2329
2330         assert(es->sp == frame->sfi->sp);
2331
2332         /* update saved registers in the stackframeinfo */
2333
2334 #if defined(ENABLE_GC_CACAO)
2335         j = 0;
2336 # if !defined(HAS_ADDRESS_REGISTER_FILE)
2337         for (i=0; i<INT_REG_CNT; ++i) {
2338                 if (nregdescint[i] == REG_SAV)
2339                         frame->sfi->intregs[j++] = es->intregs[i];
2340         }
2341 # else
2342         for (i=0; i<ADR_REG_CNT; ++i) {
2343                 if (nregdescadr[i] == REG_SAV)
2344                         frame->sfi->adrregs[j++] = es->adrregs[i];
2345         }
2346 # endif
2347
2348         /* XXX leave float registers untouched here */
2349 #endif
2350
2351         /* restore saved registers */
2352
2353         j = 0;
2354         for (i=0; i<INT_REG_CNT; ++i) {
2355                 if (nregdescint[i] == REG_SAV)
2356                         es->intregs[i] = frame->nativesavint[j++];
2357         }
2358
2359         j = 0;
2360         for (i=0; i<FLT_REG_CNT; ++i) {
2361                 if (nregdescfloat[i] == REG_SAV)
2362                         es->fltregs[i] = frame->nativesavflt[j++];
2363         }
2364
2365 #if defined(HAS_ADDRESS_REGISTER_FILE)
2366         j = 0;
2367         for (i=0; i<ADR_REG_CNT; ++i) {
2368                 if (nregdescadr[i] == REG_SAV)
2369                         es->adrregs[i] = frame->nativesavadr[j++];
2370         }
2371 #endif
2372
2373         /* skip the native frame on the machine stack */
2374
2375         es->sp -= frame->nativeframesize;
2376
2377         /* set the pc the next frame must return to */
2378
2379         es->pc = frame->nativepc;
2380 }
2381
2382
2383 /* replace_recover_source_state ************************************************
2384
2385    Recover the source state from the given replacement point and execution
2386    state.
2387
2388    IN:
2389        rp...............replacement point that has been reached, if any
2390            sfi..............stackframeinfo, if called from native code
2391            es...............execution state at the replacement point rp
2392
2393    RETURN VALUE:
2394        the source state
2395
2396 *******************************************************************************/
2397
2398 sourcestate_t *replace_recover_source_state(rplpoint *rp,
2399                                                                                         stackframeinfo_t *sfi,
2400                                                                                     executionstate_t *es)
2401 {
2402         sourcestate_t *ss;
2403         u1            *ra;
2404         bool           locked;
2405 #if defined(REPLACE_STATISTICS)
2406         s4             depth;
2407 #endif
2408
2409         /* create the source frame structure in dump memory */
2410
2411         ss = DNEW(sourcestate_t);
2412         ss->frames = NULL;
2413
2414         /* each iteration of the loop recovers one source frame */
2415
2416         depth = 0;
2417         locked = false;
2418
2419         while (rp || sfi) {
2420
2421                 DOLOG( executionstate_println(es); );
2422
2423                 /* if we are not at a replacement point, it is a native frame */
2424
2425                 if (rp == NULL) {
2426                         DOLOG( printf("native frame: sfi: "); replace_stackframeinfo_println(sfi); );
2427
2428                         locked = true;
2429                         replace_pop_native_frame(es, ss, sfi);
2430                         sfi = sfi->prev;
2431
2432                         if (es->code == NULL)
2433                                 continue;
2434
2435                         goto after_machine_frame;
2436                 }
2437
2438                 /* read the values for this source frame from the execution state */
2439
2440                 DOLOG( printf("recovering source state for%s:\n",
2441                                         (ss->frames == NULL) ? " TOPFRAME" : "");
2442                            replace_replacement_point_println(rp, 1); );
2443
2444                 replace_read_executionstate(rp, es, ss, ss->frames == NULL);
2445
2446 #if defined(ENABLE_VMLOG)
2447                 vmlog_cacao_unrol_method(ss->frames->method);
2448 #endif
2449
2450 #if defined(REPLACE_STATISTICS)
2451                 REPLACE_COUNT(stat_frames);
2452                 depth++;
2453                 replace_statistics_source_frame(ss->frames);
2454 #endif
2455
2456                 /* in locked areas (below native frames), identity map the frame */
2457
2458                 if (locked) {
2459                         ss->frames->torp = ss->frames->fromrp;
2460                         ss->frames->tocode = ss->frames->fromcode;
2461                 }
2462
2463                 /* unroll to the next (outer) frame */
2464
2465                 if (rp->parent) {
2466                         /* this frame is in inlined code */
2467
2468                         DOLOG( printf("INLINED!\n"); );
2469
2470                         rp = rp->parent;
2471
2472                         assert(rp->type == RPLPOINT_TYPE_INLINE);
2473                         REPLACE_COUNT(stat_unroll_inline);
2474                 }
2475                 else {
2476                         /* this frame had been called at machine-level. pop it. */
2477
2478                         DOLOG( printf("UNWIND\n"); );
2479
2480                         ra = replace_pop_activation_record(es, ss->frames);
2481                         if (ra == NULL) {
2482                                 DOLOG( printf("REACHED NATIVE CODE\n"); );
2483
2484                                 rp = NULL;
2485
2486 #if !defined(ENABLE_GC_CACAO)
2487                                 break; /* XXX remove to activate native frames */
2488 #endif
2489                                 continue;
2490                         }
2491
2492                         /* find the replacement point at the call site */
2493
2494 after_machine_frame:
2495                         rp = replace_find_replacement_point_for_pc(es->code, es->pc);
2496
2497                         if (rp == NULL)
2498                                 vm_abort("could not find replacement point while unrolling call");
2499
2500                         DOLOG( printf("found replacement point.\n");
2501                                         replace_replacement_point_println(rp, 1); );
2502
2503                         assert(rp->type == RPLPOINT_TYPE_CALL);
2504                         REPLACE_COUNT(stat_unroll_call);
2505                 }
2506         } /* end loop over source frames */
2507
2508         REPLACE_COUNT_DIST(stat_dist_frames, depth);
2509
2510         return ss;
2511 }
2512
2513
2514 /* replace_map_source_state ****************************************************
2515
2516    Map each source frame in the given source state to a target replacement
2517    point and compilation unit. If no valid code is available for a source
2518    frame, it is (re)compiled.
2519
2520    IN:
2521        ss...............the source state
2522
2523    OUT:
2524        ss...............the source state, modified: The `torp` and `tocode`
2525                             fields of each source frame are set.
2526
2527    RETURN VALUE:
2528        true.............everything went ok
2529            false............an exception has been thrown
2530
2531 *******************************************************************************/
2532
2533 static bool replace_map_source_state(sourcestate_t *ss)
2534 {
2535         sourceframe_t *frame;
2536         codeinfo      *code;
2537         rplpoint      *rp;
2538         rplpoint      *parent; /* parent of inlined rplpoint */
2539 #if defined(REPLACE_STATISTICS)
2540         codeinfo      *oldcode;
2541 #endif
2542
2543         parent = NULL;
2544         code = NULL;
2545
2546         /* iterate over the source frames from outermost to innermost */
2547
2548         for (frame = ss->frames; frame != NULL; frame = frame->down) {
2549
2550                 /* XXX skip native frames */
2551
2552                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
2553                         parent = NULL;
2554                         continue;
2555                 }
2556
2557                 /* map frames which are not already mapped */
2558
2559                 if (frame->tocode) {
2560                         code = frame->tocode;
2561                         rp = frame->torp;
2562                         assert(rp);
2563                 }
2564                 else {
2565                         assert(frame->torp == NULL);
2566
2567                         if (parent == NULL) {
2568                                 /* find code for this frame */
2569
2570 #if defined(REPLACE_STATISTICS)
2571                                 oldcode = frame->method->code;
2572 #endif
2573                                 /* request optimization of hot methods and their callers */
2574
2575                                 if (frame->method->hitcountdown < 0
2576                                         || (frame->down && frame->down->method->hitcountdown < 0))
2577                                         jit_request_optimization(frame->method);
2578
2579                                 code = jit_get_current_code(frame->method);
2580
2581                                 if (code == NULL)
2582                                         return false; /* exception */
2583
2584                                 REPLACE_COUNT_IF(stat_recompile, code != oldcode);
2585                         }
2586
2587                         assert(code);
2588
2589                         /* map this frame */
2590
2591                         rp = replace_find_replacement_point(code, frame, parent);
2592
2593                         frame->tocode = code;
2594                         frame->torp = rp;
2595                 }
2596
2597                 if (rp->type == RPLPOINT_TYPE_CALL) {
2598                         parent = NULL;
2599                 }
2600                 else {
2601                         /* inlining */
2602                         parent = rp;
2603                 }
2604         }
2605
2606         return true;
2607 }
2608
2609
2610 /* replace_map_source_state_identity *******************************************
2611
2612    Map each source frame in the given source state to the same replacement
2613    point and compilation unit it was derived from. This is mainly used for
2614    garbage collection.
2615
2616    IN:
2617        ss...............the source state
2618
2619    OUT:
2620        ss...............the source state, modified: The `torp` and `tocode`
2621                             fields of each source frame are set.
2622
2623 *******************************************************************************/
2624
2625 #if defined(ENABLE_GC_CACAO)
2626 static void replace_map_source_state_identity(sourcestate_t *ss)
2627 {
2628         sourceframe_t *frame;
2629
2630         /* iterate over the source frames from outermost to innermost */
2631
2632         for (frame = ss->frames; frame != NULL; frame = frame->down) {
2633
2634                 /* skip native frames */
2635
2636                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
2637                         continue;
2638                 }
2639
2640                 /* map frames using the identity mapping */
2641
2642                 if (frame->tocode) {
2643                         assert(frame->tocode == frame->fromcode);
2644                         assert(frame->torp   == frame->fromrp);
2645                 } else {
2646                         assert(frame->tocode == NULL);
2647                         assert(frame->torp   == NULL);
2648                         frame->tocode = frame->fromcode;
2649                         frame->torp   = frame->fromrp;
2650                 }
2651         }
2652 }
2653 #endif
2654
2655
2656 /* replace_build_execution_state ***********************************************
2657
2658    Build an execution state for the given (mapped) source state.
2659
2660    !!! CAUTION: This function rewrites the machine stack !!!
2661
2662    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2663
2664    IN:
2665        ss...............the source state. Must have been mapped by
2666                                                 replace_map_source_state before.
2667            es...............the base execution state on which to build
2668
2669    OUT:
2670        *es..............the new execution state
2671
2672 *******************************************************************************/
2673
2674 static void replace_build_execution_state(sourcestate_t *ss,
2675                                                                                   executionstate_t *es)
2676 {
2677         rplpoint      *rp;
2678         sourceframe_t *prevframe;
2679         rplpoint      *parent;
2680
2681         parent = NULL;
2682         prevframe = NULL;
2683         rp = NULL;
2684
2685         while (ss->frames) {
2686
2687                 if (REPLACE_IS_NATIVE_FRAME(ss->frames)) {
2688                         prevframe = ss->frames;
2689                         replace_push_native_frame(es, ss);
2690                         parent = NULL;
2691                         rp = NULL;
2692                         continue;
2693                 }
2694
2695                 if (parent == NULL) {
2696                         /* create a machine-level stack frame */
2697
2698                         DOLOG( printf("pushing activation record for:\n");
2699                                    if (rp) replace_replacement_point_println(rp, 1);
2700                                    else printf("\tfirst frame\n"); );
2701
2702                         replace_push_activation_record(es, rp, prevframe, ss->frames);
2703
2704                         DOLOG( executionstate_println(es); );
2705                 }
2706
2707                 rp = ss->frames->torp;
2708                 assert(rp);
2709
2710                 DOLOG( printf("creating execution state for%s:\n",
2711                                 (ss->frames->down == NULL) ? " TOPFRAME" : "");
2712                            replace_replacement_point_println(ss->frames->fromrp, 1);
2713                            replace_replacement_point_println(rp, 1); );
2714
2715                 es->code = ss->frames->tocode;
2716                 prevframe = ss->frames;
2717
2718 #if defined(ENABLE_VMLOG)
2719                 vmlog_cacao_rerol_method(ss->frames->method);
2720 #endif
2721
2722                 replace_write_executionstate(rp, es, ss, ss->frames->down == NULL);
2723
2724                 DOLOG( executionstate_println(es); );
2725
2726                 if (rp->type == RPLPOINT_TYPE_CALL) {
2727                         parent = NULL;
2728                 }
2729                 else {
2730                         /* inlining */
2731                         parent = rp;
2732                 }
2733         }
2734 }
2735
2736
2737 /* replace_me ******************************************************************
2738
2739    This function is called by the signal handler when a thread reaches
2740    a replacement point. `replace_me` must map the execution state to the
2741    target replacement point and let execution continue there.
2742
2743    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2744
2745    IN:
2746        rp...............replacement point that has been reached
2747        es...............execution state read by signal handler
2748
2749 *******************************************************************************/
2750
2751 static void replace_me(rplpoint *rp, executionstate_t *es)
2752 {
2753         stackframeinfo_t    *sfi;
2754         sourcestate_t       *ss;
2755         sourceframe_t       *frame;
2756         codeinfo            *origcode;
2757         rplpoint            *origrp;
2758 #if defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)
2759         threadobject        *thread;
2760 #endif
2761         int32_t              dumpmarker;
2762
2763         origcode = es->code;
2764         origrp   = rp;
2765
2766         DOLOG_SHORT( printf("REPLACING(%d %p): (id %d %p) ",
2767                                  stat_replacements, (void*)THREADOBJECT,
2768                                  rp->id, (void*)rp);
2769                                  method_println(es->code->m); );
2770
2771         DOLOG( replace_replacement_point_println(rp, 1); );
2772
2773         REPLACE_COUNT(stat_replacements);
2774
2775         /* mark start of dump memory area */
2776
2777         DMARKER;
2778
2779         /* Get the stackframeinfo for the current thread. */
2780
2781         sfi = threads_get_current_stackframeinfo();
2782
2783         /* recover source state */
2784
2785         ss = replace_recover_source_state(rp, sfi, es);
2786
2787 #if defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)
2788         /* if there is a collection pending, we assume the replacement point should
2789            suspend this thread */
2790
2791         if (gc_pending) {
2792
2793                 thread = THREADOBJECT;
2794
2795                 DOLOG_SHORT( printf("REPLACEMENT: Suspending thread for GC now!\n"); );
2796
2797                 /* map the sourcestate using the identity mapping */
2798                 replace_map_source_state_identity(ss);
2799
2800                 /* since we enter the same method again, we turn off rps now */
2801                 /* XXX michi: can we really do this? what if the rp was active before
2802                    we activated it for the gc? */
2803                 replace_deactivate_replacement_points(origcode);
2804
2805                 /* remember executionstate and sourcestate for this thread */
2806                 GC_EXECUTIONSTATE = es;
2807                 GC_SOURCESTATE    = ss;
2808
2809                 /* really suspend this thread now (PC = 0) */
2810                 threads_suspend_ack(NULL, NULL);
2811
2812                 DOLOG_SHORT( printf("REPLACEMENT: Resuming thread after GC now!\n"); );
2813
2814         } else {
2815 #endif /*defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)*/
2816
2817         /* map the source state */
2818
2819         if (!replace_map_source_state(ss))
2820                 vm_abort("exception during method replacement");
2821
2822         DOLOG( replace_sourcestate_println(ss); );
2823
2824         DOLOG_SHORT( replace_sourcestate_println_short(ss); );
2825
2826 #if !defined(NDEBUG)
2827         /* avoid infinite loops by self-replacement, only if not in testing mode */
2828
2829         if (!opt_TestReplacement) {
2830                 frame = ss->frames;
2831                 while (frame->down)
2832                         frame = frame->down;
2833
2834                 if (frame->torp == origrp) {
2835                         DOLOG_SHORT(
2836                                 printf("WARNING: identity replacement, turning off rps to avoid infinite loop\n");
2837                         );
2838                         replace_deactivate_replacement_points(origcode);
2839                 }
2840         }
2841 #endif
2842
2843 #if defined(ENABLE_THREADS) && defined(ENABLE_GC_CACAO)
2844         }
2845 #endif
2846
2847         /* build the new execution state */
2848
2849         replace_build_execution_state(ss, es);
2850
2851 #if !defined(NDEBUG)
2852         /* continue execution after patched machine code, if testing mode enabled */
2853
2854         if (opt_TestReplacement)
2855                 es->pc += REPLACEMENT_PATCH_SIZE;
2856 #endif
2857
2858         /* release dump area */
2859
2860         DRELEASE;
2861 }
2862
2863
2864 /* replace_me_wrapper **********************************************************
2865
2866    This function is called by the signal handler. It determines if there
2867    is an active replacement point pending at the given PC and returns
2868    accordingly.
2869
2870    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2871
2872    IN:
2873        pc...............the program counter that triggered the replacement.
2874        context..........the context (machine state) to which the
2875                             replacement should be applied.
2876
2877    OUT:
2878        context..........the context after replacement finished.
2879
2880    RETURN VALUE:
2881        true.............replacement done, everything went ok
2882        false............no replacement done, context unchanged
2883
2884 *******************************************************************************/
2885
2886 bool replace_me_wrapper(u1 *pc, void *context)
2887 {
2888         codeinfo         *code;
2889         rplpoint         *rp;
2890         executionstate_t  es;
2891
2892         /* search the codeinfo for the given PC */
2893
2894         code = code_find_codeinfo_for_pc(pc);
2895         assert(code);
2896
2897         /* search for a replacement point at the given PC */
2898
2899         rp = replace_find_replacement_point_for_pc(code, pc);
2900
2901         /* check if the replacement point belongs to given PC and is active */
2902
2903         if ((rp != NULL) && (rp->pc == pc) && (rp->flags & RPLPOINT_FLAG_ACTIVE)) {
2904
2905 #if !defined(NDEBUG)
2906                 executionstate_sanity_check(context);
2907 #endif
2908
2909                 /* set codeinfo pointer in execution state */
2910
2911                 es.code = code;
2912
2913                 /* read execution state from current context */
2914
2915                 md_executionstate_read(&es, context);
2916
2917                 DOLOG( printf("REPLACEMENT READ: ");
2918                            executionstate_println(&es); );
2919
2920                 /* do the actual replacement */
2921
2922                 replace_me(rp, &es);
2923
2924                 /* write execution state to current context */
2925
2926                 md_executionstate_write(&es, context);
2927
2928                 DOLOG( printf("REPLACEMENT WRITE: ");
2929                            executionstate_println(&es); );
2930
2931                 /* new code is entered after returning */
2932
2933                 DOLOG( printf("JUMPING IN!\n"); fflush(stdout); );
2934                 return true;
2935         }
2936         else
2937                 return false;
2938 }
2939
2940
2941 /******************************************************************************/
2942 /* NOTE: Stuff specific to the exact GC is below.                             */
2943 /******************************************************************************/
2944
2945 #if defined(ENABLE_GC_CACAO)
2946 void replace_gc_from_native(threadobject *thread, u1 *pc, u1 *sp)
2947 {
2948         stackframeinfo_t *sfi;
2949         executionstate_t *es;
2950         sourcestate_t    *ss;
2951
2952         /* Get the stackframeinfo of this thread. */
2953
2954         assert(thread == THREADOBJECT);
2955
2956         sfi = threads_get_current_stackframeinfo();
2957
2958         /* create the execution state */
2959         es = DNEW(executionstate_t);
2960         es->pc = pc;
2961         es->sp = sp;
2962         es->pv = 0;      /* since we are in a native, PV is invalid! */
2963         es->code = NULL; /* since we are in a native, we do not have a codeinfo */
2964
2965         /* we assume we are in a native (no replacement point)! */
2966         ss = replace_recover_source_state(NULL, sfi, es);
2967
2968         /* map the sourcestate using the identity mapping */
2969         replace_map_source_state_identity(ss);
2970
2971         /* remember executionstate and sourcestate for this thread */
2972         GC_EXECUTIONSTATE = es;
2973         GC_SOURCESTATE    = ss;
2974 }
2975 #endif
2976
2977 #if defined(ENABLE_GC_CACAO)
2978 void replace_gc_into_native(threadobject *thread)
2979 {
2980         executionstate_t *es;
2981         sourcestate_t    *ss;
2982
2983         /* get the executionstate and sourcestate for the given thread */
2984         es = GC_EXECUTIONSTATE;
2985         ss = GC_SOURCESTATE;
2986
2987         /* rebuild the stack of the given thread */
2988         replace_build_execution_state(ss, es);
2989 }
2990 #endif
2991
2992
2993 /******************************************************************************/
2994 /* NOTE: No important code below.                                             */
2995 /******************************************************************************/
2996
2997
2998 /* statistics *****************************************************************/
2999
3000 #if defined(REPLACE_STATISTICS)
3001 static void print_freq(FILE *file,int *array,int limit)
3002 {
3003         int i;
3004         int sum = 0;
3005         int cum = 0;
3006         for (i=0; i<limit; ++i)
3007                 sum += array[i];
3008         sum += array[limit];
3009         for (i=0; i<limit; ++i) {
3010                 cum += array[i];
3011                 fprintf(file,"      %3d: %8d (cum %3d%%)\n",
3012                                 i, array[i], (sum) ? ((100*cum)/sum) : 0);
3013         }
3014         fprintf(file,"    >=%3d: %8d\n",limit,array[limit]);
3015 }
3016 #endif /* defined(REPLACE_STATISTICS) */
3017
3018
3019 #if defined(REPLACE_STATISTICS)
3020
3021 #define REPLACE_PRINT_DIST(name, array)                              \
3022     printf("    " name " distribution:\n");                          \
3023     print_freq(stdout, (array), sizeof(array)/sizeof(int) - 1);
3024
3025 void replace_print_statistics(void)
3026 {
3027         printf("replacement statistics:\n");
3028         printf("    # of replacements:   %d\n", stat_replacements);
3029         printf("    # of frames:         %d\n", stat_frames);
3030         printf("    # of recompilations: %d\n", stat_recompile);
3031         printf("    patched static calls:%d\n", stat_staticpatch);
3032         printf("    unrolled inlines:    %d\n", stat_unroll_inline);
3033         printf("    unrolled calls:      %d\n", stat_unroll_call);
3034         REPLACE_PRINT_DIST("frame depth", stat_dist_frames);
3035         REPLACE_PRINT_DIST("locals per frame", stat_dist_locals);
3036         REPLACE_PRINT_DIST("ADR locals per frame", stat_dist_locals_adr);
3037         REPLACE_PRINT_DIST("primitive locals per frame", stat_dist_locals_prim);
3038         REPLACE_PRINT_DIST("RET locals per frame", stat_dist_locals_ret);
3039         REPLACE_PRINT_DIST("void locals per frame", stat_dist_locals_void);
3040         REPLACE_PRINT_DIST("stack slots per frame", stat_dist_stack);
3041         REPLACE_PRINT_DIST("ADR stack slots per frame", stat_dist_stack_adr);
3042         REPLACE_PRINT_DIST("primitive stack slots per frame", stat_dist_stack_prim);
3043         REPLACE_PRINT_DIST("RET stack slots per frame", stat_dist_stack_ret);
3044         printf("\n");
3045         printf("    # of methods:            %d\n", stat_methods);
3046         printf("    # of replacement points: %d\n", stat_rploints);
3047         printf("    # of regallocs:          %d\n", stat_regallocs);
3048         printf("        per rplpoint:        %f\n", (double)stat_regallocs / stat_rploints);
3049         printf("        per method:          %f\n", (double)stat_regallocs / stat_methods);
3050         REPLACE_PRINT_DIST("replacement points per method", stat_dist_method_rplpoints);
3051         printf("\n");
3052
3053 }
3054 #endif /* defined(REPLACE_STATISTICS) */
3055
3056
3057 #if defined(REPLACE_STATISTICS)
3058 static void replace_statistics_source_frame(sourceframe_t *frame)
3059 {
3060         int adr = 0;
3061         int ret = 0;
3062         int prim = 0;
3063         int vd = 0;
3064         int n = 0;
3065         int i;
3066
3067         for (i=0; i<frame->javalocalcount; ++i) {
3068                 switch (frame->javalocaltype[i]) {
3069                         case TYPE_ADR: adr++; break;
3070                         case TYPE_RET: ret++; break;
3071                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
3072                         case TYPE_VOID: vd++; break;
3073                         default: assert(0);
3074                 }
3075                 n++;
3076         }
3077         REPLACE_COUNT_DIST(stat_dist_locals, n);
3078         REPLACE_COUNT_DIST(stat_dist_locals_adr, adr);
3079         REPLACE_COUNT_DIST(stat_dist_locals_void, vd);
3080         REPLACE_COUNT_DIST(stat_dist_locals_ret, ret);
3081         REPLACE_COUNT_DIST(stat_dist_locals_prim, prim);
3082         adr = ret = prim = n = 0;
3083         for (i=0; i<frame->javastackdepth; ++i) {
3084                 switch (frame->javastacktype[i]) {
3085                         case TYPE_ADR: adr++; break;
3086                         case TYPE_RET: ret++; break;
3087                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
3088                 }
3089                 n++;
3090         }
3091         REPLACE_COUNT_DIST(stat_dist_stack, n);
3092         REPLACE_COUNT_DIST(stat_dist_stack_adr, adr);
3093         REPLACE_COUNT_DIST(stat_dist_stack_ret, ret);
3094         REPLACE_COUNT_DIST(stat_dist_stack_prim, prim);
3095 }
3096 #endif /* defined(REPLACE_STATISTICS) */
3097
3098
3099 /* debugging helpers **********************************************************/
3100
3101 /* replace_replacement_point_println *******************************************
3102
3103    Print replacement point info.
3104
3105    IN:
3106        rp...............the replacement point to print
3107
3108 *******************************************************************************/
3109
3110 #if !defined(NDEBUG)
3111
3112 #define TYPECHAR(t)  (((t) >= 0 && (t) <= TYPE_RET) ? show_jit_type_letters[t] : '?')
3113
3114 static char *replace_type_str[] = {
3115         "STD",
3116         "EXH",
3117         "SBR",
3118         "CALL",
3119         "INLINE",
3120         "RETURN",
3121         "BODY"
3122 };
3123
3124 void replace_replacement_point_println(rplpoint *rp, int depth)
3125 {
3126         int j;
3127         int index;
3128
3129         if (!rp) {
3130                 printf("(rplpoint *)NULL\n");
3131                 return;
3132         }
3133
3134         for (j=0; j<depth; ++j)
3135                 putchar('\t');
3136
3137         printf("rplpoint (id %d) %p pc:%p+%d type:%s",
3138                         rp->id, (void*)rp,rp->pc,rp->callsize,
3139                         replace_type_str[rp->type]);
3140         if (rp->flags & RPLPOINT_FLAG_NOTRAP)
3141                 printf(" NOTRAP");
3142         if (rp->flags & RPLPOINT_FLAG_COUNTDOWN)
3143                 printf(" COUNTDOWN");
3144         if (rp->flags & RPLPOINT_FLAG_ACTIVE)
3145                 printf(" ACTIVE");
3146         printf(" parent:%p\n", (void*)rp->parent);
3147         for (j=0; j<depth; ++j)
3148                 putchar('\t');
3149         printf("ra:%d = [",     rp->regalloccount);
3150
3151         for (j=0; j<rp->regalloccount; ++j) {
3152                 if (j)
3153                         putchar(' ');
3154                 index = rp->regalloc[j].index;
3155                 switch (index) {
3156                         case RPLALLOC_STACK: printf("S"); break;
3157                         case RPLALLOC_PARAM: printf("P"); break;
3158                         case RPLALLOC_SYNC : printf("Y"); break;
3159                         default: printf("%d", index);
3160                 }
3161                 printf(":%1c:", TYPECHAR(rp->regalloc[j].type));
3162                 if (rp->regalloc[j].type == TYPE_RET) {
3163                         printf("ret(L%03d)", rp->regalloc[j].regoff);
3164                 }
3165                 else {
3166                         show_allocation(rp->regalloc[j].type, rp->regalloc[j].flags, rp->regalloc[j].regoff);
3167                 }
3168         }
3169
3170         printf("]\n");
3171         for (j=0; j<depth; ++j)
3172                 putchar('\t');
3173         printf("method: ");
3174         method_print(rp->method);
3175
3176         printf("\n");
3177 }
3178 #endif /* !defined(NDEBUG) */
3179
3180
3181 /* replace_show_replacement_points *********************************************
3182
3183    Print replacement point info.
3184
3185    IN:
3186        code.............codeinfo whose replacement points should be printed.
3187
3188 *******************************************************************************/
3189
3190 #if !defined(NDEBUG)
3191 void replace_show_replacement_points(codeinfo *code)
3192 {
3193         int i;
3194         int depth;
3195         rplpoint *rp;
3196         rplpoint *parent;
3197
3198         if (!code) {
3199                 printf("(codeinfo *)NULL\n");
3200                 return;
3201         }
3202
3203         printf("\treplacement points: %d\n",code->rplpointcount);
3204
3205         printf("\ttotal allocations : %d\n",code->regalloccount);
3206         printf("\tsaved int regs    : %d\n",code->savedintcount);
3207         printf("\tsaved flt regs    : %d\n",code->savedfltcount);
3208 #if defined(HAS_ADDRESS_REGISTER_FILE)
3209         printf("\tsaved adr regs    : %d\n",code->savedadrcount);
3210 #endif
3211         printf("\tmemuse            : %d\n",code->memuse);
3212
3213         printf("\n");
3214
3215         for (i=0; i<code->rplpointcount; ++i) {
3216                 rp = code->rplpoints + i;
3217
3218                 depth = 1;
3219                 parent = rp->parent;
3220                 while (parent) {
3221                         depth++;
3222                         parent = parent->parent;
3223                 }
3224                 replace_replacement_point_println(rp, depth);
3225         }
3226 }
3227 #endif
3228
3229
3230 #if !defined(NDEBUG)
3231 static void java_value_print(s4 type, replace_val_t value)
3232 {
3233         java_object_t *obj;
3234         utf           *u;
3235
3236         printf("%016llx",(unsigned long long) value.l);
3237
3238         if (type < 0 || type > TYPE_RET)
3239                 printf(" <INVALID TYPE:%d>", type);
3240         else
3241                 printf(" %s", show_jit_type_names[type]);
3242
3243         if (type == TYPE_ADR && value.a != NULL) {
3244                 obj = value.a;
3245                 putchar(' ');
3246                 utf_display_printable_ascii_classname(obj->vftbl->clazz->name);
3247
3248                 if (obj->vftbl->clazz == class_java_lang_String) {
3249                         printf(" \"");
3250                         u = javastring_toutf(obj, false);
3251                         utf_display_printable_ascii(u);
3252                         printf("\"");
3253                 }
3254         }
3255         else if (type == TYPE_INT) {
3256                 printf(" %ld", (long) value.i);
3257         }
3258         else if (type == TYPE_LNG) {
3259                 printf(" %lld", (long long) value.l);
3260         }
3261         else if (type == TYPE_FLT) {
3262                 printf(" %f", value.f);
3263         }
3264         else if (type == TYPE_DBL) {
3265                 printf(" %f", value.d);
3266         }
3267 }
3268 #endif /* !defined(NDEBUG) */
3269
3270
3271 #if !defined(NDEBUG)
3272 void replace_source_frame_println(sourceframe_t *frame)
3273 {
3274         s4 i,j;
3275         s4 t;
3276
3277         if (REPLACE_IS_NATIVE_FRAME(frame)) {
3278                 printf("\tNATIVE\n");
3279                 printf("\tsfi: "); replace_stackframeinfo_println(frame->sfi);
3280                 printf("\tnativepc: %p\n", frame->nativepc);
3281                 printf("\tframesize: %d\n", frame->nativeframesize);
3282
3283                 j = 0;
3284                 for (i=0; i<INT_REG_CNT; ++i) {
3285                         if (nregdescint[i] == REG_SAV)
3286                                 printf("\t%s = %p\n", abi_registers_integer_name[i], (void*)frame->nativesavint[j++]);
3287                 }
3288
3289                 j = 0;
3290                 for (i=0; i<FLT_REG_CNT; ++i) {
3291                         if (nregdescfloat[i] == REG_SAV)
3292                                 printf("\tF%02d = %f\n", i, frame->nativesavflt[j++]);
3293                 }
3294
3295                 printf("\n");
3296                 return;
3297         }
3298
3299         printf("\t");
3300         method_println(frame->method);
3301         printf("\tid: %d\n", frame->id);
3302         printf("\ttype: %s\n", replace_type_str[frame->type]);
3303         printf("\n");
3304
3305         if (frame->instance.a) {
3306                 printf("\tinstance: ");
3307                 java_value_print(TYPE_ADR, frame->instance);
3308                 printf("\n");
3309         }
3310
3311         if (frame->javalocalcount) {
3312                 printf("\tlocals (%d):\n",frame->javalocalcount);
3313                 for (i=0; i<frame->javalocalcount; ++i) {
3314                         t = frame->javalocaltype[i];
3315                         if (t == TYPE_VOID) {
3316                                 printf("\tlocal[ %2d] = void\n",i);
3317                         }
3318                         else {
3319                                 printf("\tlocal[%c%2d] = ",TYPECHAR(t),i);
3320                                 java_value_print(t, frame->javalocals[i]);
3321                                 printf("\n");
3322                         }
3323                 }
3324                 printf("\n");
3325         }
3326
3327         if (frame->javastackdepth) {
3328                 printf("\tstack (depth %d):\n",frame->javastackdepth);
3329                 for (i=0; i<frame->javastackdepth; ++i) {
3330                         t = frame->javastacktype[i];
3331                         if (t == TYPE_VOID) {
3332                                 printf("\tstack[%2d] = void", i);
3333                         }
3334                         else {
3335                                 printf("\tstack[%2d] = ",i);
3336                                 java_value_print(frame->javastacktype[i], frame->javastack[i]);
3337                                 printf("\n");
3338                         }
3339                 }
3340                 printf("\n");
3341         }
3342
3343         if (frame->syncslotcount) {
3344                 printf("\tsynchronization slots (%d):\n",frame->syncslotcount);
3345                 for (i=0; i<frame->syncslotcount; ++i) {
3346                         printf("\tslot[%2d] = ",i);
3347 #ifdef HAS_4BYTE_STACKSLOT
3348                         printf("%08lx\n",(unsigned long) frame->syncslots[i].p);
3349 #else
3350                         printf("%016llx\n",(unsigned long long) frame->syncslots[i].p);
3351 #endif
3352                 }
3353                 printf("\n");
3354         }
3355
3356         if (frame->fromcode) {
3357                 printf("\tfrom %p ", (void*)frame->fromcode);
3358                 method_println(frame->fromcode->m);
3359         }
3360         if (frame->tocode) {
3361                 printf("\tto %p ", (void*)frame->tocode);
3362                 method_println(frame->tocode->m);
3363         }
3364
3365         if (frame->fromrp) {
3366                 printf("\tfrom replacement point:\n");
3367                 replace_replacement_point_println(frame->fromrp, 2);
3368         }
3369         if (frame->torp) {
3370                 printf("\tto replacement point:\n");
3371                 replace_replacement_point_println(frame->torp, 2);
3372         }
3373
3374         printf("\n");
3375 }
3376 #endif /* !defined(NDEBUG) */
3377
3378
3379 /* replace_sourcestate_println *************************************************
3380
3381    Print source state
3382
3383    IN:
3384        ss...............the source state to print
3385
3386 *******************************************************************************/
3387
3388 #if !defined(NDEBUG)
3389 void replace_sourcestate_println(sourcestate_t *ss)
3390 {
3391         int i;
3392         sourceframe_t *frame;
3393
3394         if (!ss) {
3395                 printf("(sourcestate_t *)NULL\n");
3396                 return;
3397         }
3398
3399         printf("sourcestate_t:\n");
3400
3401         for (i=0, frame = ss->frames; frame != NULL; frame = frame->down, ++i) {
3402                 printf("    frame %d:\n", i);
3403                 replace_source_frame_println(frame);
3404         }
3405 }
3406 #endif
3407
3408
3409 /* replace_sourcestate_println_short *******************************************
3410
3411    Print a compact representation of the given source state.
3412
3413    IN:
3414        ss...............the source state to print
3415
3416 *******************************************************************************/
3417
3418 #if !defined(NDEBUG)
3419 void replace_sourcestate_println_short(sourcestate_t *ss)
3420 {
3421         sourceframe_t *frame;
3422
3423         for (frame = ss->frames; frame != NULL; frame = frame->down) {
3424                 printf("\t");
3425
3426                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
3427                         printf("NATIVE (pc %p size %d) ",
3428                                    (void*)frame->nativepc, frame->nativeframesize);
3429                         replace_stackframeinfo_println(frame->sfi);
3430                         continue;
3431                 }
3432
3433                 if (frame->torp) {
3434                         printf("%c", (frame->torp == frame->fromrp) ? '=' : '+');
3435                 }
3436
3437                 printf("%s", replace_type_str[frame->fromrp->type]);
3438
3439                 if (frame->torp && frame->torp->type != frame->fromrp->type)
3440                         printf("->%s", replace_type_str[frame->torp->type]);
3441
3442                 if (frame->tocode != frame->fromcode)
3443                         printf(" (%p->%p/%d) ",
3444                                    (void*) frame->fromcode, (void*) frame->tocode,
3445                                    frame->fromrp->id);
3446                 else
3447                         printf(" (%p/%d) ", (void*) frame->fromcode, frame->fromrp->id);
3448
3449                 method_println(frame->method);
3450         }
3451 }
3452 #endif
3453
3454 #if !defined(NDEBUG)
3455 static void replace_stackframeinfo_println(stackframeinfo_t *sfi)
3456 {
3457         printf("prev=%p pv=%p sp=%p ra=%p xpc=%p method=",
3458                         (void*)sfi->prev, (void*)sfi->pv, (void*)sfi->sp,
3459                         (void*)sfi->ra, (void*)sfi->xpc);
3460
3461         if (sfi->code)
3462                 method_println(sfi->code->m);
3463         else
3464                 printf("(nil)\n");
3465 }
3466 #endif
3467
3468
3469 /*
3470  * These are local overrides for various environment variables in Emacs.
3471  * Please do not remove this and leave it at the end of the file, where
3472  * Emacs will automagically detect them.
3473  * ---------------------------------------------------------------------
3474  * Local variables:
3475  * mode: c
3476  * indent-tabs-mode: t
3477  * c-basic-offset: 4
3478  * tab-width: 4
3479  * End:
3480  * vim:noexpandtab:sw=4:ts=4:
3481  */