src/vm/jit/replace.c

   1 /* src/vm/jit/replace.c - on-stack replacement of methods
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27 #include "config.h"
  28 #include "vm/types.h"
  29
  30 #include <assert.h>
  31 #include <stdlib.h>
  32
  33 #include "arch.h"
  34
  35 #include "mm/memory.h"
  36
  37 #include "threads/threads-common.h"
  38
  39 #include "toolbox/logging.h"
  40
  41 #include "vm/stringlocal.h"
  42
  43 #include "vm/jit/abi.h"
  44 #include "vm/jit/asmpart.h"
  45 #include "vm/jit/disass.h"
  46 #include "vm/jit/jit.h"
  47 #include "vm/jit/md.h"
  48 #include "vm/jit/methodheader.h"
  49 #include "vm/jit/replace.h"
  50 #include "vm/jit/show.h"
  51 #include "vm/jit/stack.h"
  52
  53 #include "vmcore/options.h"
  54 #include "vmcore/classcache.h"
  55
  56
  57 #define REPLACE_PATCH_DYNAMIC_CALL
  58 /*#define REPLACE_PATCH_ALL*/
  59
  60 #if defined(ENABLE_VMLOG)
  61 #include <vmlog_cacao.h>
  62 #endif
  63
  64 /*** architecture-dependent configuration *************************************/
  65
  66 /* first unset the macros (default) */
  67 #undef REPLACE_RA_BETWEEN_FRAMES
  68 #undef REPLACE_RA_TOP_OF_FRAME
  69 #undef REPLACE_RA_LINKAGE_AREA
  70 #undef REPLACE_LEAFMETHODS_RA_REGISTER
  71 #undef REPLACE_REG_RA
  72
  73 /* i386, x86_64 and m68k */
  74 #if defined(__I386__) || defined(__X86_64__) || defined(__M68K__)
  75 #define REPLACE_RA_BETWEEN_FRAMES
  76 /* alpha */
  77 #elif defined(__ALPHA__)
  78 #define REPLACE_RA_TOP_OF_FRAME
  79 #define REPLACE_LEAFMETHODS_RA_REGISTER
  80 #define REPLACE_REG_RA REG_RA
  81 /* powerpc */
  82 #elif defined(__POWERPC__)
  83 #define REPLACE_RA_LINKAGE_AREA
  84 #define REPLACE_LEAFMETHODS_RA_REGISTER
  85 #define REPLACE_REG_RA REG_ITMP3 /* the execution state has the LR in itmp3 */
  86 /* s390 */
  87 #elif defined(__S390__)
  88 #define REPLACE_RA_TOP_OF_FRAME
  89 #define REPLACE_REG_RA REG_ITMP3
  90 #endif
  91
  92
  93 /*** configuration of native stack slot size **********************************/
  94
  95 /* XXX this should be in md-abi.h files, probably */
  96
  97 #if defined(HAS_4BYTE_STACKSLOT)
  98 #define SIZE_OF_STACKSLOT      4
  99 #define STACK_SLOTS_PER_FLOAT  2
 100 typedef u4 stackslot_t;
 101 #else
 102 #define SIZE_OF_STACKSLOT      8
 103 #define STACK_SLOTS_PER_FLOAT  1
 104 typedef u8 stackslot_t;
 105 #endif
 106
 107
 108 /*** debugging ****************************************************************/
 109
 110 #if !defined(NDEBUG)
 111 static void java_value_print(s4 type, replace_val_t value);
 112 static void replace_stackframeinfo_println(stackframeinfo *sfi);
 113 #endif
 114
 115 #if !defined(NDEBUG)
 116 #define DOLOG(code)        do{ if (opt_TraceReplacement > 1) { code; } } while(0)
 117 #define DOLOG_SHORT(code)  do{ if (opt_TraceReplacement > 0) { code; } } while(0)
 118 #else
 119 #define DOLOG(code)
 120 #define DOLOG_SHORT(code)
 121 #endif
 122
 123
 124 /*** statistics ***************************************************************/
 125
 126 #define REPLACE_STATISTICS
 127
 128 #if defined(REPLACE_STATISTICS)
 129
 130 static int stat_replacements = 0;
 131 static int stat_frames = 0;
 132 static int stat_recompile = 0;
 133 static int stat_staticpatch = 0;
 134 static int stat_unroll_inline = 0;
 135 static int stat_unroll_call = 0;
 136 static int stat_dist_frames[20] = { 0 };
 137 static int stat_dist_locals[20] = { 0 };
 138 static int stat_dist_locals_adr[10] = { 0 };
 139 static int stat_dist_locals_prim[10] = { 0 };
 140 static int stat_dist_locals_ret[10] = { 0 };
 141 static int stat_dist_locals_void[10] = { 0 };
 142 static int stat_dist_stack[10] = { 0 };
 143 static int stat_dist_stack_adr[10] = { 0 };
 144 static int stat_dist_stack_prim[10] = { 0 };
 145 static int stat_dist_stack_ret[10] = { 0 };
 146 static int stat_methods = 0;
 147 static int stat_rploints = 0;
 148 static int stat_regallocs = 0;
 149 static int stat_dist_method_rplpoints[20] = { 0 };
 150
 151 #define REPLACE_COUNT(cnt)  (cnt)++
 152 #define REPLACE_COUNT_IF(cnt, cond)  do{ if(cond) (cnt)++; } while(0)
 153 #define REPLACE_COUNT_INC(cnt, inc)  ((cnt) += (inc))
 154
 155 #define REPLACE_COUNT_DIST(array, val)                               \
 156     do {                                                             \
 157         int limit = (sizeof(array) / sizeof(int)) - 1;               \
 158         if ((val) < (limit)) (array)[val]++;                         \
 159         else (array)[limit]++;                                       \
 160     } while (0)
 161
 162 static void replace_statistics_source_frame(sourceframe_t *frame);
 163
 164 #else
 165
 166 #define REPLACE_COUNT(cnt)
 167 #define REPLACE_COUNT_IF(cnt, cond)
 168 #define REPLACE_COUNT_INC(cnt, inc)
 169 #define REPLACE_COUNT_DIST(array, val)
 170
 171 #endif /* defined(REPLACE_STATISTICS) */
 172
 173
 174 /*** constants used internally ************************************************/
 175
 176 #define TOP_IS_NORMAL    0
 177 #define TOP_IS_ON_STACK  1
 178 #define TOP_IS_IN_ITMP1  2
 179 #define TOP_IS_VOID      3
 180
 181
 182 /******************************************************************************/
 183 /* PART I: Creating / freeing replacement points                              */
 184 /******************************************************************************/
 185
 186
 187 /* replace_create_replacement_point ********************************************
 188
 189    Create a replacement point.
 190
 191    IN:
 192        jd...............current jitdata
 193            iinfo............inlining info for the current position
 194            rp...............pre-allocated (uninitialized) rplpoint
 195            type.............RPLPOINT_TYPE constant
 196            iptr.............current instruction
 197            *pra.............current rplalloc pointer
 198            javalocals.......the javalocals at the current point
 199            stackvars........the stack variables at the current point
 200            stackdepth.......the stack depth at the current point
 201            paramcount.......number of parameters at the start of stackvars
 202
 203    OUT:
 204        *rpa.............points to the next free rplalloc
 205
 206 *******************************************************************************/
 207
 208 static void replace_create_replacement_point(jitdata *jd,
 209                                                                                          insinfo_inline *iinfo,
 210                                                                                          rplpoint *rp,
 211                                                                                          s4 type,
 212                                                                                          instruction *iptr,
 213                                                                                          rplalloc **pra,
 214                                                                                          s4 *javalocals,
 215                                                                                          s4 *stackvars,
 216                                                                                          s4 stackdepth,
 217                                                                                          s4 paramcount)
 218 {
 219         rplalloc *ra;
 220         s4        i;
 221         varinfo  *v;
 222         s4        index;
 223
 224         ra = *pra;
 225
 226         REPLACE_COUNT(stat_rploints);
 227
 228         rp->method = (iinfo) ? iinfo->method : jd->m;
 229         rp->pc = NULL;        /* set by codegen */
 230         rp->callsize = 0;     /* set by codegen */
 231         rp->regalloc = ra;
 232         rp->flags = 0;
 233         rp->type = type;
 234         rp->id = iptr->flags.bits >> INS_FLAG_ID_SHIFT;
 235
 236         /* XXX unify these two fields */
 237         rp->parent = (iinfo) ? iinfo->rp : NULL;
 238
 239         /* store local allocation info of javalocals */
 240
 241         if (javalocals) {
 242                 for (i = 0; i < rp->method->maxlocals; ++i) {
 243                         index = javalocals[i];
 244                         if (index == UNUSED)
 245                                 continue;
 246
 247                         ra->index = i;
 248                         if (index >= 0) {
 249                                 v = VAR(index);
 250                                 ra->flags = v->flags & (INMEMORY);
 251                                 ra->regoff = v->vv.regoff;
 252                                 ra->type = v->type;
 253                         }
 254                         else {
 255                                 ra->regoff = RETADDR_FROM_JAVALOCAL(index);
 256                                 ra->type = TYPE_RET;
 257                                 ra->flags = 0;
 258                         }
 259                         ra++;
 260                 }
 261         }
 262
 263         /* store allocation info of java stack vars */
 264
 265         for (i = 0; i < stackdepth; ++i) {
 266                 v = VAR(stackvars[i]);
 267                 ra->flags = v->flags & (INMEMORY);
 268                 ra->index = (i < paramcount) ? RPLALLOC_PARAM : RPLALLOC_STACK;
 269                 ra->type  = v->type;
 270                 /* XXX how to handle locals on the stack containing returnAddresses? */
 271                 if (v->type == TYPE_RET) {
 272                         assert(stackvars[i] >= jd->localcount);
 273                         ra->regoff = v->vv.retaddr->nr;
 274                 }
 275                 else
 276                         ra->regoff = v->vv.regoff;
 277                 ra++;
 278         }
 279
 280         /* total number of allocations */
 281
 282         rp->regalloccount = ra - rp->regalloc;
 283
 284         *pra = ra;
 285 }
 286
 287
 288 /* replace_create_inline_start_replacement_point *******************************
 289
 290    Create an INLINE_START replacement point.
 291
 292    IN:
 293        jd...............current jitdata
 294            rp...............pre-allocated (uninitialized) rplpoint
 295            iptr.............current instruction
 296            *pra.............current rplalloc pointer
 297            javalocals.......the javalocals at the current point
 298
 299    OUT:
 300        *rpa.............points to the next free rplalloc
 301
 302    RETURN VALUE:
 303        the insinfo_inline * for the following inlined body
 304
 305 *******************************************************************************/
 306
 307 static insinfo_inline * replace_create_inline_start_replacement_point(
 308                                                                                          jitdata *jd,
 309                                                                                          rplpoint *rp,
 310                                                                                          instruction *iptr,
 311                                                                                          rplalloc **pra,
 312                                                                                          s4 *javalocals)
 313 {
 314         insinfo_inline *calleeinfo;
 315         rplalloc       *ra;
 316
 317         calleeinfo = iptr->sx.s23.s3.inlineinfo;
 318
 319         calleeinfo->rp = rp;
 320
 321         replace_create_replacement_point(jd, calleeinfo->parent, rp,
 322                         RPLPOINT_TYPE_INLINE, iptr, pra,
 323                         javalocals,
 324                         calleeinfo->stackvars, calleeinfo->stackvarscount,
 325                         calleeinfo->paramcount);
 326
 327         if (calleeinfo->synclocal != UNUSED) {
 328                 ra = (*pra)++;
 329                 ra->index  = RPLALLOC_SYNC;
 330                 ra->regoff = jd->var[calleeinfo->synclocal].vv.regoff;
 331                 ra->flags  = jd->var[calleeinfo->synclocal].flags & INMEMORY;
 332                 ra->type   = TYPE_ADR;
 333
 334                 rp->regalloccount++;
 335         }
 336
 337         return calleeinfo;
 338 }
 339
 340
 341 /* replace_create_replacement_points *******************************************
 342
 343    Create the replacement points for the given code.
 344
 345    IN:
 346        jd...............current jitdata, must not have any replacement points
 347
 348    OUT:
 349        code->rplpoints.......set to the list of replacement points
 350            code->rplpointcount...number of replacement points
 351            code->regalloc........list of allocation info
 352            code->regalloccount...total length of allocation info list
 353            code->globalcount.....number of global allocations at the
 354                                  start of code->regalloc
 355
 356    RETURN VALUE:
 357        true.............everything ok
 358        false............an exception has been thrown
 359
 360 *******************************************************************************/
 361
 362 #define CLEAR_javalocals(array, method)                              \
 363     do {                                                             \
 364         for (i=0; i<(method)->maxlocals; ++i)                        \
 365             (array)[i] = UNUSED;                                     \
 366     } while (0)
 367
 368 #define COPY_OR_CLEAR_javalocals(dest, array, method)                \
 369     do {                                                             \
 370         if ((array) != NULL)                                         \
 371             MCOPY((dest), (array), s4, (method)->maxlocals);         \
 372         else                                                         \
 373             CLEAR_javalocals((dest), (method));                      \
 374     } while (0)
 375
 376 #define COUNT_javalocals(array, method, counter)                     \
 377     do {                                                             \
 378         for (i=0; i<(method)->maxlocals; ++i)                        \
 379             if ((array)[i] != UNUSED)                                \
 380                                 (counter)++;                                         \
 381     } while (0)
 382
 383 bool replace_create_replacement_points(jitdata *jd)
 384 {
 385         codeinfo        *code;
 386         registerdata    *rd;
 387         basicblock      *bptr;
 388         int              count;
 389         methodinfo      *m;
 390         rplpoint        *rplpoints;
 391         rplpoint        *rp;
 392         int              alloccount;
 393         rplalloc        *regalloc;
 394         rplalloc        *ra;
 395         int              i;
 396         instruction     *iptr;
 397         instruction     *iend;
 398         s4              *javalocals;
 399         s4              *jl;
 400         methoddesc      *md;
 401         insinfo_inline  *iinfo;
 402         s4               startcount;
 403         s4               firstcount;
 404 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 405         bool             needentry;
 406 #endif
 407
 408         REPLACE_COUNT(stat_methods);
 409
 410         /* get required compiler data */
 411
 412         code = jd->code;
 413         rd   = jd->rd;
 414
 415         /* assert that we wont overwrite already allocated data */
 416
 417         assert(code);
 418         assert(code->m);
 419         assert(code->rplpoints == NULL);
 420         assert(code->rplpointcount == 0);
 421         assert(code->regalloc == NULL);
 422         assert(code->regalloccount == 0);
 423         assert(code->globalcount == 0);
 424
 425         m = code->m;
 426
 427         /* set codeinfo flags */
 428
 429         if (jd->isleafmethod)
 430                 CODE_SETFLAG_LEAFMETHOD(code);
 431
 432         /* in instance methods, we may need a rplpoint at the method entry */
 433
 434 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 435         if (!(m->flags & ACC_STATIC)) {
 436                 jd->basicblocks[0].bitflags |= BBFLAG_REPLACEMENT;
 437                 needentry = true;
 438         }
 439         else {
 440                 needentry = false;
 441         }
 442 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
 443
 444         /* iterate over the basic block list to find replacement points */
 445
 446         count = 0;
 447         alloccount = 0;
 448
 449         javalocals = DMNEW(s4, jd->maxlocals);
 450
 451         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 452
 453                 /* skip dead code */
 454
 455                 if (bptr->flags < BBFINISHED)
 456                         continue;
 457
 458                 /* get info about this block */
 459
 460                 m = bptr->method;
 461                 iinfo = bptr->inlineinfo;
 462
 463                 /* initialize javalocals at the start of this block */
 464
 465                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 466
 467                 /* iterate over the instructions */
 468
 469                 iptr = bptr->iinstr;
 470                 iend = iptr + bptr->icount;
 471                 startcount = count;
 472                 firstcount = count;
 473
 474                 for (; iptr != iend; ++iptr) {
 475                         switch (iptr->opc) {
 476                                 case ICMD_INVOKESTATIC:
 477                                 case ICMD_INVOKESPECIAL:
 478                                 case ICMD_INVOKEVIRTUAL:
 479                                 case ICMD_INVOKEINTERFACE:
 480                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 481                                         count++;
 482                                         COUNT_javalocals(javalocals, m, alloccount);
 483                                         alloccount += iptr->s1.argcount;
 484                                         if (iinfo)
 485                                                 alloccount -= iinfo->throughcount;
 486                                         break;
 487
 488                                 case ICMD_ISTORE:
 489                                 case ICMD_LSTORE:
 490                                 case ICMD_FSTORE:
 491                                 case ICMD_DSTORE:
 492                                 case ICMD_ASTORE:
 493                                         stack_javalocals_store(iptr, javalocals);
 494                                         break;
 495
 496                                 case ICMD_IRETURN:
 497                                 case ICMD_LRETURN:
 498                                 case ICMD_FRETURN:
 499                                 case ICMD_DRETURN:
 500                                 case ICMD_ARETURN:
 501                                         alloccount += 1;
 502                                         /* FALLTHROUGH! */
 503                                 case ICMD_RETURN:
 504                                         count++;
 505                                         break;
 506
 507                                 case ICMD_INLINE_START:
 508                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 509
 510                                         count++;
 511                                         COUNT_javalocals(javalocals, m, alloccount);
 512                                         alloccount += iinfo->stackvarscount;
 513                                         if (iinfo->synclocal != UNUSED)
 514                                                 alloccount++;
 515
 516                                         m = iinfo->method;
 517                                         /* javalocals may be set at next block start, or now */
 518                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 519                                         break;
 520
 521                                 case ICMD_INLINE_BODY:
 522                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 523
 524                                         jl = iinfo->javalocals_start;
 525                                         if (jl == NULL) {
 526                                                 /* get the javalocals from the following block start */
 527                                                 assert(bptr->next);
 528                                                 jl = bptr->next->javalocals;
 529                                         }
 530                                         count++;
 531                                         COUNT_javalocals(jl, m, alloccount);
 532                                         break;
 533
 534                                 case ICMD_INLINE_END:
 535                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 536                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 537                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 538                                         m = iinfo->outer;
 539                                         if (iinfo->javalocals_end)
 540                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 541                                         iinfo = iinfo->parent;
 542                                         break;
 543                         }
 544
 545                         if (iptr == bptr->iinstr)
 546                                 firstcount = count;
 547                 } /* end instruction loop */
 548
 549                 /* create replacement points at targets of backward branches */
 550                 /* We only need the replacement point there, if there is no  */
 551                 /* replacement point inside the block.                       */
 552
 553                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 554 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
 555                         int test = (needentry && bptr == jd->basicblocks) ? firstcount : count;
 556 #else
 557                         int test = count;
 558 #endif
 559                         if (test > startcount) {
 560                                 /* we don't need an extra rplpoint */
 561                                 bptr->bitflags &= ~BBFLAG_REPLACEMENT;
 562                         }
 563                         else {
 564                                 count++;
 565                                 alloccount += bptr->indepth;
 566                                 if (bptr->inlineinfo)
 567                                         alloccount -= bptr->inlineinfo->throughcount;
 568
 569                                 COUNT_javalocals(bptr->javalocals, bptr->method, alloccount);
 570                         }
 571                 }
 572
 573         } /* end basicblock loop */
 574
 575         /* if no points were found, there's nothing to do */
 576
 577         if (!count)
 578                 return true;
 579
 580         /* allocate replacement point array and allocation array */
 581
 582         rplpoints = MNEW(rplpoint, count);
 583         regalloc = MNEW(rplalloc, alloccount);
 584         ra = regalloc;
 585
 586         /* initialize replacement point structs */
 587
 588         rp = rplpoints;
 589
 590         /* XXX try to share code with the counting loop! */
 591
 592         for (bptr = jd->basicblocks; bptr; bptr = bptr->next) {
 593                 /* skip dead code */
 594
 595                 if (bptr->flags < BBFINISHED)
 596                         continue;
 597
 598                 /* get info about this block */
 599
 600                 m = bptr->method;
 601                 iinfo = bptr->inlineinfo;
 602
 603                 /* initialize javalocals at the start of this block */
 604
 605                 COPY_OR_CLEAR_javalocals(javalocals, bptr->javalocals, m);
 606
 607                 /* create replacement points at targets of backward branches */
 608
 609                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 610
 611                         i = (iinfo) ? iinfo->throughcount : 0;
 612                         replace_create_replacement_point(jd, iinfo, rp++,
 613                                         bptr->type, bptr->iinstr, &ra,
 614                                         bptr->javalocals, bptr->invars + i, bptr->indepth - i, 0);
 615
 616                         if (JITDATA_HAS_FLAG_COUNTDOWN(jd))
 617                                 rp[-1].flags |= RPLPOINT_FLAG_COUNTDOWN;
 618                 }
 619
 620                 /* iterate over the instructions */
 621
 622                 iptr = bptr->iinstr;
 623                 iend = iptr + bptr->icount;
 624
 625                 for (; iptr != iend; ++iptr) {
 626                         switch (iptr->opc) {
 627                                 case ICMD_INVOKESTATIC:
 628                                 case ICMD_INVOKESPECIAL:
 629                                 case ICMD_INVOKEVIRTUAL:
 630                                 case ICMD_INVOKEINTERFACE:
 631                                         INSTRUCTION_GET_METHODDESC(iptr, md);
 632
 633                                         i = (iinfo) ? iinfo->throughcount : 0;
 634                                         replace_create_replacement_point(jd, iinfo, rp++,
 635                                                         RPLPOINT_TYPE_CALL, iptr, &ra,
 636                                                         javalocals, iptr->sx.s23.s2.args,
 637                                                         iptr->s1.argcount - i,
 638                                                         md->paramcount);
 639                                         break;
 640
 641                                 case ICMD_ISTORE:
 642                                 case ICMD_LSTORE:
 643                                 case ICMD_FSTORE:
 644                                 case ICMD_DSTORE:
 645                                 case ICMD_ASTORE:
 646                                         stack_javalocals_store(iptr, javalocals);
 647                                         break;
 648
 649                                 case ICMD_IRETURN:
 650                                 case ICMD_LRETURN:
 651                                 case ICMD_FRETURN:
 652                                 case ICMD_DRETURN:
 653                                 case ICMD_ARETURN:
 654                                         replace_create_replacement_point(jd, iinfo, rp++,
 655                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 656                                                         NULL, &(iptr->s1.varindex), 1, 0);
 657                                         break;
 658
 659                                 case ICMD_RETURN:
 660                                         replace_create_replacement_point(jd, iinfo, rp++,
 661                                                         RPLPOINT_TYPE_RETURN, iptr, &ra,
 662                                                         NULL, NULL, 0, 0);
 663                                         break;
 664
 665                                 case ICMD_INLINE_START:
 666                                         iinfo = replace_create_inline_start_replacement_point(
 667                                                                 jd, rp++, iptr, &ra, javalocals);
 668                                         m = iinfo->method;
 669                                         /* javalocals may be set at next block start, or now */
 670                                         COPY_OR_CLEAR_javalocals(javalocals, iinfo->javalocals_start, m);
 671                                         break;
 672
 673                                 case ICMD_INLINE_BODY:
 674                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo);
 675
 676                                         jl = iinfo->javalocals_start;
 677                                         if (jl == NULL) {
 678                                                 /* get the javalocals from the following block start */
 679                                                 assert(bptr->next);
 680                                                 jl = bptr->next->javalocals;
 681                                         }
 682                                         /* create a non-trappable rplpoint */
 683                                         replace_create_replacement_point(jd, iinfo, rp++,
 684                                                         RPLPOINT_TYPE_BODY, iptr, &ra,
 685                                                         jl, NULL, 0, 0);
 686                                         rp[-1].flags |= RPLPOINT_FLAG_NOTRAP;
 687                                         break;
 688
 689                                 case ICMD_INLINE_END:
 690                                         assert(iinfo == iptr->sx.s23.s3.inlineinfo ||
 691                                                    iinfo == iptr->sx.s23.s3.inlineinfo->parent);
 692                                         iinfo = iptr->sx.s23.s3.inlineinfo;
 693                                         m = iinfo->outer;
 694                                         if (iinfo->javalocals_end)
 695                                                 MCOPY(javalocals, iinfo->javalocals_end, s4, m->maxlocals);
 696                                         iinfo = iinfo->parent;
 697                                         break;
 698                         }
 699                 } /* end instruction loop */
 700         } /* end basicblock loop */
 701
 702         assert((rp - rplpoints) == count);
 703         assert((ra - regalloc) == alloccount);
 704
 705         /* store the data in the codeinfo */
 706
 707         code->rplpoints     = rplpoints;
 708         code->rplpointcount = count;
 709         code->regalloc      = regalloc;
 710         code->regalloccount = alloccount;
 711         code->globalcount   = 0;
 712         code->savedintcount = INT_SAV_CNT - rd->savintreguse;
 713         code->savedfltcount = FLT_SAV_CNT - rd->savfltreguse;
 714 #if defined(HAS_ADDRESS_REGISTER_FILE)
 715         code->savedadrcount = ADR_SAV_CNT - rd->savadrreguse;
 716 #endif
 717         code->memuse        = rd->memuse;
 718         code->stackframesize = jd->cd->stackframesize;
 719
 720         REPLACE_COUNT_DIST(stat_dist_method_rplpoints, count);
 721         REPLACE_COUNT_INC(stat_regallocs, alloccount);
 722
 723         /* everything alright */
 724
 725         return true;
 726 }
 727
 728
 729 /* replace_free_replacement_points *********************************************
 730
 731    Free memory used by replacement points.
 732
 733    IN:
 734        code.............codeinfo whose replacement points should be freed.
 735
 736 *******************************************************************************/
 737
 738 void replace_free_replacement_points(codeinfo *code)
 739 {
 740         assert(code);
 741
 742         if (code->rplpoints)
 743                 MFREE(code->rplpoints,rplpoint,code->rplpointcount);
 744
 745         if (code->regalloc)
 746                 MFREE(code->regalloc,rplalloc,code->regalloccount);
 747
 748         code->rplpoints = NULL;
 749         code->rplpointcount = 0;
 750         code->regalloc = NULL;
 751         code->regalloccount = 0;
 752         code->globalcount = 0;
 753 }
 754
 755
 756 /******************************************************************************/
 757 /* PART II: Activating / deactivating replacement points                      */
 758 /******************************************************************************/
 759
 760
 761 /* replace_activate_replacement_points *****************************************
 762
 763    Activate the replacement points of the given compilation unit. When this
 764    function returns, the replacement points are "armed", so each thread
 765    reaching one of the points will enter the replacement mechanism.
 766
 767    IN:
 768        code.............codeinfo of which replacement points should be
 769                                                 activated
 770            mappable.........if true, only mappable replacement points are
 771                                                 activated
 772
 773 *******************************************************************************/
 774
 775 void replace_activate_replacement_points(codeinfo *code, bool mappable)
 776 {
 777         rplpoint *rp;
 778         s4        i;
 779         s4        count;
 780         s4        index;
 781         u1       *savedmcode;
 782
 783         assert(code->savedmcode == NULL);
 784
 785         /* count trappable replacement points */
 786
 787         count = 0;
 788         index = 0;
 789         i = code->rplpointcount;
 790         rp = code->rplpoints;
 791         for (; i--; rp++) {
 792                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 793                         continue;
 794
 795                 index++;
 796
 797                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 798                         continue;
 799
 800                 count++;
 801         }
 802
 803         /* allocate buffer for saved machine code */
 804
 805         savedmcode = MNEW(u1, count * REPLACEMENT_PATCH_SIZE);
 806         code->savedmcode = savedmcode;
 807         savedmcode += count * REPLACEMENT_PATCH_SIZE;
 808
 809         /* activate trappable replacement points */
 810         /* (in reverse order to handle overlapping points within basic blocks) */
 811
 812         i = code->rplpointcount;
 813         rp = code->rplpoints + i;
 814         while (rp--, i--) {
 815                 assert(!(rp->flags & RPLPOINT_FLAG_ACTIVE));
 816
 817                 if (rp->flags & RPLPOINT_FLAG_NOTRAP)
 818                         continue;
 819
 820                 index--;
 821
 822                 if (mappable && (rp->type == RPLPOINT_TYPE_RETURN))
 823                         continue;
 824
 825                 DOLOG( printf("activate replacement point:\n");
 826                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 827
 828                 savedmcode -= REPLACEMENT_PATCH_SIZE;
 829
 830 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__) || defined(__S390__)) && defined(ENABLE_JIT)
 831                 md_patch_replacement_point(code, index, rp, savedmcode);
 832 #endif
 833                 rp->flags |= RPLPOINT_FLAG_ACTIVE;
 834         }
 835
 836         assert(savedmcode == code->savedmcode);
 837 }
 838
 839
 840 /* replace_deactivate_replacement_points ***************************************
 841
 842    Deactivate a replacement points in the given compilation unit.
 843    When this function returns, the replacement points will be "un-armed",
 844    that is a each thread reaching a point will just continue normally.
 845
 846    IN:
 847        code.............the compilation unit
 848
 849 *******************************************************************************/
 850
 851 void replace_deactivate_replacement_points(codeinfo *code)
 852 {
 853         rplpoint *rp;
 854         s4        i;
 855         s4        count;
 856         u1       *savedmcode;
 857
 858         if (code->savedmcode == NULL) {
 859                 /* disarm countdown points by patching the branches */
 860
 861                 i = code->rplpointcount;
 862                 rp = code->rplpoints;
 863                 for (; i--; rp++) {
 864                         if ((rp->flags & (RPLPOINT_FLAG_ACTIVE | RPLPOINT_FLAG_COUNTDOWN))
 865                                         == RPLPOINT_FLAG_COUNTDOWN)
 866                         {
 867 #if 0
 868                                 *(s4*) (rp->pc + 9) = 0; /* XXX machine dependent! */
 869 #endif
 870                         }
 871                 }
 872                 return;
 873         }
 874
 875         assert(code->savedmcode != NULL);
 876         savedmcode = code->savedmcode;
 877
 878         /* de-activate each trappable replacement point */
 879
 880         i = code->rplpointcount;
 881         rp = code->rplpoints;
 882         count = 0;
 883         for (; i--; rp++) {
 884                 if (!(rp->flags & RPLPOINT_FLAG_ACTIVE))
 885                         continue;
 886
 887                 count++;
 888
 889                 DOLOG( printf("deactivate replacement point:\n");
 890                            replace_replacement_point_println(rp, 1); fflush(stdout); );
 891
 892 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__) || defined(__S390__)) && defined(ENABLE_JIT)
 893                 md_patch_replacement_point(code, -1, rp, savedmcode);
 894 #endif
 895
 896                 rp->flags &= ~RPLPOINT_FLAG_ACTIVE;
 897
 898                 savedmcode += REPLACEMENT_PATCH_SIZE;
 899         }
 900
 901         assert(savedmcode == code->savedmcode + count * REPLACEMENT_PATCH_SIZE);
 902
 903         /* free saved machine code */
 904
 905         MFREE(code->savedmcode, u1, count * REPLACEMENT_PATCH_SIZE);
 906         code->savedmcode = NULL;
 907 }
 908
 909
 910 /******************************************************************************/
 911 /* PART III: The replacement mechanism                                        */
 912 /******************************************************************************/
 913
 914
 915 /* replace_read_value **********************************************************
 916
 917    Read a value with the given allocation from the execution state.
 918
 919    IN:
 920            es...............execution state
 921            ra...............allocation
 922            javaval..........where to put the value
 923
 924    OUT:
 925        *javaval.........the value
 926
 927 *******************************************************************************/
 928
 929 static void replace_read_value(executionstate_t *es,
 930                                                            rplalloc *ra,
 931                                                            replace_val_t *javaval)
 932 {
 933         if (ra->flags & INMEMORY) {
 934                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
 935 #ifdef HAS_4BYTE_STACKSLOT
 936                 if (IS_2_WORD_TYPE(ra->type)) {
 937                         javaval->l = *(u8*)(es->sp + ra->regoff);
 938                 }
 939                 else {
 940 #endif
 941                         javaval->p = *(ptrint*)(es->sp + ra->regoff);
 942 #ifdef HAS_4BYTE_STACKSLOT
 943                 }
 944 #endif
 945         }
 946         else {
 947                 /* allocated register */
 948                 if (IS_FLT_DBL_TYPE(ra->type)) {
 949                         javaval->d = es->fltregs[ra->regoff];
 950
 951                         if (ra->type == TYPE_FLT)
 952                                 javaval->f = javaval->d;
 953                 }
 954 #if defined(HAS_ADDRESS_REGISTER_FILE)
 955                 else if (IS_ADR_TYPE(ra->type)) {
 956                         javaval->p = es->adrregs[ra->regoff];
 957                 }
 958 #endif
 959                 else {
 960 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
 961                         if (ra->type == TYPE_LNG) {
 962                                 javaval->words.lo = es->intregs[GET_LOW_REG(ra->regoff)];
 963                                 javaval->words.hi = es->intregs[GET_HIGH_REG(ra->regoff)];
 964                         }
 965                         else
 966 #endif /* defined(SUPPORT_COMBINE_INTEGER_REGISTERS) */
 967                                 javaval->p = es->intregs[ra->regoff];
 968                 }
 969         }
 970 }
 971
 972
 973 /* replace_write_value *********************************************************
 974
 975    Write a value to the given allocation in the execution state.
 976
 977    IN:
 978            es...............execution state
 979            ra...............allocation
 980            *javaval.........the value
 981
 982 *******************************************************************************/
 983
 984 static void replace_write_value(executionstate_t *es,
 985                                                             rplalloc *ra,
 986                                                             replace_val_t *javaval)
 987 {
 988         if (ra->flags & INMEMORY) {
 989                 /* XXX HAS_4BYTE_STACKSLOT may not be the right discriminant here */
 990 #ifdef HAS_4BYTE_STACKSLOT
 991                 if (IS_2_WORD_TYPE(ra->type)) {
 992                         *(u8*)(es->sp + ra->regoff) = javaval->l;
 993                 }
 994                 else {
 995 #endif
 996                         *(ptrint*)(es->sp + ra->regoff) = javaval->p;
 997 #ifdef HAS_4BYTE_STACKSLOT
 998                 }
 999 #endif
1000         }
1001         else {
1002                 /* allocated register */
1003                 switch (ra->type) {
1004                         case TYPE_FLT:
1005                                 es->fltregs[ra->regoff] = (double) javaval->f;
1006                                 break;
1007                         case TYPE_DBL:
1008                                 es->fltregs[ra->regoff] = javaval->d;
1009                                 break;
1010 #if defined(SUPPORT_COMBINE_INTEGER_REGISTERS)
1011                         case TYPE_LNG:
1012                                 es->intregs[GET_LOW_REG(ra->regoff)] = javaval->words.lo;
1013                                 es->intregs[GET_HIGH_REG(ra->regoff)] = javaval->words.hi;
1014                                 break;
1015 #endif
1016 #if defined(HAS_ADDRESS_REGISTER_FILE)
1017                         case TYPE_ADR:
1018                                 es->adrregs[ra->regoff] = javaval->p;
1019 #endif
1020                         default:
1021                                 es->intregs[ra->regoff] = javaval->p;
1022                 }
1023         }
1024 }
1025
1026
1027 /* replace_new_sourceframe *****************************************************
1028
1029    Allocate a new source frame and insert it at the front of the frame list.
1030
1031    IN:
1032            ss...............the source state
1033
1034    OUT:
1035            ss->frames.......set to new frame (the new head of the frame list).
1036
1037    RETURN VALUE:
1038        returns the new frame
1039
1040 *******************************************************************************/
1041
1042 static sourceframe_t *replace_new_sourceframe(sourcestate_t *ss)
1043 {
1044         sourceframe_t *frame;
1045
1046         frame = DNEW(sourceframe_t);
1047         MZERO(frame, sourceframe_t, 1);
1048
1049         frame->down = ss->frames;
1050         ss->frames = frame;
1051
1052         return frame;
1053 }
1054
1055
1056 /* replace_read_executionstate *************************************************
1057
1058    Read a source frame from the given executions state.
1059    The new source frame is pushed to the front of the frame list of the
1060    source state.
1061
1062    IN:
1063        rp...............replacement point at which `es` was taken
1064            es...............execution state
1065            ss...............the source state to add the source frame to
1066            topframe.........true, if the first (top-most) source frame on the
1067                             stack is to be read
1068
1069    OUT:
1070        *ss..............the source state with the newly created source frame
1071                             added
1072
1073 *******************************************************************************/
1074
1075 static s4 replace_normalize_type_map[] = {
1076 /* RPLPOINT_TYPE_STD    |--> */ RPLPOINT_TYPE_STD,
1077 /* RPLPOINT_TYPE_EXH    |--> */ RPLPOINT_TYPE_STD,
1078 /* RPLPOINT_TYPE_SBR    |--> */ RPLPOINT_TYPE_STD,
1079 /* RPLPOINT_TYPE_CALL   |--> */ RPLPOINT_TYPE_CALL,
1080 /* RPLPOINT_TYPE_INLINE |--> */ RPLPOINT_TYPE_CALL,
1081 /* RPLPOINT_TYPE_RETURN |--> */ RPLPOINT_TYPE_RETURN,
1082 /* RPLPOINT_TYPE_BODY   |--> */ RPLPOINT_TYPE_STD
1083 };
1084
1085
1086 static void replace_read_executionstate(rplpoint *rp,
1087                                                                                 executionstate_t *es,
1088                                                                                 sourcestate_t *ss,
1089                                                                                 bool topframe)
1090 {
1091         methodinfo    *m;
1092         codeinfo      *code;
1093         int            count;
1094         int            i;
1095         rplalloc      *ra;
1096         sourceframe_t *frame;
1097         int            topslot;
1098         stackslot_t   *sp;
1099         stackslot_t   *basesp;
1100
1101         code = code_find_codeinfo_for_pc(rp->pc);
1102         m = rp->method;
1103         topslot = TOP_IS_NORMAL;
1104
1105         /* stack pointer */
1106
1107         sp = (stackslot_t *) es->sp;
1108
1109         /* in some cases the top stack slot is passed in REG_ITMP1 */
1110
1111         if (rp->type == BBTYPE_EXH) {
1112                 topslot = TOP_IS_IN_ITMP1;
1113         }
1114
1115         /* calculate base stack pointer */
1116
1117         basesp = sp + code->stackframesize;
1118
1119         /* create the source frame */
1120
1121         frame = replace_new_sourceframe(ss);
1122         frame->method = rp->method;
1123         frame->id = rp->id;
1124         assert(rp->type >= 0 && rp->type < sizeof(replace_normalize_type_map)/sizeof(s4));
1125         frame->type = replace_normalize_type_map[rp->type];
1126         frame->fromrp = rp;
1127         frame->fromcode = code;
1128
1129         /* read local variables */
1130
1131         count = m->maxlocals;
1132         frame->javalocalcount = count;
1133         frame->javalocals = DMNEW(replace_val_t, count);
1134         frame->javalocaltype = DMNEW(u1, count);
1135
1136         /* mark values as undefined */
1137         for (i=0; i<count; ++i) {
1138 #if !defined(NDEBUG)
1139                 frame->javalocals[i].l = (u8) 0x00dead0000dead00ULL;
1140 #endif
1141                 frame->javalocaltype[i] = TYPE_VOID;
1142         }
1143
1144         /* some entries in the intregs array are not meaningful */
1145         /*es->intregs[REG_ITMP3] = (u8) 0x11dead1111dead11ULL;*/
1146 #if !defined(NDEBUG)
1147         es->intregs[REG_SP   ] = (ptrint) 0x11dead1111dead11ULL;
1148 #ifdef REG_PV
1149         es->intregs[REG_PV   ] = (ptrint) 0x11dead1111dead11ULL;
1150 #endif
1151 #endif /* !defined(NDEBUG) */
1152
1153         /* read javalocals */
1154
1155         count = rp->regalloccount;
1156         ra = rp->regalloc;
1157
1158         while (count && (i = ra->index) >= 0) {
1159                 assert(i < m->maxlocals);
1160                 frame->javalocaltype[i] = ra->type;
1161                 if (ra->type == TYPE_RET)
1162                         frame->javalocals[i].i = ra->regoff;
1163                 else
1164                         replace_read_value(es, ra, frame->javalocals + i);
1165                 ra++;
1166                 count--;
1167         }
1168
1169         /* read instance, if this is the first rplpoint */
1170
1171 #if defined(REPLACE_PATCH_DYNAMIC_CALL)
1172         if (topframe && !(rp->method->flags & ACC_STATIC) && rp == code->rplpoints) {
1173 #if 1
1174                 /* we are at the start of the method body, so if local 0 is set, */
1175                 /* it is the instance.                                           */
1176                 if (frame->javalocaltype[0] == TYPE_ADR)
1177                         frame->instance = frame->javalocals[0];
1178 #else
1179                 rplalloc instra;
1180                 methoddesc *md;
1181
1182                 md = rp->method->parseddesc;
1183                 assert(md->params);
1184                 assert(md->paramcount >= 1);
1185                 instra.type = TYPE_ADR;
1186                 instra.regoff = md->params[0].regoff;
1187                 if (md->params[0].inmemory) {
1188                         instra.flags = INMEMORY;
1189                         instra.regoff += (1 + code->stackframesize) * SIZE_OF_STACKSLOT;
1190                 }
1191                 else {
1192                         instra.flags = 0;
1193                 }
1194                 replace_read_value(es, &instra, &(frame->instance));
1195 #endif
1196         }
1197 #endif /* defined(REPLACE_PATCH_DYNAMIC_CALL) */
1198
1199         /* read stack slots */
1200
1201         frame->javastackdepth = count;
1202         frame->javastack = DMNEW(replace_val_t, count);
1203         frame->javastacktype = DMNEW(u1, count);
1204
1205 #if !defined(NDEBUG)
1206         /* mark values as undefined */
1207         for (i=0; i<count; ++i) {
1208                 frame->javastack[i].l = (u8) 0x00dead0000dead00ULL;
1209                 frame->javastacktype[i] = TYPE_VOID;
1210         }
1211 #endif /* !defined(NDEBUG) */
1212
1213         i = 0;
1214
1215         /* the first stack slot is special in SBR and EXH blocks */
1216
1217         if (topslot == TOP_IS_ON_STACK) {
1218                 assert(count);
1219
1220                 assert(ra->index == RPLALLOC_STACK);
1221                 assert(ra->type == TYPE_ADR);
1222                 frame->javastack[i].p = sp[-1];
1223                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1224                 count--;
1225                 i++;
1226                 ra++;
1227         }
1228         else if (topslot == TOP_IS_IN_ITMP1) {
1229                 assert(count);
1230
1231                 assert(ra->index == RPLALLOC_STACK);
1232                 assert(ra->type == TYPE_ADR);
1233                 frame->javastack[i].p = es->intregs[REG_ITMP1];
1234                 frame->javastacktype[i] = TYPE_ADR; /* XXX RET */
1235                 count--;
1236                 i++;
1237                 ra++;
1238         }
1239         else if (topslot == TOP_IS_VOID) {
1240                 assert(count);
1241
1242                 assert(ra->index == RPLALLOC_STACK);
1243                 frame->javastack[i].l = 0;
1244                 frame->javastacktype[i] = TYPE_VOID;
1245                 count--;
1246                 i++;
1247                 ra++;
1248         }
1249
1250         /* read remaining stack slots */
1251
1252         for (; count--; ra++) {
1253                 if (ra->index == RPLALLOC_SYNC) {
1254                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1255
1256                         /* only read synchronization slots when traversing an inline point */
1257
1258                         if (!topframe) {
1259                                 sourceframe_t *calleeframe = frame->down;
1260                                 assert(calleeframe);
1261                                 assert(calleeframe->syncslotcount == 0);
1262                                 assert(calleeframe->syncslots == NULL);
1263
1264                                 calleeframe->syncslotcount = 1;
1265                                 calleeframe->syncslots = DMNEW(replace_val_t, 1);
1266                                 replace_read_value(es,ra,calleeframe->syncslots);
1267                         }
1268
1269                         frame->javastackdepth--;
1270                         continue;
1271                 }
1272
1273                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1274
1275                 /* do not read parameters of calls down the call chain */
1276
1277                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1278                         frame->javastackdepth--;
1279                 }
1280                 else {
1281                         if (ra->type == TYPE_RET)
1282                                 frame->javastack[i].i = ra->regoff;
1283                         else
1284                                 replace_read_value(es,ra,frame->javastack + i);
1285                         frame->javastacktype[i] = ra->type;
1286                         i++;
1287                 }
1288         }
1289 }
1290
1291
1292 /* replace_write_executionstate ************************************************
1293
1294    Pop a source frame from the front of the frame list of the given source state
1295    and write its values into the execution state.
1296
1297    IN:
1298        rp...............replacement point for which execution state should be
1299                             created
1300            es...............the execution state to modify
1301            ss...............the given source state
1302            topframe.........true, if this is the last (top-most) source frame to be
1303                             translated
1304
1305    OUT:
1306        *es..............the execution state derived from the source state
1307
1308 *******************************************************************************/
1309
1310 static void replace_write_executionstate(rplpoint *rp,
1311                                                                                  executionstate_t *es,
1312                                                                                  sourcestate_t *ss,
1313                                                                                  bool topframe)
1314 {
1315         methodinfo     *m;
1316         codeinfo       *code;
1317         int             count;
1318         int             i;
1319         rplalloc       *ra;
1320         sourceframe_t  *frame;
1321         int             topslot;
1322         stackslot_t    *sp;
1323         stackslot_t    *basesp;
1324
1325         code = code_find_codeinfo_for_pc(rp->pc);
1326         m = rp->method;
1327         topslot = TOP_IS_NORMAL;
1328
1329         /* pop a source frame */
1330
1331         frame = ss->frames;
1332         assert(frame);
1333         ss->frames = frame->down;
1334
1335         /* calculate stack pointer */
1336
1337         sp = (stackslot_t *) es->sp;
1338
1339         basesp = sp + code->stackframesize;
1340
1341         /* in some cases the top stack slot is passed in REG_ITMP1 */
1342
1343         if (rp->type == BBTYPE_EXH) {
1344                 topslot = TOP_IS_IN_ITMP1;
1345         }
1346
1347         /* write javalocals */
1348
1349         ra = rp->regalloc;
1350         count = rp->regalloccount;
1351
1352         while (count && (i = ra->index) >= 0) {
1353                 assert(i < m->maxlocals);
1354                 assert(i < frame->javalocalcount);
1355                 assert(ra->type == frame->javalocaltype[i]);
1356                 if (ra->type == TYPE_RET) {
1357                         /* XXX assert that it matches this rplpoint */
1358                 }
1359                 else
1360                         replace_write_value(es, ra, frame->javalocals + i);
1361                 count--;
1362                 ra++;
1363         }
1364
1365         /* write stack slots */
1366
1367         i = 0;
1368
1369         /* the first stack slot is special in SBR and EXH blocks */
1370
1371         if (topslot == TOP_IS_ON_STACK) {
1372                 assert(count);
1373
1374                 assert(ra->index == RPLALLOC_STACK);
1375                 assert(i < frame->javastackdepth);
1376                 assert(frame->javastacktype[i] == TYPE_ADR);
1377                 sp[-1] = frame->javastack[i].p;
1378                 count--;
1379                 i++;
1380                 ra++;
1381         }
1382         else if (topslot == TOP_IS_IN_ITMP1) {
1383                 assert(count);
1384
1385                 assert(ra->index == RPLALLOC_STACK);
1386                 assert(i < frame->javastackdepth);
1387                 assert(frame->javastacktype[i] == TYPE_ADR);
1388                 es->intregs[REG_ITMP1] = frame->javastack[i].p;
1389                 count--;
1390                 i++;
1391                 ra++;
1392         }
1393         else if (topslot == TOP_IS_VOID) {
1394                 assert(count);
1395
1396                 assert(ra->index == RPLALLOC_STACK);
1397                 assert(i < frame->javastackdepth);
1398                 assert(frame->javastacktype[i] == TYPE_VOID);
1399                 count--;
1400                 i++;
1401                 ra++;
1402         }
1403
1404         /* write remaining stack slots */
1405
1406         for (; count--; ra++) {
1407                 if (ra->index == RPLALLOC_SYNC) {
1408                         assert(rp->type == RPLPOINT_TYPE_INLINE);
1409
1410                         /* only write synchronization slots when traversing an inline point */
1411
1412                         if (!topframe) {
1413                                 assert(frame->down);
1414                                 assert(frame->down->syncslotcount == 1); /* XXX need to understand more cases */
1415                                 assert(frame->down->syncslots != NULL);
1416
1417                                 replace_write_value(es,ra,frame->down->syncslots);
1418                         }
1419                         continue;
1420                 }
1421
1422                 assert(ra->index == RPLALLOC_STACK || ra->index == RPLALLOC_PARAM);
1423
1424                 /* do not write parameters of calls down the call chain */
1425
1426                 if (!topframe && ra->index == RPLALLOC_PARAM) {
1427                         /* skip it */
1428                 }
1429                 else {
1430                         assert(i < frame->javastackdepth);
1431                         assert(ra->type == frame->javastacktype[i]);
1432                         if (ra->type == TYPE_RET) {
1433                                 /* XXX assert that it matches this rplpoint */
1434                         }
1435                         else {
1436                                 replace_write_value(es,ra,frame->javastack + i);
1437                         }
1438                         i++;
1439                 }
1440         }
1441
1442         /* set new pc */
1443
1444         es->pc = rp->pc;
1445 }
1446
1447
1448 /* replace_pop_activation_record ***********************************************
1449
1450    Peel a stack frame from the execution state.
1451
1452    *** This function imitates the effects of the method epilog ***
1453    *** and returning from the method call.                     ***
1454
1455    IN:
1456            es...............execution state
1457            frame............source frame, receives synchronization slots
1458
1459    OUT:
1460        *es..............the execution state after popping the stack frame
1461
1462 *******************************************************************************/
1463
1464 u1* replace_pop_activation_record(executionstate_t *es,
1465                                                                   sourceframe_t *frame)
1466 {
1467         u1 *ra;
1468         u1 *pv;
1469         s4 reg;
1470         s4 i;
1471         s4 count;
1472         codeinfo *code;
1473         stackslot_t *basesp;
1474         stackslot_t *sp;
1475
1476         assert(es->code);
1477         assert(frame);
1478
1479         /* read the return address */
1480
1481 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1482         if (CODE_IS_LEAFMETHOD(es->code))
1483                 ra = (u1*) (ptrint) es->intregs[REPLACE_REG_RA];
1484         else
1485 #endif
1486                 ra = md_stacktrace_get_returnaddress(es->sp,
1487                                 SIZE_OF_STACKSLOT * es->code->stackframesize);
1488
1489         DOLOG( printf("return address: %p\n", (void*)ra); );
1490
1491         assert(ra);
1492
1493         /* calculate the base of the stack frame */
1494
1495         sp = (stackslot_t *) es->sp;
1496         basesp = sp + es->code->stackframesize;
1497
1498         /* read slots used for synchronization */
1499
1500         assert(frame->syncslotcount == 0);
1501         assert(frame->syncslots == NULL);
1502         count = code_get_sync_slot_count(es->code);
1503         frame->syncslotcount = count;
1504         frame->syncslots = DMNEW(replace_val_t, count);
1505         for (i=0; i<count; ++i) {
1506                 frame->syncslots[i].p = sp[es->code->memuse + i]; /* XXX */
1507         }
1508
1509         /* restore return address, if part of frame */
1510
1511 #if defined(REPLACE_RA_TOP_OF_FRAME)
1512 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1513         if (!CODE_IS_LEAFMETHOD(es->code))
1514 #endif
1515                 es->intregs[REPLACE_REG_RA] = *--basesp;
1516 #endif /* REPLACE_RA_TOP_OF_FRAME */
1517
1518 #if defined(REPLACE_RA_LINKAGE_AREA)
1519 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1520         if (!CODE_IS_LEAFMETHOD(es->code))
1521 #endif
1522                 es->intregs[REPLACE_REG_RA] = basesp[LA_LR_OFFSET / sizeof(stackslot_t)];
1523 #endif /* REPLACE_RA_LINKAGE_AREA */
1524
1525         /* restore saved int registers */
1526
1527         reg = INT_REG_CNT;
1528         for (i=0; i<es->code->savedintcount; ++i) {
1529                 while (nregdescint[--reg] != REG_SAV)
1530                         ;
1531                 es->intregs[reg] = *--basesp;
1532         }
1533
1534         /* restore saved flt registers */
1535
1536         /* XXX align? */
1537         reg = FLT_REG_CNT;
1538         for (i=0; i<es->code->savedfltcount; ++i) {
1539                 while (nregdescfloat[--reg] != REG_SAV)
1540                         ;
1541                 basesp -= STACK_SLOTS_PER_FLOAT;
1542                 es->fltregs[reg] = *(double*)basesp;
1543         }
1544
1545 #if defined(HAS_ADDRESS_REGISTER_FILE)
1546         /* restore saved adr registers */
1547
1548         reg = ADR_REG_CNT;
1549         for (i=0; i<es->code->savedadrcount; ++i) {
1550                 while (nregdescadr[--reg] != REG_SAV)
1551                         ;
1552                 es->adrregs[reg] = *--basesp;
1553         }
1554 #endif
1555
1556         /* adjust the stackpointer */
1557
1558         es->sp += SIZE_OF_STACKSLOT * es->code->stackframesize;
1559
1560 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1561         es->sp += SIZE_OF_STACKSLOT; /* skip return address */
1562 #endif
1563
1564         /* Set the new pc. Subtract one so we do not hit the replacement point */
1565         /* of the instruction following the call, if there is one.             */
1566
1567         es->pc = ra - 1;
1568
1569         /* find the new codeinfo */
1570
1571         pv = md_codegen_get_pv_from_pc(ra);
1572
1573         DOLOG( printf("PV = %p\n", (void*) pv); );
1574
1575         if (pv == NULL) /* XXX can this really happen? */
1576                 return NULL;
1577
1578         code = *(codeinfo **)(pv + CodeinfoPointer);
1579
1580         DOLOG( printf("CODE = %p\n", (void*) code); );
1581
1582         /* return NULL if we reached native code */
1583
1584         es->pv = pv;
1585         es->code = code;
1586
1587         /* in debugging mode clobber non-saved registers */
1588
1589 #if !defined(NDEBUG)
1590         /* for debugging */
1591         for (i=0; i<INT_REG_CNT; ++i)
1592                 if ((nregdescint[i] != REG_SAV)
1593 #if defined(REG_RA)
1594                                 && (i != REPLACE_REG_RA)
1595 #endif
1596                         )
1597                         es->intregs[i] = (ptrint) 0x33dead3333dead33ULL;
1598         for (i=0; i<FLT_REG_CNT; ++i)
1599                 if (nregdescfloat[i] != REG_SAV)
1600                         *(u8*)&(es->fltregs[i]) = 0x33dead3333dead33ULL;
1601 # if defined(HAS_ADDRESS_REGISTER_FILE)
1602         for (i=0; i<ADR_REG_CNT; ++i)
1603                 if (nregdescadr[i] != REG_SAV)
1604                         es->adrregs[i] = (ptrint) 0x33dead3333dead33ULL;
1605 # endif
1606 #endif /* !defined(NDEBUG) */
1607
1608         return (code) ? ra : NULL;
1609 }
1610
1611
1612 /* replace_patch_method_pointer ************************************************
1613
1614    Patch a method pointer (may be in code, data segment, vftbl, or interface
1615    table).
1616
1617    IN:
1618            mpp..............address of the method pointer to patch
1619            entrypoint.......the new entrypoint of the method
1620            kind.............kind of call to patch, used only for debugging
1621
1622 *******************************************************************************/
1623
1624 static void replace_patch_method_pointer(methodptr *mpp,
1625                                                                                  methodptr entrypoint,
1626                                                                                  const char *kind)
1627 {
1628 #if !defined(NDEBUG)
1629         codeinfo       *oldcode;
1630         codeinfo       *newcode;
1631 #endif
1632
1633         DOLOG( printf("patch method pointer from: %p to %p\n",
1634                                   (void*) *mpp, (void*)entrypoint); );
1635
1636 #if !defined(NDEBUG)
1637         oldcode = *(codeinfo **)((u1*)(*mpp) + CodeinfoPointer);
1638         newcode = *(codeinfo **)((u1*)(entrypoint) + CodeinfoPointer);
1639
1640         DOLOG_SHORT( printf("\tpatch %s %p ", kind, (void*) oldcode);
1641                                  method_println(oldcode->m);
1642                                  printf("\t      with      %p ", (void*) newcode);
1643                                  method_println(newcode->m); );
1644
1645         assert(oldcode->m == newcode->m);
1646 #endif
1647
1648         /* write the new entrypoint */
1649
1650         *mpp = (methodptr) entrypoint;
1651 }
1652
1653
1654 /* replace_patch_class *********************************************************
1655
1656    Patch a method in the given class.
1657
1658    IN:
1659            vftbl............vftbl of the class
1660            m................the method to patch
1661            oldentrypoint....the old entrypoint to replace
1662            entrypoint.......the new entrypoint
1663
1664 *******************************************************************************/
1665
1666 void replace_patch_class(vftbl_t *vftbl,
1667                                                  methodinfo *m,
1668                                                  u1 *oldentrypoint,
1669                                                  u1 *entrypoint)
1670 {
1671         s4                 i;
1672         methodptr         *mpp;
1673         methodptr         *mppend;
1674
1675         /* patch the vftbl of the class */
1676
1677         replace_patch_method_pointer(vftbl->table + m->vftblindex,
1678                                                                  entrypoint,
1679                                                                  "virtual  ");
1680
1681         /* patch the interface tables */
1682
1683         assert(oldentrypoint);
1684
1685         for (i=0; i < vftbl->interfacetablelength; ++i) {
1686                 mpp = vftbl->interfacetable[-i];
1687                 mppend = mpp + vftbl->interfacevftbllength[i];
1688                 for (; mpp != mppend; ++mpp)
1689                         if (*mpp == oldentrypoint) {
1690                                 replace_patch_method_pointer(mpp, entrypoint, "interface");
1691                         }
1692         }
1693 }
1694
1695
1696 /* replace_patch_class_hierarchy ***********************************************
1697
1698    Patch a method in all loaded classes.
1699
1700    IN:
1701            m................the method to patch
1702            oldentrypoint....the old entrypoint to replace
1703            entrypoint.......the new entrypoint
1704
1705 *******************************************************************************/
1706
1707 struct replace_patch_data_t {
1708         methodinfo *m;
1709         u1         *oldentrypoint;
1710         u1         *entrypoint;
1711 };
1712
1713 #define CODEINFO_OF_CODE(entrypoint) \
1714         (*(codeinfo **)((u1*)(entrypoint) + CodeinfoPointer))
1715
1716 #define METHOD_OF_CODE(entrypoint) \
1717         (CODEINFO_OF_CODE(entrypoint)->m)
1718
1719 void replace_patch_callback(classinfo *c, struct replace_patch_data_t *pd)
1720 {
1721         vftbl_t *vftbl = c->vftbl;
1722
1723         if (vftbl != NULL
1724                 && vftbl->vftbllength > pd->m->vftblindex
1725                 && vftbl->table[pd->m->vftblindex] != &asm_abstractmethoderror
1726                 && METHOD_OF_CODE(vftbl->table[pd->m->vftblindex]) == pd->m)
1727         {
1728                 replace_patch_class(c->vftbl, pd->m, pd->oldentrypoint, pd->entrypoint);
1729         }
1730 }
1731
1732 void replace_patch_class_hierarchy(methodinfo *m,
1733                                                                    u1 *oldentrypoint,
1734                                                                    u1 *entrypoint)
1735 {
1736         struct replace_patch_data_t pd;
1737
1738         pd.m = m;
1739         pd.oldentrypoint = oldentrypoint;
1740         pd.entrypoint = entrypoint;
1741
1742         DOLOG_SHORT( printf("patching class hierarchy: ");
1743                              method_println(m); );
1744
1745         classcache_foreach_loaded_class(
1746                         (classcache_foreach_functionptr_t) &replace_patch_callback,
1747                         (void*) &pd);
1748 }
1749
1750
1751 /* replace_patch_future_calls **************************************************
1752
1753    Analyse a call site and depending on the kind of call patch the call, the
1754    virtual function table, or the interface table.
1755
1756    IN:
1757            ra...............return address pointing after the call site
1758            callerframe......source frame of the caller
1759            calleeframe......source frame of the callee, must have been mapped
1760
1761 *******************************************************************************/
1762
1763 void replace_patch_future_calls(u1 *ra,
1764                                                                 sourceframe_t *callerframe,
1765                                                                 sourceframe_t *calleeframe)
1766 {
1767         u1             *patchpos;
1768         methodptr       entrypoint;
1769         methodptr       oldentrypoint;
1770         bool            atentry;
1771         stackframeinfo  sfi;
1772         codeinfo       *calleecode;
1773         methodinfo     *calleem;
1774         java_object_t  *obj;
1775         vftbl_t        *vftbl;
1776
1777         assert(ra);
1778         assert(callerframe->down == calleeframe);
1779
1780         /* get the new codeinfo and the method that shall be entered */
1781
1782         calleecode = calleeframe->tocode;
1783         assert(calleecode);
1784
1785         calleem = calleeframe->method;
1786         assert(calleem == calleecode->m);
1787
1788         entrypoint = (methodptr) calleecode->entrypoint;
1789
1790         /* check if we are at an method entry rplpoint at the innermost frame */
1791
1792         atentry = (calleeframe->down == NULL)
1793                         && !(calleem->flags & ACC_STATIC)
1794                         && (calleeframe->fromrp->id == 0); /* XXX */
1795
1796         /* get the position to patch, in case it was a statically bound call   */
1797
1798         sfi.pv = callerframe->fromcode->entrypoint;
1799         patchpos = md_get_method_patch_address(ra, &sfi, NULL);
1800
1801         if (patchpos == NULL) {
1802                 /* the call was dispatched dynamically */
1803
1804                 /* we can only patch such calls if we are at the entry point */
1805
1806                 if (!atentry)
1807                         return;
1808
1809                 assert((calleem->flags & ACC_STATIC) == 0);
1810
1811                 oldentrypoint = calleeframe->fromcode->entrypoint;
1812
1813                 /* we need to know the instance */
1814
1815                 if (!calleeframe->instance.a) {
1816                         DOLOG_SHORT( printf("WARNING: object instance unknown!\n"); );
1817                         replace_patch_class_hierarchy(calleem, oldentrypoint, entrypoint);
1818                         return;
1819                 }
1820
1821                 /* get the vftbl */
1822
1823                 obj = calleeframe->instance.a;
1824                 vftbl = obj->vftbl;
1825
1826                 assert(vftbl->class->vftbl == vftbl);
1827
1828                 DOLOG_SHORT( printf("\tclass: "); class_println(vftbl->class); );
1829
1830                 replace_patch_class(vftbl, calleem, oldentrypoint, entrypoint);
1831         }
1832         else {
1833                 /* the call was statically bound */
1834
1835                 replace_patch_method_pointer((methodptr *) patchpos, entrypoint, "static   ");
1836         }
1837 }
1838
1839
1840 /* replace_push_activation_record **********************************************
1841
1842    Push a stack frame onto the execution state.
1843
1844    *** This function imitates the effects of a call and the ***
1845    *** method prolog of the callee.                         ***
1846
1847    IN:
1848            es...............execution state
1849            rpcall...........the replacement point at the call site
1850            callerframe......source frame of the caller, or NULL for creating the
1851                             first frame
1852            calleeframe......source frame of the callee, must have been mapped
1853
1854    OUT:
1855        *es..............the execution state after pushing the stack frame
1856
1857 *******************************************************************************/
1858
1859 void replace_push_activation_record(executionstate_t *es,
1860                                                                         rplpoint *rpcall,
1861                                                                         sourceframe_t *callerframe,
1862                                                                         sourceframe_t *calleeframe)
1863 {
1864         s4           reg;
1865         s4           i;
1866         s4           count;
1867         stackslot_t *basesp;
1868         stackslot_t *sp;
1869         u1          *ra;
1870         codeinfo    *calleecode;
1871
1872         assert(es);
1873         assert(!rpcall || callerframe);
1874     assert(!rpcall || rpcall->type == RPLPOINT_TYPE_CALL);
1875         assert(!rpcall || rpcall == callerframe->torp);
1876         assert(calleeframe);
1877         assert(!callerframe || calleeframe == callerframe->down);
1878
1879         /* the compilation unit we are entering */
1880
1881         calleecode = calleeframe->tocode;
1882         assert(calleecode);
1883
1884         /* calculate the return address */
1885
1886         if (rpcall)
1887                 ra = rpcall->pc + rpcall->callsize;
1888         else
1889                 ra = es->pc + 1 /* XXX this is ugly */;
1890
1891         /* write the return address */
1892
1893 #if defined(REPLACE_RA_BETWEEN_FRAMES)
1894         es->sp -= SIZE_OF_STACKSLOT;
1895
1896         *((stackslot_t *)es->sp) = (stackslot_t) ra;
1897 #endif /* REPLACE_RA_BETWEEN_FRAMES */
1898
1899 #if defined(REPLACE_REG_RA)
1900         es->intregs[REPLACE_REG_RA] = (ptrint) ra;
1901 #endif
1902
1903         /* we move into a new code unit */
1904
1905         es->code = calleecode;
1906
1907         /* set the new pc XXX not needed? */
1908
1909         es->pc = calleecode->entrypoint;
1910
1911         /* build the stackframe */
1912
1913         DOLOG( printf("building stackframe of %d words at %p\n",
1914                                   calleecode->stackframesize, (void*)es->sp); );
1915
1916         sp = (stackslot_t *) es->sp;
1917         basesp = sp;
1918
1919         sp -= calleecode->stackframesize;
1920         es->sp = (u1*) sp;
1921
1922         /* in debug mode, invalidate stack frame first */
1923
1924         /* XXX may not invalidate linkage area used by native code! */
1925 #if !defined(NDEBUG) && 0
1926         for (i=0; i<(basesp - sp); ++i) {
1927                 sp[i] = 0xdeaddeadU;
1928         }
1929 #endif
1930
1931         /* save the return address register */
1932
1933 #if defined(REPLACE_RA_TOP_OF_FRAME)
1934 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1935         if (!CODE_IS_LEAFMETHOD(calleecode))
1936 #endif
1937                 *--basesp = (ptrint) ra;
1938 #endif /* REPLACE_RA_TOP_OF_FRAME */
1939
1940 #if defined(REPLACE_RA_LINKAGE_AREA)
1941 #if defined(REPLACE_LEAFMETHODS_RA_REGISTER)
1942         if (!CODE_IS_LEAFMETHOD(calleecode))
1943 #endif
1944                 basesp[LA_LR_OFFSET / sizeof(stackslot_t)] = (ptrint) ra;
1945 #endif /* REPLACE_RA_LINKAGE_AREA */
1946
1947         /* save int registers */
1948
1949         reg = INT_REG_CNT;
1950         for (i=0; i<calleecode->savedintcount; ++i) {
1951                 while (nregdescint[--reg] != REG_SAV)
1952                         ;
1953                 *--basesp = es->intregs[reg];
1954
1955                 /* XXX may not clobber saved regs used by native code! */
1956 #if !defined(NDEBUG) && 0
1957                 es->intregs[reg] = (ptrint) 0x44dead4444dead44ULL;
1958 #endif
1959         }
1960
1961         /* save flt registers */
1962
1963         /* XXX align? */
1964         reg = FLT_REG_CNT;
1965         for (i=0; i<calleecode->savedfltcount; ++i) {
1966                 while (nregdescfloat[--reg] != REG_SAV)
1967                         ;
1968                 basesp -= STACK_SLOTS_PER_FLOAT;
1969                 *(double*)basesp = es->fltregs[reg];
1970
1971                 /* XXX may not clobber saved regs used by native code! */
1972 #if !defined(NDEBUG) && 0
1973                 *(u8*)&(es->fltregs[reg]) = 0x44dead4444dead44ULL;
1974 #endif
1975         }
1976
1977 #if defined(HAS_ADDRESS_REGISTER_FILE)
1978         /* save adr registers */
1979
1980         reg = ADR_REG_CNT;
1981         for (i=0; i<calleecode->savedadrcount; ++i) {
1982                 while (nregdescadr[--reg] != REG_SAV)
1983                         ;
1984                 *--basesp = es->adrregs[reg];
1985
1986                 /* XXX may not clobber saved regs used by native code! */
1987 #if !defined(NDEBUG) && 0
1988                 es->adrregs[reg] = (ptrint) 0x44dead4444dead44ULL;
1989 #endif
1990         }
1991 #endif
1992
1993         /* write slots used for synchronization */
1994
1995         count = code_get_sync_slot_count(calleecode);
1996         assert(count == calleeframe->syncslotcount);
1997         for (i=0; i<count; ++i) {
1998                 sp[calleecode->memuse + i] = calleeframe->syncslots[i].p;
1999         }
2000
2001         /* set the PV */
2002
2003         es->pv = calleecode->entrypoint;
2004
2005         /* redirect future invocations */
2006
2007         if (callerframe && rpcall) {
2008 #if defined(REPLACE_PATCH_ALL)
2009                 if (rpcall->type == callerframe->fromrp->type)
2010 #else
2011                 if (rpcall == callerframe->fromrp)
2012 #endif
2013                         replace_patch_future_calls(ra, callerframe, calleeframe);
2014         }
2015 }
2016
2017
2018 /* replace_find_replacement_point **********************************************
2019
2020    Find the replacement point in the given code corresponding to the
2021    position given in the source frame.
2022
2023    IN:
2024            code.............the codeinfo in which to search the rplpoint
2025            frame............the source frame defining the position to look for
2026            parent...........parent replacement point to match
2027
2028    RETURN VALUE:
2029        the replacement point
2030
2031 *******************************************************************************/
2032
2033 rplpoint * replace_find_replacement_point(codeinfo *code,
2034                                                                                   sourceframe_t *frame,
2035                                                                                   rplpoint *parent)
2036 {
2037         methodinfo *m;
2038         rplpoint *rp;
2039         s4        i;
2040         s4        j;
2041         s4        stacki;
2042         rplalloc *ra;
2043
2044         assert(code);
2045         assert(frame);
2046
2047         DOLOG( printf("searching replacement point for:\n");
2048                    replace_source_frame_println(frame); );
2049
2050         m = frame->method;
2051
2052         DOLOG( printf("code = %p\n", (void*)code); );
2053
2054         rp = code->rplpoints;
2055         i = code->rplpointcount;
2056         while (i--) {
2057                 if (rp->id == frame->id && rp->method == frame->method
2058                                 && rp->parent == parent
2059                                 && replace_normalize_type_map[rp->type] == frame->type)
2060                 {
2061                         /* check if returnAddresses match */
2062                         /* XXX optimize: only do this if JSRs in method */
2063                         DOLOG( printf("checking match for:");
2064                                    replace_replacement_point_println(rp, 1); fflush(stdout); );
2065                         ra = rp->regalloc;
2066                         stacki = 0;
2067                         for (j = rp->regalloccount; j--; ++ra) {
2068                                 if (ra->type == TYPE_RET) {
2069                                         if (ra->index == RPLALLOC_STACK) {
2070                                                 assert(stacki < frame->javastackdepth);
2071                                                 if (frame->javastack[stacki].i != ra->regoff)
2072                                                         goto no_match;
2073                                                 stacki++;
2074                                         }
2075                                         else {
2076                                                 assert(ra->index >= 0 && ra->index < frame->javalocalcount);
2077                                                 if (frame->javalocals[ra->index].i != ra->regoff)
2078                                                         goto no_match;
2079                                         }
2080                                 }
2081                         }
2082
2083                         /* found */
2084                         return rp;
2085                 }
2086 no_match:
2087                 rp++;
2088         }
2089
2090 #if !defined(NDEBUG)
2091         printf("candidate replacement points were:\n");
2092         rp = code->rplpoints;
2093         i = code->rplpointcount;
2094         for (; i--; ++rp) {
2095                 replace_replacement_point_println(rp, 1);
2096         }
2097 #endif
2098
2099         vm_abort("no matching replacement point found");
2100         return NULL; /* NOT REACHED */
2101 }
2102
2103
2104 /* replace_find_replacement_point_for_pc ***************************************
2105
2106    Find the nearest replacement point at or before the given PC.
2107
2108    IN:
2109        code.............compilation unit the PC is in
2110            pc...............the machine code PC
2111
2112    RETURN VALUE:
2113        the replacement point found, or
2114            NULL if no replacement point was found
2115
2116 *******************************************************************************/
2117
2118 rplpoint *replace_find_replacement_point_for_pc(codeinfo *code, u1 *pc)
2119 {
2120         rplpoint *found;
2121         rplpoint *rp;
2122         s4        i;
2123
2124         DOLOG( printf("searching for rp in %p ", (void*)code);
2125                    method_println(code->m); );
2126
2127         found = NULL;
2128
2129         rp = code->rplpoints;
2130         for (i=0; i<code->rplpointcount; ++i, ++rp) {
2131                 DOLOG( replace_replacement_point_println(rp, 2); );
2132                 if (rp->pc <= pc)
2133                         found = rp;
2134         }
2135
2136         return found;
2137 }
2138
2139
2140 /* replace_pop_native_frame ****************************************************
2141
2142    Unroll a native frame in the execution state and create a source frame
2143    for it.
2144
2145    IN:
2146            es...............current execution state
2147            ss...............the current source state
2148            sfi..............stackframeinfo for the native frame
2149
2150    OUT:
2151        es...............execution state after unrolling the native frame
2152            ss...............gets the added native source frame
2153
2154 *******************************************************************************/
2155
2156 static void replace_pop_native_frame(executionstate_t *es,
2157                                                                          sourcestate_t *ss,
2158                                                                          stackframeinfo *sfi)
2159 {
2160         sourceframe_t *frame;
2161         codeinfo      *code;
2162         s4             i,j;
2163
2164         assert(sfi);
2165
2166         frame = replace_new_sourceframe(ss);
2167
2168         frame->sfi = sfi;
2169
2170         /* remember pc and size of native frame */
2171
2172         frame->nativepc = es->pc;
2173         frame->nativeframesize = sfi->sp - es->sp;
2174         assert(frame->nativeframesize >= 0);
2175
2176         /* remember values of saved registers */
2177
2178         j = 0;
2179         for (i=0; i<INT_REG_CNT; ++i) {
2180                 if (nregdescint[i] == REG_SAV)
2181                         frame->nativesavint[j++] = es->intregs[i];
2182         }
2183
2184         j = 0;
2185         for (i=0; i<FLT_REG_CNT; ++i) {
2186                 if (nregdescfloat[i] == REG_SAV)
2187                         frame->nativesavflt[j++] = es->fltregs[i];
2188         }
2189
2190 #if defined(HAS_ADDRESS_REGISTER_FILE)
2191         j = 0;
2192         for (i=0; i<ADR_REG_CNT; ++i) {
2193                 if (nregdescadr[i] == REG_SAV)
2194                         frame->nativesavadr[j++] = es->adrregs[i];
2195         }
2196 #endif
2197
2198         /* restore saved registers */
2199
2200 #if 0
2201         /* XXX we don't have them, yet, in the sfi, so clear them */
2202
2203         for (i=0; i<INT_REG_CNT; ++i) {
2204                 if (nregdescint[i] == REG_SAV)
2205                         es->intregs[i] = 0;
2206         }
2207
2208         for (i=0; i<FLT_REG_CNT; ++i) {
2209                 if (nregdescfloat[i] == REG_SAV)
2210                         es->fltregs[i] = 0.0;
2211         }
2212
2213 # if defined(HAS_ADDRESS_REGISTER_FILE)
2214         for (i=0; i<ADR_REG_CNT; ++i) {
2215                 if (nregdescadr[i] == REG_SAV)
2216                         es->adrregs[i] = 0;
2217         }
2218 # endif
2219 #endif
2220
2221         /* restore pv, pc, and sp */
2222
2223         if (sfi->pv == NULL) {
2224                 /* frame of a native function call */
2225                 es->pv = md_codegen_get_pv_from_pc(sfi->ra);
2226         }
2227         else {
2228                 es->pv = sfi->pv;
2229         }
2230         es->pc = ((sfi->xpc) ? sfi->xpc : sfi->ra) - 1;
2231         es->sp = sfi->sp;
2232
2233         /* find the new codeinfo */
2234
2235         DOLOG( printf("PV = %p\n", (void*) es->pv); );
2236
2237         assert(es->pv != NULL);
2238
2239         code = *(codeinfo **)(es->pv + CodeinfoPointer);
2240
2241         DOLOG( printf("CODE = %p\n", (void*) code); );
2242
2243         es->code = code;
2244 }
2245
2246
2247 /* replace_push_native_frame ***************************************************
2248
2249    Rebuild a native frame onto the execution state and remove its source frame.
2250
2251    Note: The native frame is "rebuild" by setting fields like PC and stack
2252          pointer in the execution state accordingly. Values in the
2253                  stackframeinfo may be modified, but the actual stack frame of the
2254                  native code is not touched.
2255
2256    IN:
2257            es...............current execution state
2258            ss...............the current source state
2259
2260    OUT:
2261        es...............execution state after re-rolling the native frame
2262            ss...............the native source frame is removed
2263
2264 *******************************************************************************/
2265
2266 static void replace_push_native_frame(executionstate_t *es, sourcestate_t *ss)
2267 {
2268         sourceframe_t *frame;
2269         s4             i,j;
2270
2271         assert(es);
2272         assert(ss);
2273
2274         DOLOG( printf("pushing native frame\n"); );
2275
2276         /* remove the frame from the source state */
2277
2278         frame = ss->frames;
2279         assert(frame);
2280         assert(REPLACE_IS_NATIVE_FRAME(frame));
2281
2282         ss->frames = frame->down;
2283
2284         /* assert that the native frame has not moved */
2285
2286         assert(es->sp == frame->sfi->sp);
2287
2288         /* restore saved registers */
2289
2290         j = 0;
2291         for (i=0; i<INT_REG_CNT; ++i) {
2292                 if (nregdescint[i] == REG_SAV)
2293                         es->intregs[i] = frame->nativesavint[j++];
2294         }
2295
2296         j = 0;
2297         for (i=0; i<FLT_REG_CNT; ++i) {
2298                 if (nregdescfloat[i] == REG_SAV)
2299                         es->fltregs[i] = frame->nativesavflt[j++];
2300         }
2301
2302 #if defined(HAS_ADDRESS_REGISTER_FILE)
2303         j = 0;
2304         for (i=0; i<ADR_REG_CNT; ++i) {
2305                 if (nregdescadr[i] == REG_SAV)
2306                         es->adrregs[i] = frame->nativesavadr[j++];
2307         }
2308 #endif
2309
2310         /* skip the native frame on the machine stack */
2311
2312         es->sp -= frame->nativeframesize;
2313
2314         /* set the pc the next frame must return to */
2315
2316         es->pc = frame->nativepc;
2317 }
2318
2319
2320 /* replace_recover_source_state ************************************************
2321
2322    Recover the source state from the given replacement point and execution
2323    state.
2324
2325    IN:
2326        rp...............replacement point that has been reached, if any
2327            sfi..............stackframeinfo, if called from native code
2328            es...............execution state at the replacement point rp
2329
2330    RETURN VALUE:
2331        the source state
2332
2333 *******************************************************************************/
2334
2335 sourcestate_t *replace_recover_source_state(rplpoint *rp,
2336                                                                                         stackframeinfo *sfi,
2337                                                                                     executionstate_t *es)
2338 {
2339         sourcestate_t *ss;
2340         u1            *ra;
2341         bool           locked;
2342 #if defined(REPLACE_STATISTICS)
2343         s4             depth;
2344 #endif
2345
2346         /* create the source frame structure in dump memory */
2347
2348         ss = DNEW(sourcestate_t);
2349         ss->frames = NULL;
2350
2351         /* get the stackframeinfo if none is given */
2352
2353         if (sfi == NULL)
2354                 sfi = STACKFRAMEINFO;
2355
2356         /* each iteration of the loop recovers one source frame */
2357
2358         depth = 0;
2359         locked = false;
2360
2361         while (rp || sfi) {
2362
2363                 DOLOG( replace_executionstate_println(es); );
2364
2365                 /* if we are not at a replacement point, it is a native frame */
2366
2367                 if (rp == NULL) {
2368                         DOLOG( printf("native frame: sfi: "); replace_stackframeinfo_println(sfi); );
2369
2370                         locked = true;
2371                         replace_pop_native_frame(es, ss, sfi);
2372                         sfi = sfi->prev;
2373
2374                         if (es->code == NULL)
2375                                 continue;
2376
2377                         goto after_machine_frame;
2378                 }
2379
2380                 /* read the values for this source frame from the execution state */
2381
2382                 DOLOG( printf("recovering source state for%s:\n",
2383                                         (ss->frames == NULL) ? " TOPFRAME" : "");
2384                            replace_replacement_point_println(rp, 1); );
2385
2386                 replace_read_executionstate(rp, es, ss, ss->frames == NULL);
2387
2388 #if defined(ENABLE_VMLOG)
2389                 vmlog_cacao_unrol_method(ss->frames->method);
2390 #endif
2391
2392 #if defined(REPLACE_STATISTICS)
2393                 REPLACE_COUNT(stat_frames);
2394                 depth++;
2395                 replace_statistics_source_frame(ss->frames);
2396 #endif
2397
2398                 /* in locked areas (below native frames), identity map the frame */
2399
2400                 if (locked) {
2401                         ss->frames->torp = ss->frames->fromrp;
2402                         ss->frames->tocode = ss->frames->fromcode;
2403                 }
2404
2405                 /* unroll to the next (outer) frame */
2406
2407                 if (rp->parent) {
2408                         /* this frame is in inlined code */
2409
2410                         DOLOG( printf("INLINED!\n"); );
2411
2412                         rp = rp->parent;
2413
2414                         assert(rp->type == RPLPOINT_TYPE_INLINE);
2415                         REPLACE_COUNT(stat_unroll_inline);
2416                 }
2417                 else {
2418                         /* this frame had been called at machine-level. pop it. */
2419
2420                         DOLOG( printf("UNWIND\n"); );
2421
2422                         ra = replace_pop_activation_record(es, ss->frames);
2423                         if (ra == NULL) {
2424                                 DOLOG( printf("REACHED NATIVE CODE\n"); );
2425
2426                                 rp = NULL;
2427
2428                                 break; /* XXX remove to activate native frames */
2429                                 continue;
2430                         }
2431
2432                         /* find the replacement point at the call site */
2433
2434 after_machine_frame:
2435                         rp = replace_find_replacement_point_for_pc(es->code, es->pc);
2436
2437                         if (rp == NULL)
2438                                 vm_abort("could not find replacement point while unrolling call");
2439
2440                         DOLOG( printf("found replacement point.\n");
2441                                         replace_replacement_point_println(rp, 1); );
2442
2443                         assert(rp->type == RPLPOINT_TYPE_CALL);
2444                         REPLACE_COUNT(stat_unroll_call);
2445                 }
2446         } /* end loop over source frames */
2447
2448         REPLACE_COUNT_DIST(stat_dist_frames, depth);
2449
2450         return ss;
2451 }
2452
2453
2454 /* replace_map_source_state ****************************************************
2455
2456    Map each source frame in the given source state to a target replacement
2457    point and compilation unit. If no valid code is available for a source
2458    frame, it is (re)compiled.
2459
2460    IN:
2461        ss...............the source state
2462
2463    OUT:
2464        ss...............the source state, modified: The `torp` and `tocode`
2465                             fields of each source frame are set.
2466
2467    RETURN VALUE:
2468        true.............everything went ok
2469            false............an exception has been thrown
2470
2471 *******************************************************************************/
2472
2473 static bool replace_map_source_state(sourcestate_t *ss)
2474 {
2475         sourceframe_t *frame;
2476         codeinfo      *code;
2477         rplpoint      *rp;
2478         rplpoint      *parent; /* parent of inlined rplpoint */
2479 #if defined(REPLACE_STATISTICS)
2480         codeinfo      *oldcode;
2481 #endif
2482
2483         parent = NULL;
2484         code = NULL;
2485
2486         /* iterate over the source frames from outermost to innermost */
2487
2488         for (frame = ss->frames; frame != NULL; frame = frame->down) {
2489
2490                 /* XXX skip native frames */
2491
2492                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
2493                         parent = NULL;
2494                         continue;
2495                 }
2496
2497                 /* map frames which are not already mapped */
2498
2499                 if (frame->tocode) {
2500                         code = frame->tocode;
2501                         rp = frame->torp;
2502                         assert(rp);
2503                 }
2504                 else {
2505                         assert(frame->torp == NULL);
2506
2507                         if (parent == NULL) {
2508                                 /* find code for this frame */
2509
2510 #if defined(REPLACE_STATISTICS)
2511                                 oldcode = frame->method->code;
2512 #endif
2513                                 /* request optimization of hot methods and their callers */
2514
2515                                 if (frame->method->hitcountdown < 0
2516                                         || (frame->down && frame->down->method->hitcountdown < 0))
2517                                         jit_request_optimization(frame->method);
2518
2519                                 code = jit_get_current_code(frame->method);
2520
2521                                 if (code == NULL)
2522                                         return false; /* exception */
2523
2524                                 REPLACE_COUNT_IF(stat_recompile, code != oldcode);
2525                         }
2526
2527                         assert(code);
2528
2529                         /* map this frame */
2530
2531                         rp = replace_find_replacement_point(code, frame, parent);
2532
2533                         frame->tocode = code;
2534                         frame->torp = rp;
2535                 }
2536
2537                 if (rp->type == RPLPOINT_TYPE_CALL) {
2538                         parent = NULL;
2539                 }
2540                 else {
2541                         /* inlining */
2542                         parent = rp;
2543                 }
2544         }
2545
2546         return true;
2547 }
2548
2549
2550 /* replace_build_execution_state_intern ****************************************
2551
2552    Build an execution state for the given (mapped) source state.
2553
2554    !!! CAUTION: This function rewrites the machine stack !!!
2555
2556    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2557
2558    IN:
2559        ss...............the source state. Must have been mapped by
2560                                                 replace_map_source_state before.
2561            es...............the base execution state on which to build
2562
2563    OUT:
2564        *es..............the new execution state
2565
2566 *******************************************************************************/
2567
2568 static void replace_build_execution_state_intern(sourcestate_t *ss,
2569                                                                                                  executionstate_t *es)
2570 {
2571         rplpoint      *rp;
2572         sourceframe_t *prevframe;
2573         rplpoint      *parent;
2574
2575         parent = NULL;
2576         prevframe = NULL;
2577         rp = NULL;
2578
2579         while (ss->frames) {
2580
2581                 if (REPLACE_IS_NATIVE_FRAME(ss->frames)) {
2582                         prevframe = ss->frames;
2583                         replace_push_native_frame(es, ss);
2584                         parent = NULL;
2585                         rp = NULL;
2586                         continue;
2587                 }
2588
2589                 if (parent == NULL) {
2590                         /* create a machine-level stack frame */
2591
2592                         DOLOG( printf("pushing activation record for:\n");
2593                                    if (rp) replace_replacement_point_println(rp, 1);
2594                                    else printf("\tfirst frame\n"); );
2595
2596                         replace_push_activation_record(es, rp, prevframe, ss->frames);
2597
2598                         DOLOG( replace_executionstate_println(es); );
2599                 }
2600
2601                 rp = ss->frames->torp;
2602                 assert(rp);
2603
2604                 DOLOG( printf("creating execution state for%s:\n",
2605                                 (ss->frames->down == NULL) ? " TOPFRAME" : "");
2606                            replace_replacement_point_println(ss->frames->fromrp, 1);
2607                            replace_replacement_point_println(rp, 1); );
2608
2609                 es->code = ss->frames->tocode;
2610                 prevframe = ss->frames;
2611
2612 #if defined(ENABLE_VMLOG)
2613                 vmlog_cacao_rerol_method(ss->frames->method);
2614 #endif
2615
2616                 replace_write_executionstate(rp, es, ss, ss->frames->down == NULL);
2617
2618                 DOLOG( replace_executionstate_println(es); );
2619
2620                 if (rp->type == RPLPOINT_TYPE_CALL) {
2621                         parent = NULL;
2622                 }
2623                 else {
2624                         /* inlining */
2625                         parent = rp;
2626                 }
2627         }
2628 }
2629
2630
2631 /* replace_build_execution_state ***********************************************
2632
2633    This function contains the final phase of replacement. It builds the new
2634    execution state, releases dump memory, and returns to the calling
2635    assembler function which finishes replacement.
2636
2637    NOTE: This function is called from asm_replacement_in, with the stack
2638          pointer at the start of the safe stack area.
2639
2640    THIS FUNCTION MUST BE CALLED USING A SAFE STACK AREA!
2641
2642    CAUTION: This function and its children must not use a lot of stack!
2643             There are only REPLACE_SAFESTACK_SIZE bytes of C stack
2644                         available.
2645
2646    IN:
2647        st...............the safestack contained the necessary data
2648
2649 *******************************************************************************/
2650
2651 void replace_build_execution_state(replace_safestack_t *st)
2652 {
2653         replace_build_execution_state_intern(st->ss, &(st->es));
2654
2655         DOLOG( replace_executionstate_println(&(st->es)); );
2656
2657         /* release dump area */
2658
2659         dump_release(st->dumpsize);
2660
2661         /* new code is entered after returning */
2662
2663         DOLOG( printf("JUMPING IN!\n"); fflush(stdout); );
2664 }
2665
2666
2667 /* replace_alloc_safestack *****************************************************
2668
2669    Allocate a safe stack area to use during the final phase of replacement.
2670    The returned area is not initialized. This must be done by the caller.
2671
2672    RETURN VALUE:
2673        a newly allocated replace_safestack_t *
2674
2675 *******************************************************************************/
2676
2677 static replace_safestack_t *replace_alloc_safestack()
2678 {
2679         u1 *mem;
2680         replace_safestack_t *st;
2681
2682         mem = MNEW(u1, sizeof(replace_safestack_t) + REPLACE_STACK_ALIGNMENT - 1);
2683
2684         st = (replace_safestack_t *) ((ptrint)(mem + REPLACE_STACK_ALIGNMENT - 1)
2685                                                                                 & ~(REPLACE_STACK_ALIGNMENT - 1));
2686
2687 #if !defined(NDEBUG)
2688         memset(st, 0xa5, sizeof(replace_safestack_t));
2689 #endif
2690
2691         st->mem = mem;
2692
2693         return st;
2694 }
2695
2696
2697 /* replace_free_safestack ******************************************************
2698
2699    Free the given safestack structure, making a copy of the contained
2700    execution state before freeing it.
2701
2702    NOTE: This function is called from asm_replacement_in.
2703
2704    IN:
2705        st...............the safestack to free
2706            tmpes............where to copy the execution state to
2707
2708    OUT:
2709            *tmpes...........receives a copy of st->es
2710
2711 *******************************************************************************/
2712
2713 void replace_free_safestack(replace_safestack_t *st, executionstate_t *tmpes)
2714 {
2715         u1 *mem;
2716
2717         /* copy the executionstate_t to the temporary location */
2718
2719         *tmpes = st->es;
2720
2721         /* get the memory address to free */
2722
2723         mem = st->mem;
2724
2725         /* destroy memory (in debug mode) */
2726
2727 #if !defined(NDEBUG)
2728         memset(st, 0xa5, sizeof(replace_safestack_t));
2729 #endif
2730
2731         /* free the safe stack struct */
2732
2733         MFREE(mem, u1, sizeof(replace_safestack_t) + REPLACE_STACK_ALIGNMENT - 1);
2734 }
2735
2736
2737 /* replace_me_wrapper **********************************************************
2738
2739    TODO: Document me!
2740
2741 *******************************************************************************/
2742
2743 bool replace_me_wrapper(u1 *pc)
2744 {
2745         codeinfo         *code;
2746         rplpoint         *rp;
2747         executionstate_t  es;
2748
2749         /* search the codeinfo for the given PC */
2750
2751         code = code_find_codeinfo_for_pc(pc);
2752         assert(code);
2753
2754         /* search for a replacement point at the given PC */
2755
2756 #if 0
2757         rp = replace_find_replacement_point_for_pc(code, pc);
2758         assert(rp == NULL || rp->pc == pc);
2759 #else
2760         {
2761                 int i;
2762                 rplpoint *rp2;
2763                 rp = NULL;
2764                 for (i=0,rp2=code->rplpoints; i<code->rplpointcount; i++,rp2++) {
2765                         if (rp2->pc == pc)
2766                                 rp = rp2;
2767                 }
2768         }
2769 #endif
2770
2771         /* check if the replacement point is active */
2772
2773         if (rp != NULL && (rp->flags & RPLPOINT_FLAG_ACTIVE)) {
2774
2775                 /*md_replace_executionstate_read(&es, context);*/
2776
2777                 replace_me(rp, &es);
2778
2779                 return true;
2780         }
2781         else
2782                 return false;
2783 }
2784
2785
2786 /* replace_me ******************************************************************
2787
2788    This function is called by asm_replacement_out when a thread reaches
2789    a replacement point. `replace_me` must map the execution state to the
2790    target replacement point and let execution continue there.
2791
2792    This function never returns!
2793
2794    IN:
2795        rp...............replacement point that has been reached
2796            es...............execution state read by asm_replacement_out
2797
2798 *******************************************************************************/
2799
2800 void replace_me(rplpoint *rp, executionstate_t *es)
2801 {
2802         sourcestate_t       *ss;
2803         sourceframe_t       *frame;
2804         s4                   dumpsize;
2805         rplpoint            *origrp;
2806         replace_safestack_t *safestack;
2807
2808         origrp = rp;
2809         es->code = code_find_codeinfo_for_pc(rp->pc);
2810
2811         DOLOG_SHORT( printf("REPLACING(%d %p): (id %d %p) ",
2812                                  stat_replacements, (void*)THREADOBJECT,
2813                                  rp->id, (void*)rp);
2814                                  method_println(es->code->m); );
2815
2816         DOLOG( replace_replacement_point_println(rp, 1);
2817                    replace_executionstate_println(es); );
2818
2819         REPLACE_COUNT(stat_replacements);
2820
2821         /* mark start of dump memory area */
2822
2823         dumpsize = dump_size();
2824
2825         /* recover source state */
2826
2827         ss = replace_recover_source_state(rp, NULL, es);
2828
2829         /* map the source state */
2830
2831         if (!replace_map_source_state(ss))
2832                 vm_abort("exception during method replacement");
2833
2834         DOLOG( replace_sourcestate_println(ss); );
2835
2836         DOLOG_SHORT( replace_sourcestate_println_short(ss); );
2837
2838         /* avoid infinite loops by self-replacement */
2839
2840         frame = ss->frames;
2841         while (frame->down)
2842                 frame = frame->down;
2843
2844         if (frame->torp == origrp) {
2845                 DOLOG_SHORT(
2846                         printf("WARNING: identity replacement, turning off rps to avoid infinite loop\n");
2847                 );
2848                 replace_deactivate_replacement_points(frame->tocode);
2849         }
2850
2851         /* write execution state of new code */
2852
2853         DOLOG( replace_executionstate_println(es); );
2854
2855         /* allocate a safe stack area and copy all needed data there */
2856
2857         safestack = replace_alloc_safestack();
2858
2859         safestack->es = *es;
2860         safestack->ss = ss;
2861         safestack->dumpsize = dumpsize;
2862
2863         /* call the assembler code for the last phase of replacement */
2864
2865 #if (defined(__I386__) || defined(__X86_64__) || defined(__ALPHA__) || defined(__POWERPC__) || defined(__MIPS__) || defined(__S390__)) && defined(ENABLE_JIT)
2866         /*asm_replacement_in(&(safestack->es), safestack);*/
2867 #endif
2868
2869         abort(); /* NOT REACHED */
2870 }
2871
2872
2873 /******************************************************************************/
2874 /* NOTE: No important code below.                                             */
2875 /******************************************************************************/
2876
2877
2878 /* statistics *****************************************************************/
2879
2880 #if defined(REPLACE_STATISTICS)
2881 static void print_freq(FILE *file,int *array,int limit)
2882 {
2883         int i;
2884         int sum = 0;
2885         int cum = 0;
2886         for (i=0; i<limit; ++i)
2887                 sum += array[i];
2888         sum += array[limit];
2889         for (i=0; i<limit; ++i) {
2890                 cum += array[i];
2891                 fprintf(file,"      %3d: %8d (cum %3d%%)\n",
2892                                 i, array[i], (sum) ? ((100*cum)/sum) : 0);
2893         }
2894         fprintf(file,"    >=%3d: %8d\n",limit,array[limit]);
2895 }
2896 #endif /* defined(REPLACE_STATISTICS) */
2897
2898
2899 #if defined(REPLACE_STATISTICS)
2900
2901 #define REPLACE_PRINT_DIST(name, array)                              \
2902     printf("    " name " distribution:\n");                          \
2903     print_freq(stdout, (array), sizeof(array)/sizeof(int) - 1);
2904
2905 void replace_print_statistics(void)
2906 {
2907         printf("replacement statistics:\n");
2908         printf("    # of replacements:   %d\n", stat_replacements);
2909         printf("    # of frames:         %d\n", stat_frames);
2910         printf("    # of recompilations: %d\n", stat_recompile);
2911         printf("    patched static calls:%d\n", stat_staticpatch);
2912         printf("    unrolled inlines:    %d\n", stat_unroll_inline);
2913         printf("    unrolled calls:      %d\n", stat_unroll_call);
2914         REPLACE_PRINT_DIST("frame depth", stat_dist_frames);
2915         REPLACE_PRINT_DIST("locals per frame", stat_dist_locals);
2916         REPLACE_PRINT_DIST("ADR locals per frame", stat_dist_locals_adr);
2917         REPLACE_PRINT_DIST("primitive locals per frame", stat_dist_locals_prim);
2918         REPLACE_PRINT_DIST("RET locals per frame", stat_dist_locals_ret);
2919         REPLACE_PRINT_DIST("void locals per frame", stat_dist_locals_void);
2920         REPLACE_PRINT_DIST("stack slots per frame", stat_dist_stack);
2921         REPLACE_PRINT_DIST("ADR stack slots per frame", stat_dist_stack_adr);
2922         REPLACE_PRINT_DIST("primitive stack slots per frame", stat_dist_stack_prim);
2923         REPLACE_PRINT_DIST("RET stack slots per frame", stat_dist_stack_ret);
2924         printf("\n");
2925         printf("    # of methods:            %d\n", stat_methods);
2926         printf("    # of replacement points: %d\n", stat_rploints);
2927         printf("    # of regallocs:          %d\n", stat_regallocs);
2928         printf("        per rplpoint:        %f\n", (double)stat_regallocs / stat_rploints);
2929         printf("        per method:          %f\n", (double)stat_regallocs / stat_methods);
2930         REPLACE_PRINT_DIST("replacement points per method", stat_dist_method_rplpoints);
2931         printf("\n");
2932
2933 }
2934 #endif /* defined(REPLACE_STATISTICS) */
2935
2936
2937 #if defined(REPLACE_STATISTICS)
2938 static void replace_statistics_source_frame(sourceframe_t *frame)
2939 {
2940         int adr = 0;
2941         int ret = 0;
2942         int prim = 0;
2943         int vd = 0;
2944         int n = 0;
2945         int i;
2946
2947         for (i=0; i<frame->javalocalcount; ++i) {
2948                 switch (frame->javalocaltype[i]) {
2949                         case TYPE_ADR: adr++; break;
2950                         case TYPE_RET: ret++; break;
2951                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
2952                         case TYPE_VOID: vd++; break;
2953                         default: assert(0);
2954                 }
2955                 n++;
2956         }
2957         REPLACE_COUNT_DIST(stat_dist_locals, n);
2958         REPLACE_COUNT_DIST(stat_dist_locals_adr, adr);
2959         REPLACE_COUNT_DIST(stat_dist_locals_void, vd);
2960         REPLACE_COUNT_DIST(stat_dist_locals_ret, ret);
2961         REPLACE_COUNT_DIST(stat_dist_locals_prim, prim);
2962         adr = ret = prim = n = 0;
2963         for (i=0; i<frame->javastackdepth; ++i) {
2964                 switch (frame->javastacktype[i]) {
2965                         case TYPE_ADR: adr++; break;
2966                         case TYPE_RET: ret++; break;
2967                         case TYPE_INT: case TYPE_LNG: case TYPE_FLT: case TYPE_DBL: prim++; break;
2968                 }
2969                 n++;
2970         }
2971         REPLACE_COUNT_DIST(stat_dist_stack, n);
2972         REPLACE_COUNT_DIST(stat_dist_stack_adr, adr);
2973         REPLACE_COUNT_DIST(stat_dist_stack_ret, ret);
2974         REPLACE_COUNT_DIST(stat_dist_stack_prim, prim);
2975 }
2976 #endif /* defined(REPLACE_STATISTICS) */
2977
2978
2979 /* debugging helpers **********************************************************/
2980
2981 /* replace_replacement_point_println *******************************************
2982
2983    Print replacement point info.
2984
2985    IN:
2986        rp...............the replacement point to print
2987
2988 *******************************************************************************/
2989
2990 #if !defined(NDEBUG)
2991
2992 #define TYPECHAR(t)  (((t) >= 0 && (t) <= TYPE_RET) ? show_jit_type_letters[t] : '?')
2993
2994 static char *replace_type_str[] = {
2995         "STD",
2996         "EXH",
2997         "SBR",
2998         "CALL",
2999         "INLINE",
3000         "RETURN",
3001         "BODY"
3002 };
3003
3004 void replace_replacement_point_println(rplpoint *rp, int depth)
3005 {
3006         int j;
3007         int index;
3008
3009         if (!rp) {
3010                 printf("(rplpoint *)NULL\n");
3011                 return;
3012         }
3013
3014         for (j=0; j<depth; ++j)
3015                 putchar('\t');
3016
3017         printf("rplpoint (id %d) %p pc:%p+%d type:%s",
3018                         rp->id, (void*)rp,rp->pc,rp->callsize,
3019                         replace_type_str[rp->type]);
3020         if (rp->flags & RPLPOINT_FLAG_NOTRAP)
3021                 printf(" NOTRAP");
3022         if (rp->flags & RPLPOINT_FLAG_COUNTDOWN)
3023                 printf(" COUNTDOWN");
3024         if (rp->flags & RPLPOINT_FLAG_ACTIVE)
3025                 printf(" ACTIVE");
3026         printf(" parent:%p\n", (void*)rp->parent);
3027         for (j=0; j<depth; ++j)
3028                 putchar('\t');
3029         printf("ra:%d = [",     rp->regalloccount);
3030
3031         for (j=0; j<rp->regalloccount; ++j) {
3032                 if (j)
3033                         putchar(' ');
3034                 index = rp->regalloc[j].index;
3035                 switch (index) {
3036                         case RPLALLOC_STACK: printf("S"); break;
3037                         case RPLALLOC_PARAM: printf("P"); break;
3038                         case RPLALLOC_SYNC : printf("Y"); break;
3039                         default: printf("%d", index);
3040                 }
3041                 printf(":%1c:", TYPECHAR(rp->regalloc[j].type));
3042                 if (rp->regalloc[j].type == TYPE_RET) {
3043                         printf("ret(L%03d)", rp->regalloc[j].regoff);
3044                 }
3045                 else {
3046                         show_allocation(rp->regalloc[j].type, rp->regalloc[j].flags, rp->regalloc[j].regoff);
3047                 }
3048         }
3049
3050         printf("]\n");
3051         for (j=0; j<depth; ++j)
3052                 putchar('\t');
3053         printf("method: ");
3054         method_print(rp->method);
3055
3056         printf("\n");
3057 }
3058 #endif /* !defined(NDEBUG) */
3059
3060
3061 /* replace_show_replacement_points *********************************************
3062
3063    Print replacement point info.
3064
3065    IN:
3066        code.............codeinfo whose replacement points should be printed.
3067
3068 *******************************************************************************/
3069
3070 #if !defined(NDEBUG)
3071 void replace_show_replacement_points(codeinfo *code)
3072 {
3073         int i;
3074         int depth;
3075         rplpoint *rp;
3076         rplpoint *parent;
3077
3078         if (!code) {
3079                 printf("(codeinfo *)NULL\n");
3080                 return;
3081         }
3082
3083         printf("\treplacement points: %d\n",code->rplpointcount);
3084
3085         printf("\ttotal allocations : %d\n",code->regalloccount);
3086         printf("\tsaved int regs    : %d\n",code->savedintcount);
3087         printf("\tsaved flt regs    : %d\n",code->savedfltcount);
3088 #if defined(HAS_ADDRESS_REGISTER_FILE)
3089         printf("\tsaved adr regs    : %d\n",code->savedadrcount);
3090 #endif
3091         printf("\tmemuse            : %d\n",code->memuse);
3092
3093         printf("\n");
3094
3095         for (i=0; i<code->rplpointcount; ++i) {
3096                 rp = code->rplpoints + i;
3097
3098                 depth = 1;
3099                 parent = rp->parent;
3100                 while (parent) {
3101                         depth++;
3102                         parent = parent->parent;
3103                 }
3104                 replace_replacement_point_println(rp, depth);
3105         }
3106 }
3107 #endif
3108
3109
3110 /* replace_executionstate_println **********************************************
3111
3112    Print execution state
3113
3114    IN:
3115        es...............the execution state to print
3116
3117 *******************************************************************************/
3118
3119 #if !defined(NDEBUG)
3120 void replace_executionstate_println(executionstate_t *es)
3121 {
3122         int i;
3123         int slots;
3124         stackslot_t *sp;
3125         int extraslots;
3126
3127         if (!es) {
3128                 printf("(executionstate_t *)NULL\n");
3129                 return;
3130         }
3131
3132         printf("executionstate_t:\n");
3133         printf("\tpc = %p",(void*)es->pc);
3134         printf("  sp = %p",(void*)es->sp);
3135         printf("  pv = %p\n",(void*)es->pv);
3136 #if defined(ENABLE_DISASSEMBLER)
3137         for (i=0; i<INT_REG_CNT; ++i) {
3138                 if (i%4 == 0)
3139                         printf("\t");
3140                 else
3141                         printf(" ");
3142 #if SIZEOF_VOID_P == 8
3143                 printf("%-3s = %016llx",abi_registers_integer_name[i],(unsigned long long)es->intregs[i]);
3144 #else
3145                 printf("%-3s = %08lx",abi_registers_integer_name[i],(unsigned long)es->intregs[i]);
3146 #endif
3147                 if (i%4 == 3)
3148                         printf("\n");
3149         }
3150         for (i=0; i<FLT_REG_CNT; ++i) {
3151                 if (i%4 == 0)
3152                         printf("\t");
3153                 else
3154                         printf(" ");
3155                 printf("F%02d = %016llx",i,(unsigned long long)es->fltregs[i]);
3156                 if (i%4 == 3)
3157                         printf("\n");
3158         }
3159 # if defined(HAS_ADDRESS_REGISTER_FILE)
3160         for (i=0; i<ADR_REG_CNT; ++i) {
3161                 if (i%4 == 0)
3162                         printf("\t");
3163                 else
3164                         printf(" ");
3165                 printf("A%02d = %016llx",i,(unsigned long long)es->adrregs[i]);
3166                 if (i%4 == 3)
3167                         printf("\n");
3168         }
3169 # endif
3170 #endif
3171
3172         sp = (stackslot_t *) es->sp;
3173
3174         extraslots = 2;
3175
3176         if (es->code) {
3177                 methoddesc *md = es->code->m->parseddesc;
3178                 slots = es->code->stackframesize;
3179                 extraslots = 1 + md->memuse;
3180         }
3181         else
3182                 slots = 0;
3183
3184
3185         if (slots) {
3186                 printf("\tstack slots(+%d) at sp:", extraslots);
3187                 for (i=0; i<slots+extraslots; ++i) {
3188                         if (i%4 == 0)
3189                                 printf("\n\t\t");
3190                         printf("M%02d%c", i, (i >= slots) ? '(' : ' ');
3191 #ifdef HAS_4BYTE_STACKSLOT
3192                         printf("%08lx",(unsigned long)*sp++);
3193 #else
3194                         printf("%016llx",(unsigned long long)*sp++);
3195 #endif
3196                         printf("%c", (i >= slots) ? ')' : ' ');
3197                 }
3198                 printf("\n");
3199         }
3200
3201         printf("\tcode: %p", (void*)es->code);
3202         if (es->code != NULL) {
3203                 printf(" stackframesize=%d ", es->code->stackframesize);
3204                 method_print(es->code->m);
3205         }
3206         printf("\n");
3207
3208         printf("\n");
3209 }
3210 #endif
3211
3212 #if !defined(NDEBUG)
3213 static void java_value_print(s4 type, replace_val_t value)
3214 {
3215         java_object_t *obj;
3216         utf           *u;
3217
3218         printf("%016llx",(unsigned long long) value.l);
3219
3220         if (type < 0 || type > TYPE_RET)
3221                 printf(" <INVALID TYPE:%d>", type);
3222         else
3223                 printf(" %s", show_jit_type_names[type]);
3224
3225         if (type == TYPE_ADR && value.a != NULL) {
3226                 obj = value.a;
3227                 putchar(' ');
3228                 utf_display_printable_ascii_classname(obj->vftbl->class->name);
3229
3230                 if (obj->vftbl->class == class_java_lang_String) {
3231                         printf(" \"");
3232                         u = javastring_toutf(obj, false);
3233                         utf_display_printable_ascii(u);
3234                         printf("\"");
3235                 }
3236         }
3237         else if (type == TYPE_INT) {
3238                 printf(" %ld", (long) value.i);
3239         }
3240         else if (type == TYPE_LNG) {
3241                 printf(" %lld", (long long) value.l);
3242         }
3243         else if (type == TYPE_FLT) {
3244                 printf(" %f", value.f);
3245         }
3246         else if (type == TYPE_DBL) {
3247                 printf(" %f", value.d);
3248         }
3249 }
3250 #endif /* !defined(NDEBUG) */
3251
3252
3253 #if !defined(NDEBUG)
3254 void replace_source_frame_println(sourceframe_t *frame)
3255 {
3256         s4 i,j;
3257         s4 t;
3258
3259         if (REPLACE_IS_NATIVE_FRAME(frame)) {
3260                 printf("\tNATIVE\n");
3261                 printf("\tsfi: "); replace_stackframeinfo_println(frame->sfi);
3262                 printf("\tnativepc: %p\n", frame->nativepc);
3263                 printf("\tframesize: %d\n", frame->nativeframesize);
3264
3265                 j = 0;
3266                 for (i=0; i<INT_REG_CNT; ++i) {
3267                         if (nregdescint[i] == REG_SAV)
3268                                 printf("\t%s = %p\n", abi_registers_integer_name[i], (void*)frame->nativesavint[j++]);
3269                 }
3270
3271                 j = 0;
3272                 for (i=0; i<FLT_REG_CNT; ++i) {
3273                         if (nregdescfloat[i] == REG_SAV)
3274                                 printf("\tF%02d = %f\n", i, frame->nativesavflt[j++]);
3275                 }
3276
3277                 printf("\n");
3278                 return;
3279         }
3280
3281         printf("\t");
3282         method_println(frame->method);
3283         printf("\tid: %d\n", frame->id);
3284         printf("\ttype: %s\n", replace_type_str[frame->type]);
3285         printf("\n");
3286
3287         if (frame->instance.a) {
3288                 printf("\tinstance: ");
3289                 java_value_print(TYPE_ADR, frame->instance);
3290                 printf("\n");
3291         }
3292
3293         if (frame->javalocalcount) {
3294                 printf("\tlocals (%d):\n",frame->javalocalcount);
3295                 for (i=0; i<frame->javalocalcount; ++i) {
3296                         t = frame->javalocaltype[i];
3297                         if (t == TYPE_VOID) {
3298                                 printf("\tlocal[ %2d] = void\n",i);
3299                         }
3300                         else {
3301                                 printf("\tlocal[%c%2d] = ",TYPECHAR(t),i);
3302                                 java_value_print(t, frame->javalocals[i]);
3303                                 printf("\n");
3304                         }
3305                 }
3306                 printf("\n");
3307         }
3308
3309         if (frame->javastackdepth) {
3310                 printf("\tstack (depth %d):\n",frame->javastackdepth);
3311                 for (i=0; i<frame->javastackdepth; ++i) {
3312                         t = frame->javastacktype[i];
3313                         if (t == TYPE_VOID) {
3314                                 printf("\tstack[%2d] = void", i);
3315                         }
3316                         else {
3317                                 printf("\tstack[%2d] = ",i);
3318                                 java_value_print(frame->javastacktype[i], frame->javastack[i]);
3319                                 printf("\n");
3320                         }
3321                 }
3322                 printf("\n");
3323         }
3324
3325         if (frame->syncslotcount) {
3326                 printf("\tsynchronization slots (%d):\n",frame->syncslotcount);
3327                 for (i=0; i<frame->syncslotcount; ++i) {
3328                         printf("\tslot[%2d] = ",i);
3329 #ifdef HAS_4BYTE_STACKSLOT
3330                         printf("%08lx\n",(unsigned long) frame->syncslots[i].p);
3331 #else
3332                         printf("%016llx\n",(unsigned long long) frame->syncslots[i].p);
3333 #endif
3334                 }
3335                 printf("\n");
3336         }
3337
3338         if (frame->fromcode) {
3339                 printf("\tfrom %p ", (void*)frame->fromcode);
3340                 method_println(frame->fromcode->m);
3341         }
3342         if (frame->tocode) {
3343                 printf("\tto %p ", (void*)frame->tocode);
3344                 method_println(frame->tocode->m);
3345         }
3346
3347         if (frame->fromrp) {
3348                 printf("\tfrom replacement point:\n");
3349                 replace_replacement_point_println(frame->fromrp, 2);
3350         }
3351         if (frame->torp) {
3352                 printf("\tto replacement point:\n");
3353                 replace_replacement_point_println(frame->torp, 2);
3354         }
3355
3356         printf("\n");
3357 }
3358 #endif /* !defined(NDEBUG) */
3359
3360
3361 /* replace_sourcestate_println *************************************************
3362
3363    Print source state
3364
3365    IN:
3366        ss...............the source state to print
3367
3368 *******************************************************************************/
3369
3370 #if !defined(NDEBUG)
3371 void replace_sourcestate_println(sourcestate_t *ss)
3372 {
3373         int i;
3374         sourceframe_t *frame;
3375
3376         if (!ss) {
3377                 printf("(sourcestate_t *)NULL\n");
3378                 return;
3379         }
3380
3381         printf("sourcestate_t:\n");
3382
3383         for (i=0, frame = ss->frames; frame != NULL; frame = frame->down, ++i) {
3384                 printf("    frame %d:\n", i);
3385                 replace_source_frame_println(frame);
3386         }
3387 }
3388 #endif
3389
3390
3391 /* replace_sourcestate_println_short *******************************************
3392
3393    Print a compact representation of the given source state.
3394
3395    IN:
3396        ss...............the source state to print
3397
3398 *******************************************************************************/
3399
3400 #if !defined(NDEBUG)
3401 void replace_sourcestate_println_short(sourcestate_t *ss)
3402 {
3403         sourceframe_t *frame;
3404
3405         for (frame = ss->frames; frame != NULL; frame = frame->down) {
3406                 printf("\t");
3407
3408                 if (REPLACE_IS_NATIVE_FRAME(frame)) {
3409                         printf("NATIVE (pc %p size %d) ",
3410                                         (void*)frame->nativepc, frame->nativeframesize);
3411                         replace_stackframeinfo_println(frame->sfi);
3412                         continue;
3413                 }
3414
3415                 if (frame->torp) {
3416                         printf("%c", (frame->torp == frame->fromrp) ? '=' : '+');
3417                 }
3418
3419                 printf("%s", replace_type_str[frame->fromrp->type]);
3420
3421                 if (frame->torp && frame->torp->type != frame->fromrp->type)
3422                         printf("->%s", replace_type_str[frame->torp->type]);
3423
3424                 if (frame->tocode != frame->fromcode)
3425                         printf(" (%p->%p/%d) ",
3426                                    (void*) frame->fromcode, (void*) frame->tocode,
3427                                    frame->fromrp->id);
3428                 else
3429                         printf(" (%p/%d) ", (void*) frame->fromcode, frame->fromrp->id);
3430
3431                 method_println(frame->method);
3432         }
3433 }
3434 #endif
3435
3436 #if !defined(NDEBUG)
3437 static void replace_stackframeinfo_println(stackframeinfo *sfi)
3438 {
3439         printf("prev=%p pv=%p sp=%p ra=%p xpc=%p method=",
3440                         (void*)sfi->prev, (void*)sfi->pv, (void*)sfi->sp,
3441                         (void*)sfi->ra, (void*)sfi->xpc);
3442
3443         if (sfi->method)
3444                 method_println(sfi->method);
3445         else
3446                 printf("(nil)\n");
3447 }
3448 #endif
3449
3450 /*
3451  * These are local overrides for various environment variables in Emacs.
3452  * Please do not remove this and leave it at the end of the file, where
3453  * Emacs will automagically detect them.
3454  * ---------------------------------------------------------------------
3455  * Local variables:
3456  * mode: c
3457  * indent-tabs-mode: t
3458  * c-basic-offset: 4
3459  * tab-width: 4
3460  * End:
3461  * vim:noexpandtab:sw=4:ts=4:
3462  */