mono/utils/valgrind.h

   1 /* -*- c -*-
   2    ----------------------------------------------------------------
   3
   4    Notice that the following BSD-style license applies to this one
   5    file (valgrind.h) only.  The rest of Valgrind is licensed under the
   6    terms of the GNU General Public License, version 2, unless
   7    otherwise indicated.  See the COPYING file in the source
   8    distribution for details.
   9
  10    ----------------------------------------------------------------
  11
  12    This file is part of Valgrind, a dynamic binary instrumentation
  13    framework.
  14
  15    Copyright (C) 2000-2009 Julian Seward.  All rights reserved.
  16
  17    Redistribution and use in source and binary forms, with or without
  18    modification, are permitted provided that the following conditions
  19    are met:
  20
  21    1. Redistributions of source code must retain the above copyright
  22       notice, this list of conditions and the following disclaimer.
  23
  24    2. The origin of this software must not be misrepresented; you must
  25       not claim that you wrote the original software.  If you use this
  26       software in a product, an acknowledgment in the product
  27       documentation would be appreciated but is not required.
  28
  29    3. Altered source versions must be plainly marked as such, and must
  30       not be misrepresented as being the original software.
  31
  32    4. The name of the author may not be used to endorse or promote
  33       products derived from this software without specific prior written
  34       permission.
  35
  36    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  37    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  38    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  40    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  42    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  43    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  44    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  45    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  46    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47
  48    ----------------------------------------------------------------
  49
  50    Notice that the above BSD-style license applies to this one file
  51    (valgrind.h) only.  The entire rest of Valgrind is licensed under
  52    the terms of the GNU General Public License, version 2.  See the
  53    COPYING file in the source distribution for details.
  54
  55    ----------------------------------------------------------------
  56 */
  57
  58
  59 /* This file is for inclusion into client (your!) code.
  60
  61    You can use these macros to manipulate and query Valgrind's
  62    execution inside your own programs.
  63
  64    The resulting executables will still run without Valgrind, just a
  65    little bit more slowly than they otherwise would, but otherwise
  66    unchanged.  When not running on valgrind, each client request
  67    consumes very few (eg. 7) instructions, so the resulting performance
  68    loss is negligible unless you plan to execute client requests
  69    millions of times per second.  Nevertheless, if that is still a
  70    problem, you can compile with the NVALGRIND symbol defined (gcc
  71    -DNVALGRIND) so that client requests are not even compiled in.  */
  72
  73 #ifndef __VALGRIND_H
  74 #define __VALGRIND_H
  75
  76 #ifdef __GNUC__
  77
  78 #include <stdarg.h>
  79
  80 /* Nb: this file might be included in a file compiled with -ansi.  So
  81    we can't use C++ style "//" comments nor the "asm" keyword (instead
  82    use "__asm__"). */
  83
  84 /* Derive some tags indicating what the target platform is.  Note
  85    that in this file we're using the compiler's CPP symbols for
  86    identifying architectures, which are different to the ones we use
  87    within the rest of Valgrind.  Note, __powerpc__ is active for both
  88    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
  89    latter (on Linux, that is). */
  90 #undef PLAT_x86_linux
  91 #undef PLAT_amd64_linux
  92 #undef PLAT_ppc32_linux
  93 #undef PLAT_ppc64_linux
  94 #undef PLAT_ppc32_aix5
  95 #undef PLAT_ppc64_aix5
  96
  97
  98 #if defined(_AIX) && defined(__64BIT__)
  99 #  define PLAT_ppc64_aix5 1
 100 #elif defined(_AIX) && !defined(__64BIT__)
 101 #  define PLAT_ppc32_aix5 1
 102 #elif defined(__APPLE__) && defined(__i386__)
 103 #  define PLAT_x86_darwin 1
 104 #elif defined(__APPLE__) && defined(__x86_64__)
 105 #  define PLAT_amd64_darwin 1
 106 #elif defined(__i386__)
 107 #  define PLAT_x86_linux 1
 108 #elif defined(__x86_64__)
 109 #  define PLAT_amd64_linux 1
 110 #elif defined(__powerpc__) && !defined(__powerpc64__)
 111 #  define PLAT_ppc32_linux 1
 112 #elif defined(__powerpc__) && defined(__powerpc64__)
 113 #  define PLAT_ppc64_linux 1
 114 #else
 115 /* If we're not compiling for our target platform, don't generate
 116    any inline asms.  */
 117 #  if !defined(NVALGRIND)
 118 #    define NVALGRIND 1
 119 #  endif
 120 #endif
 121
 122
 123 /* ------------------------------------------------------------------ */
 124 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
 125 /* in here of use to end-users -- skip to the next section.           */
 126 /* ------------------------------------------------------------------ */
 127
 128 #if defined(NVALGRIND)
 129
 130 /* Define NVALGRIND to completely remove the Valgrind magic sequence
 131    from the compiled code (analogous to NDEBUG's effects on
 132    assert()) */
 133 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 134         _zzq_rlval, _zzq_default, _zzq_request,                   \
 135         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 136    {                                                              \
 137       (_zzq_rlval) = (_zzq_default);                              \
 138    }
 139
 140 #else  /* ! NVALGRIND */
 141
 142 /* The following defines the magic code sequences which the JITter
 143    spots and handles magically.  Don't look too closely at them as
 144    they will rot your brain.
 145
 146    The assembly code sequences for all architectures is in this one
 147    file.  This is because this file must be stand-alone, and we don't
 148    want to have multiple files.
 149
 150    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
 151    value gets put in the return slot, so that everything works when
 152    this is executed not under Valgrind.  Args are passed in a memory
 153    block, and so there's no intrinsic limit to the number that could
 154    be passed, but it's currently five.
 155
 156    The macro args are:
 157       _zzq_rlval    result lvalue
 158       _zzq_default  default value (result returned when running on real CPU)
 159       _zzq_request  request code
 160       _zzq_arg1..5  request params
 161
 162    The other two macros are used to support function wrapping, and are
 163    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
 164    guest's NRADDR pseudo-register and whatever other information is
 165    needed to safely run the call original from the wrapper: on
 166    ppc64-linux, the R2 value at the divert point is also needed.  This
 167    information is abstracted into a user-visible type, OrigFn.
 168
 169    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
 170    guest, but guarantees that the branch instruction will not be
 171    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
 172    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
 173    complete inline asm, since it needs to be combined with more magic
 174    inline asm stuff to be useful.
 175 */
 176
 177 /* ------------------------- x86-{linux,darwin} ---------------- */
 178
 179 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 180
 181 typedef
 182    struct {
 183       unsigned int nraddr; /* where's the code? */
 184    }
 185    OrigFn;
 186
 187 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 188                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
 189                      "roll $29, %%edi ; roll $19, %%edi\n\t"
 190
 191 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 192         _zzq_rlval, _zzq_default, _zzq_request,                   \
 193         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 194   { volatile unsigned int _zzq_args[6];                           \
 195     volatile unsigned int _zzq_result;                            \
 196     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 197     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 198     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 199     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 200     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 201     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 202     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 203                      /* %EDX = client_request ( %EAX ) */         \
 204                      "xchgl %%ebx,%%ebx"                          \
 205                      : "=d" (_zzq_result)                         \
 206                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 207                      : "cc", "memory"                             \
 208                     );                                            \
 209     _zzq_rlval = _zzq_result;                                     \
 210   }
 211
 212 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 213   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 214     volatile unsigned int __addr;                                 \
 215     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 216                      /* %EAX = guest_NRADDR */                    \
 217                      "xchgl %%ecx,%%ecx"                          \
 218                      : "=a" (__addr)                              \
 219                      :                                            \
 220                      : "cc", "memory"                             \
 221                     );                                            \
 222     _zzq_orig->nraddr = __addr;                                   \
 223   }
 224
 225 #define VALGRIND_CALL_NOREDIR_EAX                                 \
 226                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 227                      /* call-noredir *%EAX */                     \
 228                      "xchgl %%edx,%%edx\n\t"
 229 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
 230
 231 /* ------------------------ amd64-{linux,darwin} --------------- */
 232
 233 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
 234
 235 typedef
 236    struct {
 237       unsigned long long int nraddr; /* where's the code? */
 238    }
 239    OrigFn;
 240
 241 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 242                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
 243                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
 244
 245 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 246         _zzq_rlval, _zzq_default, _zzq_request,                   \
 247         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 248   { volatile unsigned long long int _zzq_args[6];                 \
 249     volatile unsigned long long int _zzq_result;                  \
 250     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 251     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 252     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 253     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 254     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 255     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 256     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 257                      /* %RDX = client_request ( %RAX ) */         \
 258                      "xchgq %%rbx,%%rbx"                          \
 259                      : "=d" (_zzq_result)                         \
 260                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 261                      : "cc", "memory"                             \
 262                     );                                            \
 263     _zzq_rlval = _zzq_result;                                     \
 264   }
 265
 266 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 267   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 268     volatile unsigned long long int __addr;                       \
 269     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 270                      /* %RAX = guest_NRADDR */                    \
 271                      "xchgq %%rcx,%%rcx"                          \
 272                      : "=a" (__addr)                              \
 273                      :                                            \
 274                      : "cc", "memory"                             \
 275                     );                                            \
 276     _zzq_orig->nraddr = __addr;                                   \
 277   }
 278
 279 #define VALGRIND_CALL_NOREDIR_RAX                                 \
 280                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 281                      /* call-noredir *%RAX */                     \
 282                      "xchgq %%rdx,%%rdx\n\t"
 283 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
 284
 285 /* ------------------------ ppc32-linux ------------------------ */
 286
 287 #if defined(PLAT_ppc32_linux)
 288
 289 typedef
 290    struct {
 291       unsigned int nraddr; /* where's the code? */
 292    }
 293    OrigFn;
 294
 295 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 296                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 297                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 298
 299 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 300         _zzq_rlval, _zzq_default, _zzq_request,                   \
 301         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 302                                                                   \
 303   {          unsigned int  _zzq_args[6];                          \
 304              unsigned int  _zzq_result;                           \
 305              unsigned int* _zzq_ptr;                              \
 306     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 307     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 308     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 309     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 310     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 311     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 312     _zzq_ptr = _zzq_args;                                         \
 313     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
 314                      "mr 4,%2\n\t" /*ptr*/                        \
 315                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 316                      /* %R3 = client_request ( %R4 ) */           \
 317                      "or 1,1,1\n\t"                               \
 318                      "mr %0,3"     /*result*/                     \
 319                      : "=b" (_zzq_result)                         \
 320                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
 321                      : "cc", "memory", "r3", "r4");               \
 322     _zzq_rlval = _zzq_result;                                     \
 323   }
 324
 325 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 326   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 327     unsigned int __addr;                                          \
 328     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 329                      /* %R3 = guest_NRADDR */                     \
 330                      "or 2,2,2\n\t"                               \
 331                      "mr %0,3"                                    \
 332                      : "=b" (__addr)                              \
 333                      :                                            \
 334                      : "cc", "memory", "r3"                       \
 335                     );                                            \
 336     _zzq_orig->nraddr = __addr;                                   \
 337   }
 338
 339 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 340                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 341                      /* branch-and-link-to-noredir *%R11 */       \
 342                      "or 3,3,3\n\t"
 343 #endif /* PLAT_ppc32_linux */
 344
 345 /* ------------------------ ppc64-linux ------------------------ */
 346
 347 #if defined(PLAT_ppc64_linux)
 348
 349 typedef
 350    struct {
 351       unsigned long long int nraddr; /* where's the code? */
 352       unsigned long long int r2;  /* what tocptr do we need? */
 353    }
 354    OrigFn;
 355
 356 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 357                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 358                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 359
 360 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 361         _zzq_rlval, _zzq_default, _zzq_request,                   \
 362         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 363                                                                   \
 364   {          unsigned long long int  _zzq_args[6];                \
 365     register unsigned long long int  _zzq_result __asm__("r3");   \
 366     register unsigned long long int* _zzq_ptr __asm__("r4");      \
 367     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 368     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 369     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 370     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 371     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 372     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 373     _zzq_ptr = _zzq_args;                                         \
 374     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 375                      /* %R3 = client_request ( %R4 ) */           \
 376                      "or 1,1,1"                                   \
 377                      : "=r" (_zzq_result)                         \
 378                      : "0" (_zzq_default), "r" (_zzq_ptr)         \
 379                      : "cc", "memory");                           \
 380     _zzq_rlval = _zzq_result;                                     \
 381   }
 382
 383 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 384   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 385     register unsigned long long int __addr __asm__("r3");         \
 386     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 387                      /* %R3 = guest_NRADDR */                     \
 388                      "or 2,2,2"                                   \
 389                      : "=r" (__addr)                              \
 390                      :                                            \
 391                      : "cc", "memory"                             \
 392                     );                                            \
 393     _zzq_orig->nraddr = __addr;                                   \
 394     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 395                      /* %R3 = guest_NRADDR_GPR2 */                \
 396                      "or 4,4,4"                                   \
 397                      : "=r" (__addr)                              \
 398                      :                                            \
 399                      : "cc", "memory"                             \
 400                     );                                            \
 401     _zzq_orig->r2 = __addr;                                       \
 402   }
 403
 404 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 405                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 406                      /* branch-and-link-to-noredir *%R11 */       \
 407                      "or 3,3,3\n\t"
 408
 409 #endif /* PLAT_ppc64_linux */
 410
 411 /* ------------------------ ppc32-aix5 ------------------------- */
 412
 413 #if defined(PLAT_ppc32_aix5)
 414
 415 typedef
 416    struct {
 417       unsigned int nraddr; /* where's the code? */
 418       unsigned int r2;  /* what tocptr do we need? */
 419    }
 420    OrigFn;
 421
 422 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 423                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 424                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 425
 426 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 427         _zzq_rlval, _zzq_default, _zzq_request,                   \
 428         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 429                                                                   \
 430   {          unsigned int  _zzq_args[7];                          \
 431     register unsigned int  _zzq_result;                           \
 432     register unsigned int* _zzq_ptr;                              \
 433     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 434     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 435     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 436     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 437     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 438     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 439     _zzq_args[6] = (unsigned int)(_zzq_default);                  \
 440     _zzq_ptr = _zzq_args;                                         \
 441     __asm__ volatile("mr 4,%1\n\t"                                \
 442                      "lwz 3, 24(4)\n\t"                           \
 443                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 444                      /* %R3 = client_request ( %R4 ) */           \
 445                      "or 1,1,1\n\t"                               \
 446                      "mr %0,3"                                    \
 447                      : "=b" (_zzq_result)                         \
 448                      : "b" (_zzq_ptr)                             \
 449                      : "r3", "r4", "cc", "memory");               \
 450     _zzq_rlval = _zzq_result;                                     \
 451   }
 452
 453 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 454   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 455     register unsigned int __addr;                                 \
 456     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 457                      /* %R3 = guest_NRADDR */                     \
 458                      "or 2,2,2\n\t"                               \
 459                      "mr %0,3"                                    \
 460                      : "=b" (__addr)                              \
 461                      :                                            \
 462                      : "r3", "cc", "memory"                       \
 463                     );                                            \
 464     _zzq_orig->nraddr = __addr;                                   \
 465     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 466                      /* %R3 = guest_NRADDR_GPR2 */                \
 467                      "or 4,4,4\n\t"                               \
 468                      "mr %0,3"                                    \
 469                      : "=b" (__addr)                              \
 470                      :                                            \
 471                      : "r3", "cc", "memory"                       \
 472                     );                                            \
 473     _zzq_orig->r2 = __addr;                                       \
 474   }
 475
 476 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 477                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 478                      /* branch-and-link-to-noredir *%R11 */       \
 479                      "or 3,3,3\n\t"
 480
 481 #endif /* PLAT_ppc32_aix5 */
 482
 483 /* ------------------------ ppc64-aix5 ------------------------- */
 484
 485 #if defined(PLAT_ppc64_aix5)
 486
 487 typedef
 488    struct {
 489       unsigned long long int nraddr; /* where's the code? */
 490       unsigned long long int r2;  /* what tocptr do we need? */
 491    }
 492    OrigFn;
 493
 494 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 495                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 496                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 497
 498 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 499         _zzq_rlval, _zzq_default, _zzq_request,                   \
 500         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 501                                                                   \
 502   {          unsigned long long int  _zzq_args[7];                \
 503     register unsigned long long int  _zzq_result;                 \
 504     register unsigned long long int* _zzq_ptr;                    \
 505     _zzq_args[0] = (unsigned int long long)(_zzq_request);        \
 506     _zzq_args[1] = (unsigned int long long)(_zzq_arg1);           \
 507     _zzq_args[2] = (unsigned int long long)(_zzq_arg2);           \
 508     _zzq_args[3] = (unsigned int long long)(_zzq_arg3);           \
 509     _zzq_args[4] = (unsigned int long long)(_zzq_arg4);           \
 510     _zzq_args[5] = (unsigned int long long)(_zzq_arg5);           \
 511     _zzq_args[6] = (unsigned int long long)(_zzq_default);        \
 512     _zzq_ptr = _zzq_args;                                         \
 513     __asm__ volatile("mr 4,%1\n\t"                                \
 514                      "ld 3, 48(4)\n\t"                            \
 515                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 516                      /* %R3 = client_request ( %R4 ) */           \
 517                      "or 1,1,1\n\t"                               \
 518                      "mr %0,3"                                    \
 519                      : "=b" (_zzq_result)                         \
 520                      : "b" (_zzq_ptr)                             \
 521                      : "r3", "r4", "cc", "memory");               \
 522     _zzq_rlval = _zzq_result;                                     \
 523   }
 524
 525 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 526   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 527     register unsigned long long int __addr;                       \
 528     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 529                      /* %R3 = guest_NRADDR */                     \
 530                      "or 2,2,2\n\t"                               \
 531                      "mr %0,3"                                    \
 532                      : "=b" (__addr)                              \
 533                      :                                            \
 534                      : "r3", "cc", "memory"                       \
 535                     );                                            \
 536     _zzq_orig->nraddr = __addr;                                   \
 537     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 538                      /* %R3 = guest_NRADDR_GPR2 */                \
 539                      "or 4,4,4\n\t"                               \
 540                      "mr %0,3"                                    \
 541                      : "=b" (__addr)                              \
 542                      :                                            \
 543                      : "r3", "cc", "memory"                       \
 544                     );                                            \
 545     _zzq_orig->r2 = __addr;                                       \
 546   }
 547
 548 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 549                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 550                      /* branch-and-link-to-noredir *%R11 */       \
 551                      "or 3,3,3\n\t"
 552
 553 #endif /* PLAT_ppc64_aix5 */
 554
 555 /* Insert assembly code for other platforms here... */
 556
 557 #endif /* NVALGRIND */
 558
 559
 560 /* ------------------------------------------------------------------ */
 561 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
 562 /* ugly.  It's the least-worst tradeoff I can think of.               */
 563 /* ------------------------------------------------------------------ */
 564
 565 /* This section defines magic (a.k.a appalling-hack) macros for doing
 566    guaranteed-no-redirection macros, so as to get from function
 567    wrappers to the functions they are wrapping.  The whole point is to
 568    construct standard call sequences, but to do the call itself with a
 569    special no-redirect call pseudo-instruction that the JIT
 570    understands and handles specially.  This section is long and
 571    repetitious, and I can't see a way to make it shorter.
 572
 573    The naming scheme is as follows:
 574
 575       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
 576
 577    'W' stands for "word" and 'v' for "void".  Hence there are
 578    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
 579    and for each, the possibility of returning a word-typed result, or
 580    no result.
 581 */
 582
 583 /* Use these to write the name of your wrapper.  NOTE: duplicates
 584    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */
 585
 586 /* Use an extra level of macroisation so as to ensure the soname/fnname
 587    args are fully macro-expanded before pasting them together. */
 588 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
 589
 590 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
 591    VG_CONCAT4(_vgwZU_,soname,_,fnname)
 592
 593 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
 594    VG_CONCAT4(_vgwZZ_,soname,_,fnname)
 595
 596 /* Use this macro from within a wrapper function to collect the
 597    context (address and possibly other info) of the original function.
 598    Once you have that you can then use it in one of the CALL_FN_
 599    macros.  The type of the argument _lval is OrigFn. */
 600 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
 601
 602 /* Derivatives of the main macros below, for calling functions
 603    returning void. */
 604
 605 #define CALL_FN_v_v(fnptr)                                        \
 606    do { volatile unsigned long _junk;                             \
 607         CALL_FN_W_v(_junk,fnptr); } while (0)
 608
 609 #define CALL_FN_v_W(fnptr, arg1)                                  \
 610    do { volatile unsigned long _junk;                             \
 611         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
 612
 613 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
 614    do { volatile unsigned long _junk;                             \
 615         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
 616
 617 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
 618    do { volatile unsigned long _junk;                             \
 619         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
 620
 621 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
 622    do { volatile unsigned long _junk;                             \
 623         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
 624
 625 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
 626    do { volatile unsigned long _junk;                             \
 627         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
 628
 629 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
 630    do { volatile unsigned long _junk;                             \
 631         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
 632
 633 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
 634    do { volatile unsigned long _junk;                             \
 635         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
 636
 637 /* ------------------------- x86-{linux,darwin} ---------------- */
 638
 639 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 640
 641 /* These regs are trashed by the hidden call.  No need to mention eax
 642    as gcc can already see that, plus causes gcc to bomb. */
 643 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
 644
 645 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
 646    long) == 4. */
 647
 648 #define CALL_FN_W_v(lval, orig)                                   \
 649    do {                                                           \
 650       volatile OrigFn        _orig = (orig);                      \
 651       volatile unsigned long _argvec[1];                          \
 652       volatile unsigned long _res;                                \
 653       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 654       __asm__ volatile(                                           \
 655          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 656          VALGRIND_CALL_NOREDIR_EAX                                \
 657          : /*out*/   "=a" (_res)                                  \
 658          : /*in*/    "a" (&_argvec[0])                            \
 659          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 660       );                                                          \
 661       lval = (__typeof__(lval)) _res;                             \
 662    } while (0)
 663
 664 #define CALL_FN_W_W(lval, orig, arg1)                             \
 665    do {                                                           \
 666       volatile OrigFn        _orig = (orig);                      \
 667       volatile unsigned long _argvec[2];                          \
 668       volatile unsigned long _res;                                \
 669       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 670       _argvec[1] = (unsigned long)(arg1);                         \
 671       __asm__ volatile(                                           \
 672          "pushl 4(%%eax)\n\t"                                     \
 673          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 674          VALGRIND_CALL_NOREDIR_EAX                                \
 675          "addl $4, %%esp\n"                                       \
 676          : /*out*/   "=a" (_res)                                  \
 677          : /*in*/    "a" (&_argvec[0])                            \
 678          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 679       );                                                          \
 680       lval = (__typeof__(lval)) _res;                             \
 681    } while (0)
 682
 683 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
 684    do {                                                           \
 685       volatile OrigFn        _orig = (orig);                      \
 686       volatile unsigned long _argvec[3];                          \
 687       volatile unsigned long _res;                                \
 688       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 689       _argvec[1] = (unsigned long)(arg1);                         \
 690       _argvec[2] = (unsigned long)(arg2);                         \
 691       __asm__ volatile(                                           \
 692          "pushl 8(%%eax)\n\t"                                     \
 693          "pushl 4(%%eax)\n\t"                                     \
 694          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 695          VALGRIND_CALL_NOREDIR_EAX                                \
 696          "addl $8, %%esp\n"                                       \
 697          : /*out*/   "=a" (_res)                                  \
 698          : /*in*/    "a" (&_argvec[0])                            \
 699          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 700       );                                                          \
 701       lval = (__typeof__(lval)) _res;                             \
 702    } while (0)
 703
 704 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
 705    do {                                                           \
 706       volatile OrigFn        _orig = (orig);                      \
 707       volatile unsigned long _argvec[4];                          \
 708       volatile unsigned long _res;                                \
 709       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 710       _argvec[1] = (unsigned long)(arg1);                         \
 711       _argvec[2] = (unsigned long)(arg2);                         \
 712       _argvec[3] = (unsigned long)(arg3);                         \
 713       __asm__ volatile(                                           \
 714          "pushl 12(%%eax)\n\t"                                    \
 715          "pushl 8(%%eax)\n\t"                                     \
 716          "pushl 4(%%eax)\n\t"                                     \
 717          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 718          VALGRIND_CALL_NOREDIR_EAX                                \
 719          "addl $12, %%esp\n"                                      \
 720          : /*out*/   "=a" (_res)                                  \
 721          : /*in*/    "a" (&_argvec[0])                            \
 722          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 723       );                                                          \
 724       lval = (__typeof__(lval)) _res;                             \
 725    } while (0)
 726
 727 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
 728    do {                                                           \
 729       volatile OrigFn        _orig = (orig);                      \
 730       volatile unsigned long _argvec[5];                          \
 731       volatile unsigned long _res;                                \
 732       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 733       _argvec[1] = (unsigned long)(arg1);                         \
 734       _argvec[2] = (unsigned long)(arg2);                         \
 735       _argvec[3] = (unsigned long)(arg3);                         \
 736       _argvec[4] = (unsigned long)(arg4);                         \
 737       __asm__ volatile(                                           \
 738          "pushl 16(%%eax)\n\t"                                    \
 739          "pushl 12(%%eax)\n\t"                                    \
 740          "pushl 8(%%eax)\n\t"                                     \
 741          "pushl 4(%%eax)\n\t"                                     \
 742          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 743          VALGRIND_CALL_NOREDIR_EAX                                \
 744          "addl $16, %%esp\n"                                      \
 745          : /*out*/   "=a" (_res)                                  \
 746          : /*in*/    "a" (&_argvec[0])                            \
 747          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 748       );                                                          \
 749       lval = (__typeof__(lval)) _res;                             \
 750    } while (0)
 751
 752 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
 753    do {                                                           \
 754       volatile OrigFn        _orig = (orig);                      \
 755       volatile unsigned long _argvec[6];                          \
 756       volatile unsigned long _res;                                \
 757       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 758       _argvec[1] = (unsigned long)(arg1);                         \
 759       _argvec[2] = (unsigned long)(arg2);                         \
 760       _argvec[3] = (unsigned long)(arg3);                         \
 761       _argvec[4] = (unsigned long)(arg4);                         \
 762       _argvec[5] = (unsigned long)(arg5);                         \
 763       __asm__ volatile(                                           \
 764          "pushl 20(%%eax)\n\t"                                    \
 765          "pushl 16(%%eax)\n\t"                                    \
 766          "pushl 12(%%eax)\n\t"                                    \
 767          "pushl 8(%%eax)\n\t"                                     \
 768          "pushl 4(%%eax)\n\t"                                     \
 769          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 770          VALGRIND_CALL_NOREDIR_EAX                                \
 771          "addl $20, %%esp\n"                                      \
 772          : /*out*/   "=a" (_res)                                  \
 773          : /*in*/    "a" (&_argvec[0])                            \
 774          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 775       );                                                          \
 776       lval = (__typeof__(lval)) _res;                             \
 777    } while (0)
 778
 779 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
 780    do {                                                           \
 781       volatile OrigFn        _orig = (orig);                      \
 782       volatile unsigned long _argvec[7];                          \
 783       volatile unsigned long _res;                                \
 784       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 785       _argvec[1] = (unsigned long)(arg1);                         \
 786       _argvec[2] = (unsigned long)(arg2);                         \
 787       _argvec[3] = (unsigned long)(arg3);                         \
 788       _argvec[4] = (unsigned long)(arg4);                         \
 789       _argvec[5] = (unsigned long)(arg5);                         \
 790       _argvec[6] = (unsigned long)(arg6);                         \
 791       __asm__ volatile(                                           \
 792          "pushl 24(%%eax)\n\t"                                    \
 793          "pushl 20(%%eax)\n\t"                                    \
 794          "pushl 16(%%eax)\n\t"                                    \
 795          "pushl 12(%%eax)\n\t"                                    \
 796          "pushl 8(%%eax)\n\t"                                     \
 797          "pushl 4(%%eax)\n\t"                                     \
 798          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 799          VALGRIND_CALL_NOREDIR_EAX                                \
 800          "addl $24, %%esp\n"                                      \
 801          : /*out*/   "=a" (_res)                                  \
 802          : /*in*/    "a" (&_argvec[0])                            \
 803          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 804       );                                                          \
 805       lval = (__typeof__(lval)) _res;                             \
 806    } while (0)
 807
 808 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 809                                  arg7)                            \
 810    do {                                                           \
 811       volatile OrigFn        _orig = (orig);                      \
 812       volatile unsigned long _argvec[8];                          \
 813       volatile unsigned long _res;                                \
 814       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 815       _argvec[1] = (unsigned long)(arg1);                         \
 816       _argvec[2] = (unsigned long)(arg2);                         \
 817       _argvec[3] = (unsigned long)(arg3);                         \
 818       _argvec[4] = (unsigned long)(arg4);                         \
 819       _argvec[5] = (unsigned long)(arg5);                         \
 820       _argvec[6] = (unsigned long)(arg6);                         \
 821       _argvec[7] = (unsigned long)(arg7);                         \
 822       __asm__ volatile(                                           \
 823          "pushl 28(%%eax)\n\t"                                    \
 824          "pushl 24(%%eax)\n\t"                                    \
 825          "pushl 20(%%eax)\n\t"                                    \
 826          "pushl 16(%%eax)\n\t"                                    \
 827          "pushl 12(%%eax)\n\t"                                    \
 828          "pushl 8(%%eax)\n\t"                                     \
 829          "pushl 4(%%eax)\n\t"                                     \
 830          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 831          VALGRIND_CALL_NOREDIR_EAX                                \
 832          "addl $28, %%esp\n"                                      \
 833          : /*out*/   "=a" (_res)                                  \
 834          : /*in*/    "a" (&_argvec[0])                            \
 835          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 836       );                                                          \
 837       lval = (__typeof__(lval)) _res;                             \
 838    } while (0)
 839
 840 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 841                                  arg7,arg8)                       \
 842    do {                                                           \
 843       volatile OrigFn        _orig = (orig);                      \
 844       volatile unsigned long _argvec[9];                          \
 845       volatile unsigned long _res;                                \
 846       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 847       _argvec[1] = (unsigned long)(arg1);                         \
 848       _argvec[2] = (unsigned long)(arg2);                         \
 849       _argvec[3] = (unsigned long)(arg3);                         \
 850       _argvec[4] = (unsigned long)(arg4);                         \
 851       _argvec[5] = (unsigned long)(arg5);                         \
 852       _argvec[6] = (unsigned long)(arg6);                         \
 853       _argvec[7] = (unsigned long)(arg7);                         \
 854       _argvec[8] = (unsigned long)(arg8);                         \
 855       __asm__ volatile(                                           \
 856          "pushl 32(%%eax)\n\t"                                    \
 857          "pushl 28(%%eax)\n\t"                                    \
 858          "pushl 24(%%eax)\n\t"                                    \
 859          "pushl 20(%%eax)\n\t"                                    \
 860          "pushl 16(%%eax)\n\t"                                    \
 861          "pushl 12(%%eax)\n\t"                                    \
 862          "pushl 8(%%eax)\n\t"                                     \
 863          "pushl 4(%%eax)\n\t"                                     \
 864          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 865          VALGRIND_CALL_NOREDIR_EAX                                \
 866          "addl $32, %%esp\n"                                      \
 867          : /*out*/   "=a" (_res)                                  \
 868          : /*in*/    "a" (&_argvec[0])                            \
 869          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 870       );                                                          \
 871       lval = (__typeof__(lval)) _res;                             \
 872    } while (0)
 873
 874 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 875                                  arg7,arg8,arg9)                  \
 876    do {                                                           \
 877       volatile OrigFn        _orig = (orig);                      \
 878       volatile unsigned long _argvec[10];                         \
 879       volatile unsigned long _res;                                \
 880       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 881       _argvec[1] = (unsigned long)(arg1);                         \
 882       _argvec[2] = (unsigned long)(arg2);                         \
 883       _argvec[3] = (unsigned long)(arg3);                         \
 884       _argvec[4] = (unsigned long)(arg4);                         \
 885       _argvec[5] = (unsigned long)(arg5);                         \
 886       _argvec[6] = (unsigned long)(arg6);                         \
 887       _argvec[7] = (unsigned long)(arg7);                         \
 888       _argvec[8] = (unsigned long)(arg8);                         \
 889       _argvec[9] = (unsigned long)(arg9);                         \
 890       __asm__ volatile(                                           \
 891          "pushl 36(%%eax)\n\t"                                    \
 892          "pushl 32(%%eax)\n\t"                                    \
 893          "pushl 28(%%eax)\n\t"                                    \
 894          "pushl 24(%%eax)\n\t"                                    \
 895          "pushl 20(%%eax)\n\t"                                    \
 896          "pushl 16(%%eax)\n\t"                                    \
 897          "pushl 12(%%eax)\n\t"                                    \
 898          "pushl 8(%%eax)\n\t"                                     \
 899          "pushl 4(%%eax)\n\t"                                     \
 900          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 901          VALGRIND_CALL_NOREDIR_EAX                                \
 902          "addl $36, %%esp\n"                                      \
 903          : /*out*/   "=a" (_res)                                  \
 904          : /*in*/    "a" (&_argvec[0])                            \
 905          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 906       );                                                          \
 907       lval = (__typeof__(lval)) _res;                             \
 908    } while (0)
 909
 910 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
 911                                   arg7,arg8,arg9,arg10)           \
 912    do {                                                           \
 913       volatile OrigFn        _orig = (orig);                      \
 914       volatile unsigned long _argvec[11];                         \
 915       volatile unsigned long _res;                                \
 916       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 917       _argvec[1] = (unsigned long)(arg1);                         \
 918       _argvec[2] = (unsigned long)(arg2);                         \
 919       _argvec[3] = (unsigned long)(arg3);                         \
 920       _argvec[4] = (unsigned long)(arg4);                         \
 921       _argvec[5] = (unsigned long)(arg5);                         \
 922       _argvec[6] = (unsigned long)(arg6);                         \
 923       _argvec[7] = (unsigned long)(arg7);                         \
 924       _argvec[8] = (unsigned long)(arg8);                         \
 925       _argvec[9] = (unsigned long)(arg9);                         \
 926       _argvec[10] = (unsigned long)(arg10);                       \
 927       __asm__ volatile(                                           \
 928          "pushl 40(%%eax)\n\t"                                    \
 929          "pushl 36(%%eax)\n\t"                                    \
 930          "pushl 32(%%eax)\n\t"                                    \
 931          "pushl 28(%%eax)\n\t"                                    \
 932          "pushl 24(%%eax)\n\t"                                    \
 933          "pushl 20(%%eax)\n\t"                                    \
 934          "pushl 16(%%eax)\n\t"                                    \
 935          "pushl 12(%%eax)\n\t"                                    \
 936          "pushl 8(%%eax)\n\t"                                     \
 937          "pushl 4(%%eax)\n\t"                                     \
 938          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 939          VALGRIND_CALL_NOREDIR_EAX                                \
 940          "addl $40, %%esp\n"                                      \
 941          : /*out*/   "=a" (_res)                                  \
 942          : /*in*/    "a" (&_argvec[0])                            \
 943          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 944       );                                                          \
 945       lval = (__typeof__(lval)) _res;                             \
 946    } while (0)
 947
 948 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
 949                                   arg6,arg7,arg8,arg9,arg10,      \
 950                                   arg11)                          \
 951    do {                                                           \
 952       volatile OrigFn        _orig = (orig);                      \
 953       volatile unsigned long _argvec[12];                         \
 954       volatile unsigned long _res;                                \
 955       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 956       _argvec[1] = (unsigned long)(arg1);                         \
 957       _argvec[2] = (unsigned long)(arg2);                         \
 958       _argvec[3] = (unsigned long)(arg3);                         \
 959       _argvec[4] = (unsigned long)(arg4);                         \
 960       _argvec[5] = (unsigned long)(arg5);                         \
 961       _argvec[6] = (unsigned long)(arg6);                         \
 962       _argvec[7] = (unsigned long)(arg7);                         \
 963       _argvec[8] = (unsigned long)(arg8);                         \
 964       _argvec[9] = (unsigned long)(arg9);                         \
 965       _argvec[10] = (unsigned long)(arg10);                       \
 966       _argvec[11] = (unsigned long)(arg11);                       \
 967       __asm__ volatile(                                           \
 968          "pushl 44(%%eax)\n\t"                                    \
 969          "pushl 40(%%eax)\n\t"                                    \
 970          "pushl 36(%%eax)\n\t"                                    \
 971          "pushl 32(%%eax)\n\t"                                    \
 972          "pushl 28(%%eax)\n\t"                                    \
 973          "pushl 24(%%eax)\n\t"                                    \
 974          "pushl 20(%%eax)\n\t"                                    \
 975          "pushl 16(%%eax)\n\t"                                    \
 976          "pushl 12(%%eax)\n\t"                                    \
 977          "pushl 8(%%eax)\n\t"                                     \
 978          "pushl 4(%%eax)\n\t"                                     \
 979          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 980          VALGRIND_CALL_NOREDIR_EAX                                \
 981          "addl $44, %%esp\n"                                      \
 982          : /*out*/   "=a" (_res)                                  \
 983          : /*in*/    "a" (&_argvec[0])                            \
 984          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 985       );                                                          \
 986       lval = (__typeof__(lval)) _res;                             \
 987    } while (0)
 988
 989 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
 990                                   arg6,arg7,arg8,arg9,arg10,      \
 991                                   arg11,arg12)                    \
 992    do {                                                           \
 993       volatile OrigFn        _orig = (orig);                      \
 994       volatile unsigned long _argvec[13];                         \
 995       volatile unsigned long _res;                                \
 996       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 997       _argvec[1] = (unsigned long)(arg1);                         \
 998       _argvec[2] = (unsigned long)(arg2);                         \
 999       _argvec[3] = (unsigned long)(arg3);                         \
1000       _argvec[4] = (unsigned long)(arg4);                         \
1001       _argvec[5] = (unsigned long)(arg5);                         \
1002       _argvec[6] = (unsigned long)(arg6);                         \
1003       _argvec[7] = (unsigned long)(arg7);                         \
1004       _argvec[8] = (unsigned long)(arg8);                         \
1005       _argvec[9] = (unsigned long)(arg9);                         \
1006       _argvec[10] = (unsigned long)(arg10);                       \
1007       _argvec[11] = (unsigned long)(arg11);                       \
1008       _argvec[12] = (unsigned long)(arg12);                       \
1009       __asm__ volatile(                                           \
1010          "pushl 48(%%eax)\n\t"                                    \
1011          "pushl 44(%%eax)\n\t"                                    \
1012          "pushl 40(%%eax)\n\t"                                    \
1013          "pushl 36(%%eax)\n\t"                                    \
1014          "pushl 32(%%eax)\n\t"                                    \
1015          "pushl 28(%%eax)\n\t"                                    \
1016          "pushl 24(%%eax)\n\t"                                    \
1017          "pushl 20(%%eax)\n\t"                                    \
1018          "pushl 16(%%eax)\n\t"                                    \
1019          "pushl 12(%%eax)\n\t"                                    \
1020          "pushl 8(%%eax)\n\t"                                     \
1021          "pushl 4(%%eax)\n\t"                                     \
1022          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1023          VALGRIND_CALL_NOREDIR_EAX                                \
1024          "addl $48, %%esp\n"                                      \
1025          : /*out*/   "=a" (_res)                                  \
1026          : /*in*/    "a" (&_argvec[0])                            \
1027          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1028       );                                                          \
1029       lval = (__typeof__(lval)) _res;                             \
1030    } while (0)
1031
1032 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
1033
1034 /* ------------------------ amd64-{linux,darwin} --------------- */
1035
1036 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
1037
1038 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1039
1040 /* These regs are trashed by the hidden call. */
1041 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1042                             "rdi", "r8", "r9", "r10", "r11"
1043
1044 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1045    long) == 8. */
1046
1047 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1048    macros.  In order not to trash the stack redzone, we need to drop
1049    %rsp by 128 before the hidden call, and restore afterwards.  The
1050    nastyness is that it is only by luck that the stack still appears
1051    to be unwindable during the hidden call - since then the behaviour
1052    of any routine using this macro does not match what the CFI data
1053    says.  Sigh.
1054
1055    Why is this important?  Imagine that a wrapper has a stack
1056    allocated local, and passes to the hidden call, a pointer to it.
1057    Because gcc does not know about the hidden call, it may allocate
1058    that local in the redzone.  Unfortunately the hidden call may then
1059    trash it before it comes to use it.  So we must step clear of the
1060    redzone, for the duration of the hidden call, to make it safe.
1061
1062    Probably the same problem afflicts the other redzone-style ABIs too
1063    (ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is
1064    self describing (none of this CFI nonsense) so at least messing
1065    with the stack pointer doesn't give a danger of non-unwindable
1066    stack. */
1067
1068 #define CALL_FN_W_v(lval, orig)                                   \
1069    do {                                                           \
1070       volatile OrigFn        _orig = (orig);                      \
1071       volatile unsigned long _argvec[1];                          \
1072       volatile unsigned long _res;                                \
1073       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1074       __asm__ volatile(                                           \
1075          "subq $128,%%rsp\n\t"                                    \
1076          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1077          VALGRIND_CALL_NOREDIR_RAX                                \
1078          "addq $128,%%rsp\n\t"                                    \
1079          : /*out*/   "=a" (_res)                                  \
1080          : /*in*/    "a" (&_argvec[0])                            \
1081          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1082       );                                                          \
1083       lval = (__typeof__(lval)) _res;                             \
1084    } while (0)
1085
1086 #define CALL_FN_W_W(lval, orig, arg1)                             \
1087    do {                                                           \
1088       volatile OrigFn        _orig = (orig);                      \
1089       volatile unsigned long _argvec[2];                          \
1090       volatile unsigned long _res;                                \
1091       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1092       _argvec[1] = (unsigned long)(arg1);                         \
1093       __asm__ volatile(                                           \
1094          "subq $128,%%rsp\n\t"                                    \
1095          "movq 8(%%rax), %%rdi\n\t"                               \
1096          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1097          VALGRIND_CALL_NOREDIR_RAX                                \
1098          "addq $128,%%rsp\n\t"                                    \
1099          : /*out*/   "=a" (_res)                                  \
1100          : /*in*/    "a" (&_argvec[0])                            \
1101          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1102       );                                                          \
1103       lval = (__typeof__(lval)) _res;                             \
1104    } while (0)
1105
1106 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1107    do {                                                           \
1108       volatile OrigFn        _orig = (orig);                      \
1109       volatile unsigned long _argvec[3];                          \
1110       volatile unsigned long _res;                                \
1111       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1112       _argvec[1] = (unsigned long)(arg1);                         \
1113       _argvec[2] = (unsigned long)(arg2);                         \
1114       __asm__ volatile(                                           \
1115          "subq $128,%%rsp\n\t"                                    \
1116          "movq 16(%%rax), %%rsi\n\t"                              \
1117          "movq 8(%%rax), %%rdi\n\t"                               \
1118          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1119          VALGRIND_CALL_NOREDIR_RAX                                \
1120          "addq $128,%%rsp\n\t"                                    \
1121          : /*out*/   "=a" (_res)                                  \
1122          : /*in*/    "a" (&_argvec[0])                            \
1123          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1124       );                                                          \
1125       lval = (__typeof__(lval)) _res;                             \
1126    } while (0)
1127
1128 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1129    do {                                                           \
1130       volatile OrigFn        _orig = (orig);                      \
1131       volatile unsigned long _argvec[4];                          \
1132       volatile unsigned long _res;                                \
1133       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1134       _argvec[1] = (unsigned long)(arg1);                         \
1135       _argvec[2] = (unsigned long)(arg2);                         \
1136       _argvec[3] = (unsigned long)(arg3);                         \
1137       __asm__ volatile(                                           \
1138          "subq $128,%%rsp\n\t"                                    \
1139          "movq 24(%%rax), %%rdx\n\t"                              \
1140          "movq 16(%%rax), %%rsi\n\t"                              \
1141          "movq 8(%%rax), %%rdi\n\t"                               \
1142          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1143          VALGRIND_CALL_NOREDIR_RAX                                \
1144          "addq $128,%%rsp\n\t"                                    \
1145          : /*out*/   "=a" (_res)                                  \
1146          : /*in*/    "a" (&_argvec[0])                            \
1147          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1148       );                                                          \
1149       lval = (__typeof__(lval)) _res;                             \
1150    } while (0)
1151
1152 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1153    do {                                                           \
1154       volatile OrigFn        _orig = (orig);                      \
1155       volatile unsigned long _argvec[5];                          \
1156       volatile unsigned long _res;                                \
1157       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1158       _argvec[1] = (unsigned long)(arg1);                         \
1159       _argvec[2] = (unsigned long)(arg2);                         \
1160       _argvec[3] = (unsigned long)(arg3);                         \
1161       _argvec[4] = (unsigned long)(arg4);                         \
1162       __asm__ volatile(                                           \
1163          "subq $128,%%rsp\n\t"                                    \
1164          "movq 32(%%rax), %%rcx\n\t"                              \
1165          "movq 24(%%rax), %%rdx\n\t"                              \
1166          "movq 16(%%rax), %%rsi\n\t"                              \
1167          "movq 8(%%rax), %%rdi\n\t"                               \
1168          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1169          VALGRIND_CALL_NOREDIR_RAX                                \
1170          "addq $128,%%rsp\n\t"                                    \
1171          : /*out*/   "=a" (_res)                                  \
1172          : /*in*/    "a" (&_argvec[0])                            \
1173          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1174       );                                                          \
1175       lval = (__typeof__(lval)) _res;                             \
1176    } while (0)
1177
1178 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1179    do {                                                           \
1180       volatile OrigFn        _orig = (orig);                      \
1181       volatile unsigned long _argvec[6];                          \
1182       volatile unsigned long _res;                                \
1183       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1184       _argvec[1] = (unsigned long)(arg1);                         \
1185       _argvec[2] = (unsigned long)(arg2);                         \
1186       _argvec[3] = (unsigned long)(arg3);                         \
1187       _argvec[4] = (unsigned long)(arg4);                         \
1188       _argvec[5] = (unsigned long)(arg5);                         \
1189       __asm__ volatile(                                           \
1190          "subq $128,%%rsp\n\t"                                    \
1191          "movq 40(%%rax), %%r8\n\t"                               \
1192          "movq 32(%%rax), %%rcx\n\t"                              \
1193          "movq 24(%%rax), %%rdx\n\t"                              \
1194          "movq 16(%%rax), %%rsi\n\t"                              \
1195          "movq 8(%%rax), %%rdi\n\t"                               \
1196          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1197          VALGRIND_CALL_NOREDIR_RAX                                \
1198          "addq $128,%%rsp\n\t"                                    \
1199          : /*out*/   "=a" (_res)                                  \
1200          : /*in*/    "a" (&_argvec[0])                            \
1201          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1202       );                                                          \
1203       lval = (__typeof__(lval)) _res;                             \
1204    } while (0)
1205
1206 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1207    do {                                                           \
1208       volatile OrigFn        _orig = (orig);                      \
1209       volatile unsigned long _argvec[7];                          \
1210       volatile unsigned long _res;                                \
1211       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1212       _argvec[1] = (unsigned long)(arg1);                         \
1213       _argvec[2] = (unsigned long)(arg2);                         \
1214       _argvec[3] = (unsigned long)(arg3);                         \
1215       _argvec[4] = (unsigned long)(arg4);                         \
1216       _argvec[5] = (unsigned long)(arg5);                         \
1217       _argvec[6] = (unsigned long)(arg6);                         \
1218       __asm__ volatile(                                           \
1219          "subq $128,%%rsp\n\t"                                    \
1220          "movq 48(%%rax), %%r9\n\t"                               \
1221          "movq 40(%%rax), %%r8\n\t"                               \
1222          "movq 32(%%rax), %%rcx\n\t"                              \
1223          "movq 24(%%rax), %%rdx\n\t"                              \
1224          "movq 16(%%rax), %%rsi\n\t"                              \
1225          "movq 8(%%rax), %%rdi\n\t"                               \
1226          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1227          "addq $128,%%rsp\n\t"                                    \
1228          VALGRIND_CALL_NOREDIR_RAX                                \
1229          : /*out*/   "=a" (_res)                                  \
1230          : /*in*/    "a" (&_argvec[0])                            \
1231          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1232       );                                                          \
1233       lval = (__typeof__(lval)) _res;                             \
1234    } while (0)
1235
1236 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1237                                  arg7)                            \
1238    do {                                                           \
1239       volatile OrigFn        _orig = (orig);                      \
1240       volatile unsigned long _argvec[8];                          \
1241       volatile unsigned long _res;                                \
1242       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1243       _argvec[1] = (unsigned long)(arg1);                         \
1244       _argvec[2] = (unsigned long)(arg2);                         \
1245       _argvec[3] = (unsigned long)(arg3);                         \
1246       _argvec[4] = (unsigned long)(arg4);                         \
1247       _argvec[5] = (unsigned long)(arg5);                         \
1248       _argvec[6] = (unsigned long)(arg6);                         \
1249       _argvec[7] = (unsigned long)(arg7);                         \
1250       __asm__ volatile(                                           \
1251          "subq $128,%%rsp\n\t"                                    \
1252          "pushq 56(%%rax)\n\t"                                    \
1253          "movq 48(%%rax), %%r9\n\t"                               \
1254          "movq 40(%%rax), %%r8\n\t"                               \
1255          "movq 32(%%rax), %%rcx\n\t"                              \
1256          "movq 24(%%rax), %%rdx\n\t"                              \
1257          "movq 16(%%rax), %%rsi\n\t"                              \
1258          "movq 8(%%rax), %%rdi\n\t"                               \
1259          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1260          VALGRIND_CALL_NOREDIR_RAX                                \
1261          "addq $8, %%rsp\n"                                       \
1262          "addq $128,%%rsp\n\t"                                    \
1263          : /*out*/   "=a" (_res)                                  \
1264          : /*in*/    "a" (&_argvec[0])                            \
1265          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1266       );                                                          \
1267       lval = (__typeof__(lval)) _res;                             \
1268    } while (0)
1269
1270 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1271                                  arg7,arg8)                       \
1272    do {                                                           \
1273       volatile OrigFn        _orig = (orig);                      \
1274       volatile unsigned long _argvec[9];                          \
1275       volatile unsigned long _res;                                \
1276       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1277       _argvec[1] = (unsigned long)(arg1);                         \
1278       _argvec[2] = (unsigned long)(arg2);                         \
1279       _argvec[3] = (unsigned long)(arg3);                         \
1280       _argvec[4] = (unsigned long)(arg4);                         \
1281       _argvec[5] = (unsigned long)(arg5);                         \
1282       _argvec[6] = (unsigned long)(arg6);                         \
1283       _argvec[7] = (unsigned long)(arg7);                         \
1284       _argvec[8] = (unsigned long)(arg8);                         \
1285       __asm__ volatile(                                           \
1286          "subq $128,%%rsp\n\t"                                    \
1287          "pushq 64(%%rax)\n\t"                                    \
1288          "pushq 56(%%rax)\n\t"                                    \
1289          "movq 48(%%rax), %%r9\n\t"                               \
1290          "movq 40(%%rax), %%r8\n\t"                               \
1291          "movq 32(%%rax), %%rcx\n\t"                              \
1292          "movq 24(%%rax), %%rdx\n\t"                              \
1293          "movq 16(%%rax), %%rsi\n\t"                              \
1294          "movq 8(%%rax), %%rdi\n\t"                               \
1295          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1296          VALGRIND_CALL_NOREDIR_RAX                                \
1297          "addq $16, %%rsp\n"                                      \
1298          "addq $128,%%rsp\n\t"                                    \
1299          : /*out*/   "=a" (_res)                                  \
1300          : /*in*/    "a" (&_argvec[0])                            \
1301          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1302       );                                                          \
1303       lval = (__typeof__(lval)) _res;                             \
1304    } while (0)
1305
1306 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1307                                  arg7,arg8,arg9)                  \
1308    do {                                                           \
1309       volatile OrigFn        _orig = (orig);                      \
1310       volatile unsigned long _argvec[10];                         \
1311       volatile unsigned long _res;                                \
1312       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1313       _argvec[1] = (unsigned long)(arg1);                         \
1314       _argvec[2] = (unsigned long)(arg2);                         \
1315       _argvec[3] = (unsigned long)(arg3);                         \
1316       _argvec[4] = (unsigned long)(arg4);                         \
1317       _argvec[5] = (unsigned long)(arg5);                         \
1318       _argvec[6] = (unsigned long)(arg6);                         \
1319       _argvec[7] = (unsigned long)(arg7);                         \
1320       _argvec[8] = (unsigned long)(arg8);                         \
1321       _argvec[9] = (unsigned long)(arg9);                         \
1322       __asm__ volatile(                                           \
1323          "subq $128,%%rsp\n\t"                                    \
1324          "pushq 72(%%rax)\n\t"                                    \
1325          "pushq 64(%%rax)\n\t"                                    \
1326          "pushq 56(%%rax)\n\t"                                    \
1327          "movq 48(%%rax), %%r9\n\t"                               \
1328          "movq 40(%%rax), %%r8\n\t"                               \
1329          "movq 32(%%rax), %%rcx\n\t"                              \
1330          "movq 24(%%rax), %%rdx\n\t"                              \
1331          "movq 16(%%rax), %%rsi\n\t"                              \
1332          "movq 8(%%rax), %%rdi\n\t"                               \
1333          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1334          VALGRIND_CALL_NOREDIR_RAX                                \
1335          "addq $24, %%rsp\n"                                      \
1336          "addq $128,%%rsp\n\t"                                    \
1337          : /*out*/   "=a" (_res)                                  \
1338          : /*in*/    "a" (&_argvec[0])                            \
1339          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1340       );                                                          \
1341       lval = (__typeof__(lval)) _res;                             \
1342    } while (0)
1343
1344 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1345                                   arg7,arg8,arg9,arg10)           \
1346    do {                                                           \
1347       volatile OrigFn        _orig = (orig);                      \
1348       volatile unsigned long _argvec[11];                         \
1349       volatile unsigned long _res;                                \
1350       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1351       _argvec[1] = (unsigned long)(arg1);                         \
1352       _argvec[2] = (unsigned long)(arg2);                         \
1353       _argvec[3] = (unsigned long)(arg3);                         \
1354       _argvec[4] = (unsigned long)(arg4);                         \
1355       _argvec[5] = (unsigned long)(arg5);                         \
1356       _argvec[6] = (unsigned long)(arg6);                         \
1357       _argvec[7] = (unsigned long)(arg7);                         \
1358       _argvec[8] = (unsigned long)(arg8);                         \
1359       _argvec[9] = (unsigned long)(arg9);                         \
1360       _argvec[10] = (unsigned long)(arg10);                       \
1361       __asm__ volatile(                                           \
1362          "subq $128,%%rsp\n\t"                                    \
1363          "pushq 80(%%rax)\n\t"                                    \
1364          "pushq 72(%%rax)\n\t"                                    \
1365          "pushq 64(%%rax)\n\t"                                    \
1366          "pushq 56(%%rax)\n\t"                                    \
1367          "movq 48(%%rax), %%r9\n\t"                               \
1368          "movq 40(%%rax), %%r8\n\t"                               \
1369          "movq 32(%%rax), %%rcx\n\t"                              \
1370          "movq 24(%%rax), %%rdx\n\t"                              \
1371          "movq 16(%%rax), %%rsi\n\t"                              \
1372          "movq 8(%%rax), %%rdi\n\t"                               \
1373          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1374          VALGRIND_CALL_NOREDIR_RAX                                \
1375          "addq $32, %%rsp\n"                                      \
1376          "addq $128,%%rsp\n\t"                                    \
1377          : /*out*/   "=a" (_res)                                  \
1378          : /*in*/    "a" (&_argvec[0])                            \
1379          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1380       );                                                          \
1381       lval = (__typeof__(lval)) _res;                             \
1382    } while (0)
1383
1384 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1385                                   arg7,arg8,arg9,arg10,arg11)     \
1386    do {                                                           \
1387       volatile OrigFn        _orig = (orig);                      \
1388       volatile unsigned long _argvec[12];                         \
1389       volatile unsigned long _res;                                \
1390       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1391       _argvec[1] = (unsigned long)(arg1);                         \
1392       _argvec[2] = (unsigned long)(arg2);                         \
1393       _argvec[3] = (unsigned long)(arg3);                         \
1394       _argvec[4] = (unsigned long)(arg4);                         \
1395       _argvec[5] = (unsigned long)(arg5);                         \
1396       _argvec[6] = (unsigned long)(arg6);                         \
1397       _argvec[7] = (unsigned long)(arg7);                         \
1398       _argvec[8] = (unsigned long)(arg8);                         \
1399       _argvec[9] = (unsigned long)(arg9);                         \
1400       _argvec[10] = (unsigned long)(arg10);                       \
1401       _argvec[11] = (unsigned long)(arg11);                       \
1402       __asm__ volatile(                                           \
1403          "subq $128,%%rsp\n\t"                                    \
1404          "pushq 88(%%rax)\n\t"                                    \
1405          "pushq 80(%%rax)\n\t"                                    \
1406          "pushq 72(%%rax)\n\t"                                    \
1407          "pushq 64(%%rax)\n\t"                                    \
1408          "pushq 56(%%rax)\n\t"                                    \
1409          "movq 48(%%rax), %%r9\n\t"                               \
1410          "movq 40(%%rax), %%r8\n\t"                               \
1411          "movq 32(%%rax), %%rcx\n\t"                              \
1412          "movq 24(%%rax), %%rdx\n\t"                              \
1413          "movq 16(%%rax), %%rsi\n\t"                              \
1414          "movq 8(%%rax), %%rdi\n\t"                               \
1415          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1416          VALGRIND_CALL_NOREDIR_RAX                                \
1417          "addq $40, %%rsp\n"                                      \
1418          "addq $128,%%rsp\n\t"                                    \
1419          : /*out*/   "=a" (_res)                                  \
1420          : /*in*/    "a" (&_argvec[0])                            \
1421          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1422       );                                                          \
1423       lval = (__typeof__(lval)) _res;                             \
1424    } while (0)
1425
1426 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1427                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1428    do {                                                           \
1429       volatile OrigFn        _orig = (orig);                      \
1430       volatile unsigned long _argvec[13];                         \
1431       volatile unsigned long _res;                                \
1432       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1433       _argvec[1] = (unsigned long)(arg1);                         \
1434       _argvec[2] = (unsigned long)(arg2);                         \
1435       _argvec[3] = (unsigned long)(arg3);                         \
1436       _argvec[4] = (unsigned long)(arg4);                         \
1437       _argvec[5] = (unsigned long)(arg5);                         \
1438       _argvec[6] = (unsigned long)(arg6);                         \
1439       _argvec[7] = (unsigned long)(arg7);                         \
1440       _argvec[8] = (unsigned long)(arg8);                         \
1441       _argvec[9] = (unsigned long)(arg9);                         \
1442       _argvec[10] = (unsigned long)(arg10);                       \
1443       _argvec[11] = (unsigned long)(arg11);                       \
1444       _argvec[12] = (unsigned long)(arg12);                       \
1445       __asm__ volatile(                                           \
1446          "subq $128,%%rsp\n\t"                                    \
1447          "pushq 96(%%rax)\n\t"                                    \
1448          "pushq 88(%%rax)\n\t"                                    \
1449          "pushq 80(%%rax)\n\t"                                    \
1450          "pushq 72(%%rax)\n\t"                                    \
1451          "pushq 64(%%rax)\n\t"                                    \
1452          "pushq 56(%%rax)\n\t"                                    \
1453          "movq 48(%%rax), %%r9\n\t"                               \
1454          "movq 40(%%rax), %%r8\n\t"                               \
1455          "movq 32(%%rax), %%rcx\n\t"                              \
1456          "movq 24(%%rax), %%rdx\n\t"                              \
1457          "movq 16(%%rax), %%rsi\n\t"                              \
1458          "movq 8(%%rax), %%rdi\n\t"                               \
1459          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1460          VALGRIND_CALL_NOREDIR_RAX                                \
1461          "addq $48, %%rsp\n"                                      \
1462          "addq $128,%%rsp\n\t"                                    \
1463          : /*out*/   "=a" (_res)                                  \
1464          : /*in*/    "a" (&_argvec[0])                            \
1465          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1466       );                                                          \
1467       lval = (__typeof__(lval)) _res;                             \
1468    } while (0)
1469
1470 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
1471
1472 /* ------------------------ ppc32-linux ------------------------ */
1473
1474 #if defined(PLAT_ppc32_linux)
1475
1476 /* This is useful for finding out about the on-stack stuff:
1477
1478    extern int f9  ( int,int,int,int,int,int,int,int,int );
1479    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
1480    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
1481    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
1482
1483    int g9 ( void ) {
1484       return f9(11,22,33,44,55,66,77,88,99);
1485    }
1486    int g10 ( void ) {
1487       return f10(11,22,33,44,55,66,77,88,99,110);
1488    }
1489    int g11 ( void ) {
1490       return f11(11,22,33,44,55,66,77,88,99,110,121);
1491    }
1492    int g12 ( void ) {
1493       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
1494    }
1495 */
1496
1497 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1498
1499 /* These regs are trashed by the hidden call. */
1500 #define __CALLER_SAVED_REGS                                       \
1501    "lr", "ctr", "xer",                                            \
1502    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
1503    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
1504    "r11", "r12", "r13"
1505
1506 /* These CALL_FN_ macros assume that on ppc32-linux,
1507    sizeof(unsigned long) == 4. */
1508
1509 #define CALL_FN_W_v(lval, orig)                                   \
1510    do {                                                           \
1511       volatile OrigFn        _orig = (orig);                      \
1512       volatile unsigned long _argvec[1];                          \
1513       volatile unsigned long _res;                                \
1514       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1515       __asm__ volatile(                                           \
1516          "mr 11,%1\n\t"                                           \
1517          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1518          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1519          "mr %0,3"                                                \
1520          : /*out*/   "=r" (_res)                                  \
1521          : /*in*/    "r" (&_argvec[0])                            \
1522          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1523       );                                                          \
1524       lval = (__typeof__(lval)) _res;                             \
1525    } while (0)
1526
1527 #define CALL_FN_W_W(lval, orig, arg1)                             \
1528    do {                                                           \
1529       volatile OrigFn        _orig = (orig);                      \
1530       volatile unsigned long _argvec[2];                          \
1531       volatile unsigned long _res;                                \
1532       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1533       _argvec[1] = (unsigned long)arg1;                           \
1534       __asm__ volatile(                                           \
1535          "mr 11,%1\n\t"                                           \
1536          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1537          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1538          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1539          "mr %0,3"                                                \
1540          : /*out*/   "=r" (_res)                                  \
1541          : /*in*/    "r" (&_argvec[0])                            \
1542          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1543       );                                                          \
1544       lval = (__typeof__(lval)) _res;                             \
1545    } while (0)
1546
1547 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1548    do {                                                           \
1549       volatile OrigFn        _orig = (orig);                      \
1550       volatile unsigned long _argvec[3];                          \
1551       volatile unsigned long _res;                                \
1552       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1553       _argvec[1] = (unsigned long)arg1;                           \
1554       _argvec[2] = (unsigned long)arg2;                           \
1555       __asm__ volatile(                                           \
1556          "mr 11,%1\n\t"                                           \
1557          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1558          "lwz 4,8(11)\n\t"                                        \
1559          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1560          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1561          "mr %0,3"                                                \
1562          : /*out*/   "=r" (_res)                                  \
1563          : /*in*/    "r" (&_argvec[0])                            \
1564          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1565       );                                                          \
1566       lval = (__typeof__(lval)) _res;                             \
1567    } while (0)
1568
1569 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1570    do {                                                           \
1571       volatile OrigFn        _orig = (orig);                      \
1572       volatile unsigned long _argvec[4];                          \
1573       volatile unsigned long _res;                                \
1574       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1575       _argvec[1] = (unsigned long)arg1;                           \
1576       _argvec[2] = (unsigned long)arg2;                           \
1577       _argvec[3] = (unsigned long)arg3;                           \
1578       __asm__ volatile(                                           \
1579          "mr 11,%1\n\t"                                           \
1580          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1581          "lwz 4,8(11)\n\t"                                        \
1582          "lwz 5,12(11)\n\t"                                       \
1583          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1584          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1585          "mr %0,3"                                                \
1586          : /*out*/   "=r" (_res)                                  \
1587          : /*in*/    "r" (&_argvec[0])                            \
1588          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1589       );                                                          \
1590       lval = (__typeof__(lval)) _res;                             \
1591    } while (0)
1592
1593 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1594    do {                                                           \
1595       volatile OrigFn        _orig = (orig);                      \
1596       volatile unsigned long _argvec[5];                          \
1597       volatile unsigned long _res;                                \
1598       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1599       _argvec[1] = (unsigned long)arg1;                           \
1600       _argvec[2] = (unsigned long)arg2;                           \
1601       _argvec[3] = (unsigned long)arg3;                           \
1602       _argvec[4] = (unsigned long)arg4;                           \
1603       __asm__ volatile(                                           \
1604          "mr 11,%1\n\t"                                           \
1605          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1606          "lwz 4,8(11)\n\t"                                        \
1607          "lwz 5,12(11)\n\t"                                       \
1608          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1609          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1610          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1611          "mr %0,3"                                                \
1612          : /*out*/   "=r" (_res)                                  \
1613          : /*in*/    "r" (&_argvec[0])                            \
1614          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1615       );                                                          \
1616       lval = (__typeof__(lval)) _res;                             \
1617    } while (0)
1618
1619 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1620    do {                                                           \
1621       volatile OrigFn        _orig = (orig);                      \
1622       volatile unsigned long _argvec[6];                          \
1623       volatile unsigned long _res;                                \
1624       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1625       _argvec[1] = (unsigned long)arg1;                           \
1626       _argvec[2] = (unsigned long)arg2;                           \
1627       _argvec[3] = (unsigned long)arg3;                           \
1628       _argvec[4] = (unsigned long)arg4;                           \
1629       _argvec[5] = (unsigned long)arg5;                           \
1630       __asm__ volatile(                                           \
1631          "mr 11,%1\n\t"                                           \
1632          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1633          "lwz 4,8(11)\n\t"                                        \
1634          "lwz 5,12(11)\n\t"                                       \
1635          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1636          "lwz 7,20(11)\n\t"                                       \
1637          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1638          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1639          "mr %0,3"                                                \
1640          : /*out*/   "=r" (_res)                                  \
1641          : /*in*/    "r" (&_argvec[0])                            \
1642          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1643       );                                                          \
1644       lval = (__typeof__(lval)) _res;                             \
1645    } while (0)
1646
1647 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1648    do {                                                           \
1649       volatile OrigFn        _orig = (orig);                      \
1650       volatile unsigned long _argvec[7];                          \
1651       volatile unsigned long _res;                                \
1652       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1653       _argvec[1] = (unsigned long)arg1;                           \
1654       _argvec[2] = (unsigned long)arg2;                           \
1655       _argvec[3] = (unsigned long)arg3;                           \
1656       _argvec[4] = (unsigned long)arg4;                           \
1657       _argvec[5] = (unsigned long)arg5;                           \
1658       _argvec[6] = (unsigned long)arg6;                           \
1659       __asm__ volatile(                                           \
1660          "mr 11,%1\n\t"                                           \
1661          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1662          "lwz 4,8(11)\n\t"                                        \
1663          "lwz 5,12(11)\n\t"                                       \
1664          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1665          "lwz 7,20(11)\n\t"                                       \
1666          "lwz 8,24(11)\n\t"                                       \
1667          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1668          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1669          "mr %0,3"                                                \
1670          : /*out*/   "=r" (_res)                                  \
1671          : /*in*/    "r" (&_argvec[0])                            \
1672          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1673       );                                                          \
1674       lval = (__typeof__(lval)) _res;                             \
1675    } while (0)
1676
1677 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1678                                  arg7)                            \
1679    do {                                                           \
1680       volatile OrigFn        _orig = (orig);                      \
1681       volatile unsigned long _argvec[8];                          \
1682       volatile unsigned long _res;                                \
1683       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1684       _argvec[1] = (unsigned long)arg1;                           \
1685       _argvec[2] = (unsigned long)arg2;                           \
1686       _argvec[3] = (unsigned long)arg3;                           \
1687       _argvec[4] = (unsigned long)arg4;                           \
1688       _argvec[5] = (unsigned long)arg5;                           \
1689       _argvec[6] = (unsigned long)arg6;                           \
1690       _argvec[7] = (unsigned long)arg7;                           \
1691       __asm__ volatile(                                           \
1692          "mr 11,%1\n\t"                                           \
1693          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1694          "lwz 4,8(11)\n\t"                                        \
1695          "lwz 5,12(11)\n\t"                                       \
1696          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1697          "lwz 7,20(11)\n\t"                                       \
1698          "lwz 8,24(11)\n\t"                                       \
1699          "lwz 9,28(11)\n\t"                                       \
1700          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1701          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1702          "mr %0,3"                                                \
1703          : /*out*/   "=r" (_res)                                  \
1704          : /*in*/    "r" (&_argvec[0])                            \
1705          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1706       );                                                          \
1707       lval = (__typeof__(lval)) _res;                             \
1708    } while (0)
1709
1710 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1711                                  arg7,arg8)                       \
1712    do {                                                           \
1713       volatile OrigFn        _orig = (orig);                      \
1714       volatile unsigned long _argvec[9];                          \
1715       volatile unsigned long _res;                                \
1716       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1717       _argvec[1] = (unsigned long)arg1;                           \
1718       _argvec[2] = (unsigned long)arg2;                           \
1719       _argvec[3] = (unsigned long)arg3;                           \
1720       _argvec[4] = (unsigned long)arg4;                           \
1721       _argvec[5] = (unsigned long)arg5;                           \
1722       _argvec[6] = (unsigned long)arg6;                           \
1723       _argvec[7] = (unsigned long)arg7;                           \
1724       _argvec[8] = (unsigned long)arg8;                           \
1725       __asm__ volatile(                                           \
1726          "mr 11,%1\n\t"                                           \
1727          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1728          "lwz 4,8(11)\n\t"                                        \
1729          "lwz 5,12(11)\n\t"                                       \
1730          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1731          "lwz 7,20(11)\n\t"                                       \
1732          "lwz 8,24(11)\n\t"                                       \
1733          "lwz 9,28(11)\n\t"                                       \
1734          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1735          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1736          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1737          "mr %0,3"                                                \
1738          : /*out*/   "=r" (_res)                                  \
1739          : /*in*/    "r" (&_argvec[0])                            \
1740          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1741       );                                                          \
1742       lval = (__typeof__(lval)) _res;                             \
1743    } while (0)
1744
1745 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1746                                  arg7,arg8,arg9)                  \
1747    do {                                                           \
1748       volatile OrigFn        _orig = (orig);                      \
1749       volatile unsigned long _argvec[10];                         \
1750       volatile unsigned long _res;                                \
1751       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1752       _argvec[1] = (unsigned long)arg1;                           \
1753       _argvec[2] = (unsigned long)arg2;                           \
1754       _argvec[3] = (unsigned long)arg3;                           \
1755       _argvec[4] = (unsigned long)arg4;                           \
1756       _argvec[5] = (unsigned long)arg5;                           \
1757       _argvec[6] = (unsigned long)arg6;                           \
1758       _argvec[7] = (unsigned long)arg7;                           \
1759       _argvec[8] = (unsigned long)arg8;                           \
1760       _argvec[9] = (unsigned long)arg9;                           \
1761       __asm__ volatile(                                           \
1762          "mr 11,%1\n\t"                                           \
1763          "addi 1,1,-16\n\t"                                       \
1764          /* arg9 */                                               \
1765          "lwz 3,36(11)\n\t"                                       \
1766          "stw 3,8(1)\n\t"                                         \
1767          /* args1-8 */                                            \
1768          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1769          "lwz 4,8(11)\n\t"                                        \
1770          "lwz 5,12(11)\n\t"                                       \
1771          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1772          "lwz 7,20(11)\n\t"                                       \
1773          "lwz 8,24(11)\n\t"                                       \
1774          "lwz 9,28(11)\n\t"                                       \
1775          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1776          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1777          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1778          "addi 1,1,16\n\t"                                        \
1779          "mr %0,3"                                                \
1780          : /*out*/   "=r" (_res)                                  \
1781          : /*in*/    "r" (&_argvec[0])                            \
1782          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1783       );                                                          \
1784       lval = (__typeof__(lval)) _res;                             \
1785    } while (0)
1786
1787 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1788                                   arg7,arg8,arg9,arg10)           \
1789    do {                                                           \
1790       volatile OrigFn        _orig = (orig);                      \
1791       volatile unsigned long _argvec[11];                         \
1792       volatile unsigned long _res;                                \
1793       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1794       _argvec[1] = (unsigned long)arg1;                           \
1795       _argvec[2] = (unsigned long)arg2;                           \
1796       _argvec[3] = (unsigned long)arg3;                           \
1797       _argvec[4] = (unsigned long)arg4;                           \
1798       _argvec[5] = (unsigned long)arg5;                           \
1799       _argvec[6] = (unsigned long)arg6;                           \
1800       _argvec[7] = (unsigned long)arg7;                           \
1801       _argvec[8] = (unsigned long)arg8;                           \
1802       _argvec[9] = (unsigned long)arg9;                           \
1803       _argvec[10] = (unsigned long)arg10;                         \
1804       __asm__ volatile(                                           \
1805          "mr 11,%1\n\t"                                           \
1806          "addi 1,1,-16\n\t"                                       \
1807          /* arg10 */                                              \
1808          "lwz 3,40(11)\n\t"                                       \
1809          "stw 3,12(1)\n\t"                                        \
1810          /* arg9 */                                               \
1811          "lwz 3,36(11)\n\t"                                       \
1812          "stw 3,8(1)\n\t"                                         \
1813          /* args1-8 */                                            \
1814          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1815          "lwz 4,8(11)\n\t"                                        \
1816          "lwz 5,12(11)\n\t"                                       \
1817          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1818          "lwz 7,20(11)\n\t"                                       \
1819          "lwz 8,24(11)\n\t"                                       \
1820          "lwz 9,28(11)\n\t"                                       \
1821          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1822          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1823          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1824          "addi 1,1,16\n\t"                                        \
1825          "mr %0,3"                                                \
1826          : /*out*/   "=r" (_res)                                  \
1827          : /*in*/    "r" (&_argvec[0])                            \
1828          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1829       );                                                          \
1830       lval = (__typeof__(lval)) _res;                             \
1831    } while (0)
1832
1833 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1834                                   arg7,arg8,arg9,arg10,arg11)     \
1835    do {                                                           \
1836       volatile OrigFn        _orig = (orig);                      \
1837       volatile unsigned long _argvec[12];                         \
1838       volatile unsigned long _res;                                \
1839       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1840       _argvec[1] = (unsigned long)arg1;                           \
1841       _argvec[2] = (unsigned long)arg2;                           \
1842       _argvec[3] = (unsigned long)arg3;                           \
1843       _argvec[4] = (unsigned long)arg4;                           \
1844       _argvec[5] = (unsigned long)arg5;                           \
1845       _argvec[6] = (unsigned long)arg6;                           \
1846       _argvec[7] = (unsigned long)arg7;                           \
1847       _argvec[8] = (unsigned long)arg8;                           \
1848       _argvec[9] = (unsigned long)arg9;                           \
1849       _argvec[10] = (unsigned long)arg10;                         \
1850       _argvec[11] = (unsigned long)arg11;                         \
1851       __asm__ volatile(                                           \
1852          "mr 11,%1\n\t"                                           \
1853          "addi 1,1,-32\n\t"                                       \
1854          /* arg11 */                                              \
1855          "lwz 3,44(11)\n\t"                                       \
1856          "stw 3,16(1)\n\t"                                        \
1857          /* arg10 */                                              \
1858          "lwz 3,40(11)\n\t"                                       \
1859          "stw 3,12(1)\n\t"                                        \
1860          /* arg9 */                                               \
1861          "lwz 3,36(11)\n\t"                                       \
1862          "stw 3,8(1)\n\t"                                         \
1863          /* args1-8 */                                            \
1864          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1865          "lwz 4,8(11)\n\t"                                        \
1866          "lwz 5,12(11)\n\t"                                       \
1867          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1868          "lwz 7,20(11)\n\t"                                       \
1869          "lwz 8,24(11)\n\t"                                       \
1870          "lwz 9,28(11)\n\t"                                       \
1871          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1872          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1873          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1874          "addi 1,1,32\n\t"                                        \
1875          "mr %0,3"                                                \
1876          : /*out*/   "=r" (_res)                                  \
1877          : /*in*/    "r" (&_argvec[0])                            \
1878          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1879       );                                                          \
1880       lval = (__typeof__(lval)) _res;                             \
1881    } while (0)
1882
1883 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1884                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1885    do {                                                           \
1886       volatile OrigFn        _orig = (orig);                      \
1887       volatile unsigned long _argvec[13];                         \
1888       volatile unsigned long _res;                                \
1889       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1890       _argvec[1] = (unsigned long)arg1;                           \
1891       _argvec[2] = (unsigned long)arg2;                           \
1892       _argvec[3] = (unsigned long)arg3;                           \
1893       _argvec[4] = (unsigned long)arg4;                           \
1894       _argvec[5] = (unsigned long)arg5;                           \
1895       _argvec[6] = (unsigned long)arg6;                           \
1896       _argvec[7] = (unsigned long)arg7;                           \
1897       _argvec[8] = (unsigned long)arg8;                           \
1898       _argvec[9] = (unsigned long)arg9;                           \
1899       _argvec[10] = (unsigned long)arg10;                         \
1900       _argvec[11] = (unsigned long)arg11;                         \
1901       _argvec[12] = (unsigned long)arg12;                         \
1902       __asm__ volatile(                                           \
1903          "mr 11,%1\n\t"                                           \
1904          "addi 1,1,-32\n\t"                                       \
1905          /* arg12 */                                              \
1906          "lwz 3,48(11)\n\t"                                       \
1907          "stw 3,20(1)\n\t"                                        \
1908          /* arg11 */                                              \
1909          "lwz 3,44(11)\n\t"                                       \
1910          "stw 3,16(1)\n\t"                                        \
1911          /* arg10 */                                              \
1912          "lwz 3,40(11)\n\t"                                       \
1913          "stw 3,12(1)\n\t"                                        \
1914          /* arg9 */                                               \
1915          "lwz 3,36(11)\n\t"                                       \
1916          "stw 3,8(1)\n\t"                                         \
1917          /* args1-8 */                                            \
1918          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1919          "lwz 4,8(11)\n\t"                                        \
1920          "lwz 5,12(11)\n\t"                                       \
1921          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1922          "lwz 7,20(11)\n\t"                                       \
1923          "lwz 8,24(11)\n\t"                                       \
1924          "lwz 9,28(11)\n\t"                                       \
1925          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1926          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1927          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1928          "addi 1,1,32\n\t"                                        \
1929          "mr %0,3"                                                \
1930          : /*out*/   "=r" (_res)                                  \
1931          : /*in*/    "r" (&_argvec[0])                            \
1932          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1933       );                                                          \
1934       lval = (__typeof__(lval)) _res;                             \
1935    } while (0)
1936
1937 #endif /* PLAT_ppc32_linux */
1938
1939 /* ------------------------ ppc64-linux ------------------------ */
1940
1941 #if defined(PLAT_ppc64_linux)
1942
1943 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1944
1945 /* These regs are trashed by the hidden call. */
1946 #define __CALLER_SAVED_REGS                                       \
1947    "lr", "ctr", "xer",                                            \
1948    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
1949    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
1950    "r11", "r12", "r13"
1951
1952 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
1953    long) == 8. */
1954
1955 #define CALL_FN_W_v(lval, orig)                                   \
1956    do {                                                           \
1957       volatile OrigFn        _orig = (orig);                      \
1958       volatile unsigned long _argvec[3+0];                        \
1959       volatile unsigned long _res;                                \
1960       /* _argvec[0] holds current r2 across the call */           \
1961       _argvec[1] = (unsigned long)_orig.r2;                       \
1962       _argvec[2] = (unsigned long)_orig.nraddr;                   \
1963       __asm__ volatile(                                           \
1964          "mr 11,%1\n\t"                                           \
1965          "std 2,-16(11)\n\t"  /* save tocptr */                   \
1966          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
1967          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
1968          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1969          "mr 11,%1\n\t"                                           \
1970          "mr %0,3\n\t"                                            \
1971          "ld 2,-16(11)" /* restore tocptr */                      \
1972          : /*out*/   "=r" (_res)                                  \
1973          : /*in*/    "r" (&_argvec[2])                            \
1974          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1975       );                                                          \
1976       lval = (__typeof__(lval)) _res;                             \
1977    } while (0)
1978
1979 #define CALL_FN_W_W(lval, orig, arg1)                             \
1980    do {                                                           \
1981       volatile OrigFn        _orig = (orig);                      \
1982       volatile unsigned long _argvec[3+1];                        \
1983       volatile unsigned long _res;                                \
1984       /* _argvec[0] holds current r2 across the call */           \
1985       _argvec[1]   = (unsigned long)_orig.r2;                     \
1986       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
1987       _argvec[2+1] = (unsigned long)arg1;                         \
1988       __asm__ volatile(                                           \
1989          "mr 11,%1\n\t"                                           \
1990          "std 2,-16(11)\n\t"  /* save tocptr */                   \
1991          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
1992          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
1993          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
1994          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1995          "mr 11,%1\n\t"                                           \
1996          "mr %0,3\n\t"                                            \
1997          "ld 2,-16(11)" /* restore tocptr */                      \
1998          : /*out*/   "=r" (_res)                                  \
1999          : /*in*/    "r" (&_argvec[2])                            \
2000          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2001       );                                                          \
2002       lval = (__typeof__(lval)) _res;                             \
2003    } while (0)
2004
2005 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2006    do {                                                           \
2007       volatile OrigFn        _orig = (orig);                      \
2008       volatile unsigned long _argvec[3+2];                        \
2009       volatile unsigned long _res;                                \
2010       /* _argvec[0] holds current r2 across the call */           \
2011       _argvec[1]   = (unsigned long)_orig.r2;                     \
2012       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2013       _argvec[2+1] = (unsigned long)arg1;                         \
2014       _argvec[2+2] = (unsigned long)arg2;                         \
2015       __asm__ volatile(                                           \
2016          "mr 11,%1\n\t"                                           \
2017          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2018          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2019          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2020          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2021          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2022          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2023          "mr 11,%1\n\t"                                           \
2024          "mr %0,3\n\t"                                            \
2025          "ld 2,-16(11)" /* restore tocptr */                      \
2026          : /*out*/   "=r" (_res)                                  \
2027          : /*in*/    "r" (&_argvec[2])                            \
2028          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2029       );                                                          \
2030       lval = (__typeof__(lval)) _res;                             \
2031    } while (0)
2032
2033 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2034    do {                                                           \
2035       volatile OrigFn        _orig = (orig);                      \
2036       volatile unsigned long _argvec[3+3];                        \
2037       volatile unsigned long _res;                                \
2038       /* _argvec[0] holds current r2 across the call */           \
2039       _argvec[1]   = (unsigned long)_orig.r2;                     \
2040       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2041       _argvec[2+1] = (unsigned long)arg1;                         \
2042       _argvec[2+2] = (unsigned long)arg2;                         \
2043       _argvec[2+3] = (unsigned long)arg3;                         \
2044       __asm__ volatile(                                           \
2045          "mr 11,%1\n\t"                                           \
2046          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2047          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2048          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2049          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2050          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2051          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2052          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2053          "mr 11,%1\n\t"                                           \
2054          "mr %0,3\n\t"                                            \
2055          "ld 2,-16(11)" /* restore tocptr */                      \
2056          : /*out*/   "=r" (_res)                                  \
2057          : /*in*/    "r" (&_argvec[2])                            \
2058          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2059       );                                                          \
2060       lval = (__typeof__(lval)) _res;                             \
2061    } while (0)
2062
2063 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2064    do {                                                           \
2065       volatile OrigFn        _orig = (orig);                      \
2066       volatile unsigned long _argvec[3+4];                        \
2067       volatile unsigned long _res;                                \
2068       /* _argvec[0] holds current r2 across the call */           \
2069       _argvec[1]   = (unsigned long)_orig.r2;                     \
2070       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2071       _argvec[2+1] = (unsigned long)arg1;                         \
2072       _argvec[2+2] = (unsigned long)arg2;                         \
2073       _argvec[2+3] = (unsigned long)arg3;                         \
2074       _argvec[2+4] = (unsigned long)arg4;                         \
2075       __asm__ volatile(                                           \
2076          "mr 11,%1\n\t"                                           \
2077          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2078          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2079          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2080          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2081          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2082          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2083          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2084          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2085          "mr 11,%1\n\t"                                           \
2086          "mr %0,3\n\t"                                            \
2087          "ld 2,-16(11)" /* restore tocptr */                      \
2088          : /*out*/   "=r" (_res)                                  \
2089          : /*in*/    "r" (&_argvec[2])                            \
2090          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2091       );                                                          \
2092       lval = (__typeof__(lval)) _res;                             \
2093    } while (0)
2094
2095 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2096    do {                                                           \
2097       volatile OrigFn        _orig = (orig);                      \
2098       volatile unsigned long _argvec[3+5];                        \
2099       volatile unsigned long _res;                                \
2100       /* _argvec[0] holds current r2 across the call */           \
2101       _argvec[1]   = (unsigned long)_orig.r2;                     \
2102       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2103       _argvec[2+1] = (unsigned long)arg1;                         \
2104       _argvec[2+2] = (unsigned long)arg2;                         \
2105       _argvec[2+3] = (unsigned long)arg3;                         \
2106       _argvec[2+4] = (unsigned long)arg4;                         \
2107       _argvec[2+5] = (unsigned long)arg5;                         \
2108       __asm__ volatile(                                           \
2109          "mr 11,%1\n\t"                                           \
2110          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2111          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2112          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2113          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2114          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2115          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2116          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2117          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2118          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2119          "mr 11,%1\n\t"                                           \
2120          "mr %0,3\n\t"                                            \
2121          "ld 2,-16(11)" /* restore tocptr */                      \
2122          : /*out*/   "=r" (_res)                                  \
2123          : /*in*/    "r" (&_argvec[2])                            \
2124          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2125       );                                                          \
2126       lval = (__typeof__(lval)) _res;                             \
2127    } while (0)
2128
2129 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2130    do {                                                           \
2131       volatile OrigFn        _orig = (orig);                      \
2132       volatile unsigned long _argvec[3+6];                        \
2133       volatile unsigned long _res;                                \
2134       /* _argvec[0] holds current r2 across the call */           \
2135       _argvec[1]   = (unsigned long)_orig.r2;                     \
2136       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2137       _argvec[2+1] = (unsigned long)arg1;                         \
2138       _argvec[2+2] = (unsigned long)arg2;                         \
2139       _argvec[2+3] = (unsigned long)arg3;                         \
2140       _argvec[2+4] = (unsigned long)arg4;                         \
2141       _argvec[2+5] = (unsigned long)arg5;                         \
2142       _argvec[2+6] = (unsigned long)arg6;                         \
2143       __asm__ volatile(                                           \
2144          "mr 11,%1\n\t"                                           \
2145          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2146          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2147          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2148          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2149          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2150          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2151          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2152          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2153          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2154          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2155          "mr 11,%1\n\t"                                           \
2156          "mr %0,3\n\t"                                            \
2157          "ld 2,-16(11)" /* restore tocptr */                      \
2158          : /*out*/   "=r" (_res)                                  \
2159          : /*in*/    "r" (&_argvec[2])                            \
2160          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2161       );                                                          \
2162       lval = (__typeof__(lval)) _res;                             \
2163    } while (0)
2164
2165 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2166                                  arg7)                            \
2167    do {                                                           \
2168       volatile OrigFn        _orig = (orig);                      \
2169       volatile unsigned long _argvec[3+7];                        \
2170       volatile unsigned long _res;                                \
2171       /* _argvec[0] holds current r2 across the call */           \
2172       _argvec[1]   = (unsigned long)_orig.r2;                     \
2173       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2174       _argvec[2+1] = (unsigned long)arg1;                         \
2175       _argvec[2+2] = (unsigned long)arg2;                         \
2176       _argvec[2+3] = (unsigned long)arg3;                         \
2177       _argvec[2+4] = (unsigned long)arg4;                         \
2178       _argvec[2+5] = (unsigned long)arg5;                         \
2179       _argvec[2+6] = (unsigned long)arg6;                         \
2180       _argvec[2+7] = (unsigned long)arg7;                         \
2181       __asm__ volatile(                                           \
2182          "mr 11,%1\n\t"                                           \
2183          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2184          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2185          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2186          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2187          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2188          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2189          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2190          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2191          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2192          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2193          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2194          "mr 11,%1\n\t"                                           \
2195          "mr %0,3\n\t"                                            \
2196          "ld 2,-16(11)" /* restore tocptr */                      \
2197          : /*out*/   "=r" (_res)                                  \
2198          : /*in*/    "r" (&_argvec[2])                            \
2199          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2200       );                                                          \
2201       lval = (__typeof__(lval)) _res;                             \
2202    } while (0)
2203
2204 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2205                                  arg7,arg8)                       \
2206    do {                                                           \
2207       volatile OrigFn        _orig = (orig);                      \
2208       volatile unsigned long _argvec[3+8];                        \
2209       volatile unsigned long _res;                                \
2210       /* _argvec[0] holds current r2 across the call */           \
2211       _argvec[1]   = (unsigned long)_orig.r2;                     \
2212       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2213       _argvec[2+1] = (unsigned long)arg1;                         \
2214       _argvec[2+2] = (unsigned long)arg2;                         \
2215       _argvec[2+3] = (unsigned long)arg3;                         \
2216       _argvec[2+4] = (unsigned long)arg4;                         \
2217       _argvec[2+5] = (unsigned long)arg5;                         \
2218       _argvec[2+6] = (unsigned long)arg6;                         \
2219       _argvec[2+7] = (unsigned long)arg7;                         \
2220       _argvec[2+8] = (unsigned long)arg8;                         \
2221       __asm__ volatile(                                           \
2222          "mr 11,%1\n\t"                                           \
2223          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2224          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2225          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2226          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2227          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2228          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2229          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2230          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2231          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2232          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2233          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2234          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2235          "mr 11,%1\n\t"                                           \
2236          "mr %0,3\n\t"                                            \
2237          "ld 2,-16(11)" /* restore tocptr */                      \
2238          : /*out*/   "=r" (_res)                                  \
2239          : /*in*/    "r" (&_argvec[2])                            \
2240          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2241       );                                                          \
2242       lval = (__typeof__(lval)) _res;                             \
2243    } while (0)
2244
2245 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2246                                  arg7,arg8,arg9)                  \
2247    do {                                                           \
2248       volatile OrigFn        _orig = (orig);                      \
2249       volatile unsigned long _argvec[3+9];                        \
2250       volatile unsigned long _res;                                \
2251       /* _argvec[0] holds current r2 across the call */           \
2252       _argvec[1]   = (unsigned long)_orig.r2;                     \
2253       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2254       _argvec[2+1] = (unsigned long)arg1;                         \
2255       _argvec[2+2] = (unsigned long)arg2;                         \
2256       _argvec[2+3] = (unsigned long)arg3;                         \
2257       _argvec[2+4] = (unsigned long)arg4;                         \
2258       _argvec[2+5] = (unsigned long)arg5;                         \
2259       _argvec[2+6] = (unsigned long)arg6;                         \
2260       _argvec[2+7] = (unsigned long)arg7;                         \
2261       _argvec[2+8] = (unsigned long)arg8;                         \
2262       _argvec[2+9] = (unsigned long)arg9;                         \
2263       __asm__ volatile(                                           \
2264          "mr 11,%1\n\t"                                           \
2265          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2266          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2267          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2268          /* arg9 */                                               \
2269          "ld  3,72(11)\n\t"                                       \
2270          "std 3,112(1)\n\t"                                       \
2271          /* args1-8 */                                            \
2272          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2273          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2274          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2275          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2276          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2277          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2278          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2279          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2280          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2281          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2282          "mr 11,%1\n\t"                                           \
2283          "mr %0,3\n\t"                                            \
2284          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2285          "addi 1,1,128"     /* restore frame */                   \
2286          : /*out*/   "=r" (_res)                                  \
2287          : /*in*/    "r" (&_argvec[2])                            \
2288          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2289       );                                                          \
2290       lval = (__typeof__(lval)) _res;                             \
2291    } while (0)
2292
2293 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2294                                   arg7,arg8,arg9,arg10)           \
2295    do {                                                           \
2296       volatile OrigFn        _orig = (orig);                      \
2297       volatile unsigned long _argvec[3+10];                       \
2298       volatile unsigned long _res;                                \
2299       /* _argvec[0] holds current r2 across the call */           \
2300       _argvec[1]   = (unsigned long)_orig.r2;                     \
2301       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2302       _argvec[2+1] = (unsigned long)arg1;                         \
2303       _argvec[2+2] = (unsigned long)arg2;                         \
2304       _argvec[2+3] = (unsigned long)arg3;                         \
2305       _argvec[2+4] = (unsigned long)arg4;                         \
2306       _argvec[2+5] = (unsigned long)arg5;                         \
2307       _argvec[2+6] = (unsigned long)arg6;                         \
2308       _argvec[2+7] = (unsigned long)arg7;                         \
2309       _argvec[2+8] = (unsigned long)arg8;                         \
2310       _argvec[2+9] = (unsigned long)arg9;                         \
2311       _argvec[2+10] = (unsigned long)arg10;                       \
2312       __asm__ volatile(                                           \
2313          "mr 11,%1\n\t"                                           \
2314          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2315          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2316          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2317          /* arg10 */                                              \
2318          "ld  3,80(11)\n\t"                                       \
2319          "std 3,120(1)\n\t"                                       \
2320          /* arg9 */                                               \
2321          "ld  3,72(11)\n\t"                                       \
2322          "std 3,112(1)\n\t"                                       \
2323          /* args1-8 */                                            \
2324          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2325          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2326          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2327          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2328          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2329          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2330          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2331          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2332          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2333          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2334          "mr 11,%1\n\t"                                           \
2335          "mr %0,3\n\t"                                            \
2336          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2337          "addi 1,1,128"     /* restore frame */                   \
2338          : /*out*/   "=r" (_res)                                  \
2339          : /*in*/    "r" (&_argvec[2])                            \
2340          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2341       );                                                          \
2342       lval = (__typeof__(lval)) _res;                             \
2343    } while (0)
2344
2345 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2346                                   arg7,arg8,arg9,arg10,arg11)     \
2347    do {                                                           \
2348       volatile OrigFn        _orig = (orig);                      \
2349       volatile unsigned long _argvec[3+11];                       \
2350       volatile unsigned long _res;                                \
2351       /* _argvec[0] holds current r2 across the call */           \
2352       _argvec[1]   = (unsigned long)_orig.r2;                     \
2353       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2354       _argvec[2+1] = (unsigned long)arg1;                         \
2355       _argvec[2+2] = (unsigned long)arg2;                         \
2356       _argvec[2+3] = (unsigned long)arg3;                         \
2357       _argvec[2+4] = (unsigned long)arg4;                         \
2358       _argvec[2+5] = (unsigned long)arg5;                         \
2359       _argvec[2+6] = (unsigned long)arg6;                         \
2360       _argvec[2+7] = (unsigned long)arg7;                         \
2361       _argvec[2+8] = (unsigned long)arg8;                         \
2362       _argvec[2+9] = (unsigned long)arg9;                         \
2363       _argvec[2+10] = (unsigned long)arg10;                       \
2364       _argvec[2+11] = (unsigned long)arg11;                       \
2365       __asm__ volatile(                                           \
2366          "mr 11,%1\n\t"                                           \
2367          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2368          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2369          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2370          /* arg11 */                                              \
2371          "ld  3,88(11)\n\t"                                       \
2372          "std 3,128(1)\n\t"                                       \
2373          /* arg10 */                                              \
2374          "ld  3,80(11)\n\t"                                       \
2375          "std 3,120(1)\n\t"                                       \
2376          /* arg9 */                                               \
2377          "ld  3,72(11)\n\t"                                       \
2378          "std 3,112(1)\n\t"                                       \
2379          /* args1-8 */                                            \
2380          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2381          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2382          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2383          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2384          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2385          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2386          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2387          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2388          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2389          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2390          "mr 11,%1\n\t"                                           \
2391          "mr %0,3\n\t"                                            \
2392          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2393          "addi 1,1,144"     /* restore frame */                   \
2394          : /*out*/   "=r" (_res)                                  \
2395          : /*in*/    "r" (&_argvec[2])                            \
2396          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2397       );                                                          \
2398       lval = (__typeof__(lval)) _res;                             \
2399    } while (0)
2400
2401 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2402                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2403    do {                                                           \
2404       volatile OrigFn        _orig = (orig);                      \
2405       volatile unsigned long _argvec[3+12];                       \
2406       volatile unsigned long _res;                                \
2407       /* _argvec[0] holds current r2 across the call */           \
2408       _argvec[1]   = (unsigned long)_orig.r2;                     \
2409       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2410       _argvec[2+1] = (unsigned long)arg1;                         \
2411       _argvec[2+2] = (unsigned long)arg2;                         \
2412       _argvec[2+3] = (unsigned long)arg3;                         \
2413       _argvec[2+4] = (unsigned long)arg4;                         \
2414       _argvec[2+5] = (unsigned long)arg5;                         \
2415       _argvec[2+6] = (unsigned long)arg6;                         \
2416       _argvec[2+7] = (unsigned long)arg7;                         \
2417       _argvec[2+8] = (unsigned long)arg8;                         \
2418       _argvec[2+9] = (unsigned long)arg9;                         \
2419       _argvec[2+10] = (unsigned long)arg10;                       \
2420       _argvec[2+11] = (unsigned long)arg11;                       \
2421       _argvec[2+12] = (unsigned long)arg12;                       \
2422       __asm__ volatile(                                           \
2423          "mr 11,%1\n\t"                                           \
2424          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2425          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2426          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2427          /* arg12 */                                              \
2428          "ld  3,96(11)\n\t"                                       \
2429          "std 3,136(1)\n\t"                                       \
2430          /* arg11 */                                              \
2431          "ld  3,88(11)\n\t"                                       \
2432          "std 3,128(1)\n\t"                                       \
2433          /* arg10 */                                              \
2434          "ld  3,80(11)\n\t"                                       \
2435          "std 3,120(1)\n\t"                                       \
2436          /* arg9 */                                               \
2437          "ld  3,72(11)\n\t"                                       \
2438          "std 3,112(1)\n\t"                                       \
2439          /* args1-8 */                                            \
2440          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2441          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2442          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2443          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2444          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2445          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2446          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2447          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2448          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2449          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2450          "mr 11,%1\n\t"                                           \
2451          "mr %0,3\n\t"                                            \
2452          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2453          "addi 1,1,144"     /* restore frame */                   \
2454          : /*out*/   "=r" (_res)                                  \
2455          : /*in*/    "r" (&_argvec[2])                            \
2456          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2457       );                                                          \
2458       lval = (__typeof__(lval)) _res;                             \
2459    } while (0)
2460
2461 #endif /* PLAT_ppc64_linux */
2462
2463 /* ------------------------ ppc32-aix5 ------------------------- */
2464
2465 #if defined(PLAT_ppc32_aix5)
2466
2467 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2468
2469 /* These regs are trashed by the hidden call. */
2470 #define __CALLER_SAVED_REGS                                       \
2471    "lr", "ctr", "xer",                                            \
2472    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2473    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2474    "r11", "r12", "r13"
2475
2476 /* Expand the stack frame, copying enough info that unwinding
2477    still works.  Trashes r3. */
2478
2479 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
2480          "addi 1,1,-" #_n_fr "\n\t"                               \
2481          "lwz  3," #_n_fr "(1)\n\t"                               \
2482          "stw  3,0(1)\n\t"
2483
2484 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
2485          "addi 1,1," #_n_fr "\n\t"
2486
2487 /* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned
2488    long) == 4. */
2489
2490 #define CALL_FN_W_v(lval, orig)                                   \
2491    do {                                                           \
2492       volatile OrigFn        _orig = (orig);                      \
2493       volatile unsigned long _argvec[3+0];                        \
2494       volatile unsigned long _res;                                \
2495       /* _argvec[0] holds current r2 across the call */           \
2496       _argvec[1] = (unsigned long)_orig.r2;                       \
2497       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2498       __asm__ volatile(                                           \
2499          "mr 11,%1\n\t"                                           \
2500          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2501          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2502          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2503          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2504          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2505          "mr 11,%1\n\t"                                           \
2506          "mr %0,3\n\t"                                            \
2507          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2508          VG_CONTRACT_FRAME_BY(512)                                \
2509          : /*out*/   "=r" (_res)                                  \
2510          : /*in*/    "r" (&_argvec[2])                            \
2511          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2512       );                                                          \
2513       lval = (__typeof__(lval)) _res;                             \
2514    } while (0)
2515
2516 #define CALL_FN_W_W(lval, orig, arg1)                             \
2517    do {                                                           \
2518       volatile OrigFn        _orig = (orig);                      \
2519       volatile unsigned long _argvec[3+1];                        \
2520       volatile unsigned long _res;                                \
2521       /* _argvec[0] holds current r2 across the call */           \
2522       _argvec[1]   = (unsigned long)_orig.r2;                     \
2523       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2524       _argvec[2+1] = (unsigned long)arg1;                         \
2525       __asm__ volatile(                                           \
2526          "mr 11,%1\n\t"                                           \
2527          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2528          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2529          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2530          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2531          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2532          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2533          "mr 11,%1\n\t"                                           \
2534          "mr %0,3\n\t"                                            \
2535          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2536          VG_CONTRACT_FRAME_BY(512)                                \
2537          : /*out*/   "=r" (_res)                                  \
2538          : /*in*/    "r" (&_argvec[2])                            \
2539          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2540       );                                                          \
2541       lval = (__typeof__(lval)) _res;                             \
2542    } while (0)
2543
2544 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2545    do {                                                           \
2546       volatile OrigFn        _orig = (orig);                      \
2547       volatile unsigned long _argvec[3+2];                        \
2548       volatile unsigned long _res;                                \
2549       /* _argvec[0] holds current r2 across the call */           \
2550       _argvec[1]   = (unsigned long)_orig.r2;                     \
2551       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2552       _argvec[2+1] = (unsigned long)arg1;                         \
2553       _argvec[2+2] = (unsigned long)arg2;                         \
2554       __asm__ volatile(                                           \
2555          "mr 11,%1\n\t"                                           \
2556          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2557          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2558          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2559          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2560          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2561          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2562          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2563          "mr 11,%1\n\t"                                           \
2564          "mr %0,3\n\t"                                            \
2565          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2566          VG_CONTRACT_FRAME_BY(512)                                \
2567          : /*out*/   "=r" (_res)                                  \
2568          : /*in*/    "r" (&_argvec[2])                            \
2569          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2570       );                                                          \
2571       lval = (__typeof__(lval)) _res;                             \
2572    } while (0)
2573
2574 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2575    do {                                                           \
2576       volatile OrigFn        _orig = (orig);                      \
2577       volatile unsigned long _argvec[3+3];                        \
2578       volatile unsigned long _res;                                \
2579       /* _argvec[0] holds current r2 across the call */           \
2580       _argvec[1]   = (unsigned long)_orig.r2;                     \
2581       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2582       _argvec[2+1] = (unsigned long)arg1;                         \
2583       _argvec[2+2] = (unsigned long)arg2;                         \
2584       _argvec[2+3] = (unsigned long)arg3;                         \
2585       __asm__ volatile(                                           \
2586          "mr 11,%1\n\t"                                           \
2587          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2588          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2589          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2590          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2591          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2592          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2593          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2594          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2595          "mr 11,%1\n\t"                                           \
2596          "mr %0,3\n\t"                                            \
2597          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2598          VG_CONTRACT_FRAME_BY(512)                                \
2599          : /*out*/   "=r" (_res)                                  \
2600          : /*in*/    "r" (&_argvec[2])                            \
2601          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2602       );                                                          \
2603       lval = (__typeof__(lval)) _res;                             \
2604    } while (0)
2605
2606 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2607    do {                                                           \
2608       volatile OrigFn        _orig = (orig);                      \
2609       volatile unsigned long _argvec[3+4];                        \
2610       volatile unsigned long _res;                                \
2611       /* _argvec[0] holds current r2 across the call */           \
2612       _argvec[1]   = (unsigned long)_orig.r2;                     \
2613       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2614       _argvec[2+1] = (unsigned long)arg1;                         \
2615       _argvec[2+2] = (unsigned long)arg2;                         \
2616       _argvec[2+3] = (unsigned long)arg3;                         \
2617       _argvec[2+4] = (unsigned long)arg4;                         \
2618       __asm__ volatile(                                           \
2619          "mr 11,%1\n\t"                                           \
2620          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2621          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2622          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2623          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2624          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2625          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2626          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2627          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2628          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2629          "mr 11,%1\n\t"                                           \
2630          "mr %0,3\n\t"                                            \
2631          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2632          VG_CONTRACT_FRAME_BY(512)                                \
2633          : /*out*/   "=r" (_res)                                  \
2634          : /*in*/    "r" (&_argvec[2])                            \
2635          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2636       );                                                          \
2637       lval = (__typeof__(lval)) _res;                             \
2638    } while (0)
2639
2640 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2641    do {                                                           \
2642       volatile OrigFn        _orig = (orig);                      \
2643       volatile unsigned long _argvec[3+5];                        \
2644       volatile unsigned long _res;                                \
2645       /* _argvec[0] holds current r2 across the call */           \
2646       _argvec[1]   = (unsigned long)_orig.r2;                     \
2647       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2648       _argvec[2+1] = (unsigned long)arg1;                         \
2649       _argvec[2+2] = (unsigned long)arg2;                         \
2650       _argvec[2+3] = (unsigned long)arg3;                         \
2651       _argvec[2+4] = (unsigned long)arg4;                         \
2652       _argvec[2+5] = (unsigned long)arg5;                         \
2653       __asm__ volatile(                                           \
2654          "mr 11,%1\n\t"                                           \
2655          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2656          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2657          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2658          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2659          "lwz  4, 8(11)\n\t" /* arg2->r4 */                       \
2660          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2661          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2662          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2663          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2664          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2665          "mr 11,%1\n\t"                                           \
2666          "mr %0,3\n\t"                                            \
2667          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2668          VG_CONTRACT_FRAME_BY(512)                                \
2669          : /*out*/   "=r" (_res)                                  \
2670          : /*in*/    "r" (&_argvec[2])                            \
2671          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2672       );                                                          \
2673       lval = (__typeof__(lval)) _res;                             \
2674    } while (0)
2675
2676 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2677    do {                                                           \
2678       volatile OrigFn        _orig = (orig);                      \
2679       volatile unsigned long _argvec[3+6];                        \
2680       volatile unsigned long _res;                                \
2681       /* _argvec[0] holds current r2 across the call */           \
2682       _argvec[1]   = (unsigned long)_orig.r2;                     \
2683       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2684       _argvec[2+1] = (unsigned long)arg1;                         \
2685       _argvec[2+2] = (unsigned long)arg2;                         \
2686       _argvec[2+3] = (unsigned long)arg3;                         \
2687       _argvec[2+4] = (unsigned long)arg4;                         \
2688       _argvec[2+5] = (unsigned long)arg5;                         \
2689       _argvec[2+6] = (unsigned long)arg6;                         \
2690       __asm__ volatile(                                           \
2691          "mr 11,%1\n\t"                                           \
2692          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2693          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2694          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2695          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2696          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2697          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2698          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2699          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2700          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2701          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2702          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2703          "mr 11,%1\n\t"                                           \
2704          "mr %0,3\n\t"                                            \
2705          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2706          VG_CONTRACT_FRAME_BY(512)                                \
2707          : /*out*/   "=r" (_res)                                  \
2708          : /*in*/    "r" (&_argvec[2])                            \
2709          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2710       );                                                          \
2711       lval = (__typeof__(lval)) _res;                             \
2712    } while (0)
2713
2714 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2715                                  arg7)                            \
2716    do {                                                           \
2717       volatile OrigFn        _orig = (orig);                      \
2718       volatile unsigned long _argvec[3+7];                        \
2719       volatile unsigned long _res;                                \
2720       /* _argvec[0] holds current r2 across the call */           \
2721       _argvec[1]   = (unsigned long)_orig.r2;                     \
2722       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2723       _argvec[2+1] = (unsigned long)arg1;                         \
2724       _argvec[2+2] = (unsigned long)arg2;                         \
2725       _argvec[2+3] = (unsigned long)arg3;                         \
2726       _argvec[2+4] = (unsigned long)arg4;                         \
2727       _argvec[2+5] = (unsigned long)arg5;                         \
2728       _argvec[2+6] = (unsigned long)arg6;                         \
2729       _argvec[2+7] = (unsigned long)arg7;                         \
2730       __asm__ volatile(                                           \
2731          "mr 11,%1\n\t"                                           \
2732          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2733          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2734          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2735          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2736          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2737          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2738          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2739          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2740          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2741          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2742          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2743          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2744          "mr 11,%1\n\t"                                           \
2745          "mr %0,3\n\t"                                            \
2746          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2747          VG_CONTRACT_FRAME_BY(512)                                \
2748          : /*out*/   "=r" (_res)                                  \
2749          : /*in*/    "r" (&_argvec[2])                            \
2750          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2751       );                                                          \
2752       lval = (__typeof__(lval)) _res;                             \
2753    } while (0)
2754
2755 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2756                                  arg7,arg8)                       \
2757    do {                                                           \
2758       volatile OrigFn        _orig = (orig);                      \
2759       volatile unsigned long _argvec[3+8];                        \
2760       volatile unsigned long _res;                                \
2761       /* _argvec[0] holds current r2 across the call */           \
2762       _argvec[1]   = (unsigned long)_orig.r2;                     \
2763       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2764       _argvec[2+1] = (unsigned long)arg1;                         \
2765       _argvec[2+2] = (unsigned long)arg2;                         \
2766       _argvec[2+3] = (unsigned long)arg3;                         \
2767       _argvec[2+4] = (unsigned long)arg4;                         \
2768       _argvec[2+5] = (unsigned long)arg5;                         \
2769       _argvec[2+6] = (unsigned long)arg6;                         \
2770       _argvec[2+7] = (unsigned long)arg7;                         \
2771       _argvec[2+8] = (unsigned long)arg8;                         \
2772       __asm__ volatile(                                           \
2773          "mr 11,%1\n\t"                                           \
2774          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2775          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2776          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2777          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2778          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2779          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2780          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2781          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2782          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2783          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2784          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
2785          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2786          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2787          "mr 11,%1\n\t"                                           \
2788          "mr %0,3\n\t"                                            \
2789          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2790          VG_CONTRACT_FRAME_BY(512)                                \
2791          : /*out*/   "=r" (_res)                                  \
2792          : /*in*/    "r" (&_argvec[2])                            \
2793          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2794       );                                                          \
2795       lval = (__typeof__(lval)) _res;                             \
2796    } while (0)
2797
2798 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2799                                  arg7,arg8,arg9)                  \
2800    do {                                                           \
2801       volatile OrigFn        _orig = (orig);                      \
2802       volatile unsigned long _argvec[3+9];                        \
2803       volatile unsigned long _res;                                \
2804       /* _argvec[0] holds current r2 across the call */           \
2805       _argvec[1]   = (unsigned long)_orig.r2;                     \
2806       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2807       _argvec[2+1] = (unsigned long)arg1;                         \
2808       _argvec[2+2] = (unsigned long)arg2;                         \
2809       _argvec[2+3] = (unsigned long)arg3;                         \
2810       _argvec[2+4] = (unsigned long)arg4;                         \
2811       _argvec[2+5] = (unsigned long)arg5;                         \
2812       _argvec[2+6] = (unsigned long)arg6;                         \
2813       _argvec[2+7] = (unsigned long)arg7;                         \
2814       _argvec[2+8] = (unsigned long)arg8;                         \
2815       _argvec[2+9] = (unsigned long)arg9;                         \
2816       __asm__ volatile(                                           \
2817          "mr 11,%1\n\t"                                           \
2818          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2819          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2820          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2821          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
2822          /* arg9 */                                               \
2823          "lwz 3,36(11)\n\t"                                       \
2824          "stw 3,56(1)\n\t"                                        \
2825          /* args1-8 */                                            \
2826          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2827          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2828          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2829          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2830          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2831          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2832          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2833          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
2834          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2835          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2836          "mr 11,%1\n\t"                                           \
2837          "mr %0,3\n\t"                                            \
2838          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2839          VG_CONTRACT_FRAME_BY(64)                                 \
2840          VG_CONTRACT_FRAME_BY(512)                                \
2841          : /*out*/   "=r" (_res)                                  \
2842          : /*in*/    "r" (&_argvec[2])                            \
2843          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2844       );                                                          \
2845       lval = (__typeof__(lval)) _res;                             \
2846    } while (0)
2847
2848 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2849                                   arg7,arg8,arg9,arg10)           \
2850    do {                                                           \
2851       volatile OrigFn        _orig = (orig);                      \
2852       volatile unsigned long _argvec[3+10];                       \
2853       volatile unsigned long _res;                                \
2854       /* _argvec[0] holds current r2 across the call */           \
2855       _argvec[1]   = (unsigned long)_orig.r2;                     \
2856       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2857       _argvec[2+1] = (unsigned long)arg1;                         \
2858       _argvec[2+2] = (unsigned long)arg2;                         \
2859       _argvec[2+3] = (unsigned long)arg3;                         \
2860       _argvec[2+4] = (unsigned long)arg4;                         \
2861       _argvec[2+5] = (unsigned long)arg5;                         \
2862       _argvec[2+6] = (unsigned long)arg6;                         \
2863       _argvec[2+7] = (unsigned long)arg7;                         \
2864       _argvec[2+8] = (unsigned long)arg8;                         \
2865       _argvec[2+9] = (unsigned long)arg9;                         \
2866       _argvec[2+10] = (unsigned long)arg10;                       \
2867       __asm__ volatile(                                           \
2868          "mr 11,%1\n\t"                                           \
2869          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2870          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2871          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2872          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
2873          /* arg10 */                                              \
2874          "lwz 3,40(11)\n\t"                                       \
2875          "stw 3,60(1)\n\t"                                        \
2876          /* arg9 */                                               \
2877          "lwz 3,36(11)\n\t"                                       \
2878          "stw 3,56(1)\n\t"                                        \
2879          /* args1-8 */                                            \
2880          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2881          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2882          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2883          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2884          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2885          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2886          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2887          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
2888          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2889          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2890          "mr 11,%1\n\t"                                           \
2891          "mr %0,3\n\t"                                            \
2892          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2893          VG_CONTRACT_FRAME_BY(64)                                 \
2894          VG_CONTRACT_FRAME_BY(512)                                \
2895          : /*out*/   "=r" (_res)                                  \
2896          : /*in*/    "r" (&_argvec[2])                            \
2897          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2898       );                                                          \
2899       lval = (__typeof__(lval)) _res;                             \
2900    } while (0)
2901
2902 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2903                                   arg7,arg8,arg9,arg10,arg11)     \
2904    do {                                                           \
2905       volatile OrigFn        _orig = (orig);                      \
2906       volatile unsigned long _argvec[3+11];                       \
2907       volatile unsigned long _res;                                \
2908       /* _argvec[0] holds current r2 across the call */           \
2909       _argvec[1]   = (unsigned long)_orig.r2;                     \
2910       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2911       _argvec[2+1] = (unsigned long)arg1;                         \
2912       _argvec[2+2] = (unsigned long)arg2;                         \
2913       _argvec[2+3] = (unsigned long)arg3;                         \
2914       _argvec[2+4] = (unsigned long)arg4;                         \
2915       _argvec[2+5] = (unsigned long)arg5;                         \
2916       _argvec[2+6] = (unsigned long)arg6;                         \
2917       _argvec[2+7] = (unsigned long)arg7;                         \
2918       _argvec[2+8] = (unsigned long)arg8;                         \
2919       _argvec[2+9] = (unsigned long)arg9;                         \
2920       _argvec[2+10] = (unsigned long)arg10;                       \
2921       _argvec[2+11] = (unsigned long)arg11;                       \
2922       __asm__ volatile(                                           \
2923          "mr 11,%1\n\t"                                           \
2924          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2925          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2926          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2927          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
2928          /* arg11 */                                              \
2929          "lwz 3,44(11)\n\t"                                       \
2930          "stw 3,64(1)\n\t"                                        \
2931          /* arg10 */                                              \
2932          "lwz 3,40(11)\n\t"                                       \
2933          "stw 3,60(1)\n\t"                                        \
2934          /* arg9 */                                               \
2935          "lwz 3,36(11)\n\t"                                       \
2936          "stw 3,56(1)\n\t"                                        \
2937          /* args1-8 */                                            \
2938          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
2939          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
2940          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
2941          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
2942          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
2943          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
2944          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
2945          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
2946          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
2947          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2948          "mr 11,%1\n\t"                                           \
2949          "mr %0,3\n\t"                                            \
2950          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
2951          VG_CONTRACT_FRAME_BY(72)                                 \
2952          VG_CONTRACT_FRAME_BY(512)                                \
2953          : /*out*/   "=r" (_res)                                  \
2954          : /*in*/    "r" (&_argvec[2])                            \
2955          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2956       );                                                          \
2957       lval = (__typeof__(lval)) _res;                             \
2958    } while (0)
2959
2960 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2961                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2962    do {                                                           \
2963       volatile OrigFn        _orig = (orig);                      \
2964       volatile unsigned long _argvec[3+12];                       \
2965       volatile unsigned long _res;                                \
2966       /* _argvec[0] holds current r2 across the call */           \
2967       _argvec[1]   = (unsigned long)_orig.r2;                     \
2968       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2969       _argvec[2+1] = (unsigned long)arg1;                         \
2970       _argvec[2+2] = (unsigned long)arg2;                         \
2971       _argvec[2+3] = (unsigned long)arg3;                         \
2972       _argvec[2+4] = (unsigned long)arg4;                         \
2973       _argvec[2+5] = (unsigned long)arg5;                         \
2974       _argvec[2+6] = (unsigned long)arg6;                         \
2975       _argvec[2+7] = (unsigned long)arg7;                         \
2976       _argvec[2+8] = (unsigned long)arg8;                         \
2977       _argvec[2+9] = (unsigned long)arg9;                         \
2978       _argvec[2+10] = (unsigned long)arg10;                       \
2979       _argvec[2+11] = (unsigned long)arg11;                       \
2980       _argvec[2+12] = (unsigned long)arg12;                       \
2981       __asm__ volatile(                                           \
2982          "mr 11,%1\n\t"                                           \
2983          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
2984          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
2985          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
2986          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
2987          /* arg12 */                                              \
2988          "lwz 3,48(11)\n\t"                                       \
2989          "stw 3,68(1)\n\t"                                        \
2990          /* arg11 */                                              \
2991          "lwz 3,44(11)\n\t"                                       \
2992          "stw 3,64(1)\n\t"                                        \
2993          /* arg10 */                                              \
2994          "lwz 3,40(11)\n\t"                                       \
2995          "stw 3,60(1)\n\t"                                        \
2996          /* arg9 */                                               \
2997          "lwz 3,36(11)\n\t"                                       \
2998          "stw 3,56(1)\n\t"                                        \
2999          /* args1-8 */                                            \
3000          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3001          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3002          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3003          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3004          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3005          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3006          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3007          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3008          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3009          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3010          "mr 11,%1\n\t"                                           \
3011          "mr %0,3\n\t"                                            \
3012          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3013          VG_CONTRACT_FRAME_BY(72)                                 \
3014          VG_CONTRACT_FRAME_BY(512)                                \
3015          : /*out*/   "=r" (_res)                                  \
3016          : /*in*/    "r" (&_argvec[2])                            \
3017          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3018       );                                                          \
3019       lval = (__typeof__(lval)) _res;                             \
3020    } while (0)
3021
3022 #endif /* PLAT_ppc32_aix5 */
3023
3024 /* ------------------------ ppc64-aix5 ------------------------- */
3025
3026 #if defined(PLAT_ppc64_aix5)
3027
3028 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3029
3030 /* These regs are trashed by the hidden call. */
3031 #define __CALLER_SAVED_REGS                                       \
3032    "lr", "ctr", "xer",                                            \
3033    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3034    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3035    "r11", "r12", "r13"
3036
3037 /* Expand the stack frame, copying enough info that unwinding
3038    still works.  Trashes r3. */
3039
3040 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3041          "addi 1,1,-" #_n_fr "\n\t"                               \
3042          "ld   3," #_n_fr "(1)\n\t"                               \
3043          "std  3,0(1)\n\t"
3044
3045 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3046          "addi 1,1," #_n_fr "\n\t"
3047
3048 /* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned
3049    long) == 8. */
3050
3051 #define CALL_FN_W_v(lval, orig)                                   \
3052    do {                                                           \
3053       volatile OrigFn        _orig = (orig);                      \
3054       volatile unsigned long _argvec[3+0];                        \
3055       volatile unsigned long _res;                                \
3056       /* _argvec[0] holds current r2 across the call */           \
3057       _argvec[1] = (unsigned long)_orig.r2;                       \
3058       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3059       __asm__ volatile(                                           \
3060          "mr 11,%1\n\t"                                           \
3061          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3062          "std  2,-16(11)\n\t" /* save tocptr */                   \
3063          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3064          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3065          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3066          "mr 11,%1\n\t"                                           \
3067          "mr %0,3\n\t"                                            \
3068          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3069          VG_CONTRACT_FRAME_BY(512)                                \
3070          : /*out*/   "=r" (_res)                                  \
3071          : /*in*/    "r" (&_argvec[2])                            \
3072          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3073       );                                                          \
3074       lval = (__typeof__(lval)) _res;                             \
3075    } while (0)
3076
3077 #define CALL_FN_W_W(lval, orig, arg1)                             \
3078    do {                                                           \
3079       volatile OrigFn        _orig = (orig);                      \
3080       volatile unsigned long _argvec[3+1];                        \
3081       volatile unsigned long _res;                                \
3082       /* _argvec[0] holds current r2 across the call */           \
3083       _argvec[1]   = (unsigned long)_orig.r2;                     \
3084       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3085       _argvec[2+1] = (unsigned long)arg1;                         \
3086       __asm__ volatile(                                           \
3087          "mr 11,%1\n\t"                                           \
3088          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3089          "std  2,-16(11)\n\t" /* save tocptr */                   \
3090          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3091          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3092          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3093          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3094          "mr 11,%1\n\t"                                           \
3095          "mr %0,3\n\t"                                            \
3096          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3097          VG_CONTRACT_FRAME_BY(512)                                \
3098          : /*out*/   "=r" (_res)                                  \
3099          : /*in*/    "r" (&_argvec[2])                            \
3100          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3101       );                                                          \
3102       lval = (__typeof__(lval)) _res;                             \
3103    } while (0)
3104
3105 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3106    do {                                                           \
3107       volatile OrigFn        _orig = (orig);                      \
3108       volatile unsigned long _argvec[3+2];                        \
3109       volatile unsigned long _res;                                \
3110       /* _argvec[0] holds current r2 across the call */           \
3111       _argvec[1]   = (unsigned long)_orig.r2;                     \
3112       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3113       _argvec[2+1] = (unsigned long)arg1;                         \
3114       _argvec[2+2] = (unsigned long)arg2;                         \
3115       __asm__ volatile(                                           \
3116          "mr 11,%1\n\t"                                           \
3117          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3118          "std  2,-16(11)\n\t" /* save tocptr */                   \
3119          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3120          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3121          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3122          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3123          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3124          "mr 11,%1\n\t"                                           \
3125          "mr %0,3\n\t"                                            \
3126          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3127          VG_CONTRACT_FRAME_BY(512)                                \
3128          : /*out*/   "=r" (_res)                                  \
3129          : /*in*/    "r" (&_argvec[2])                            \
3130          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3131       );                                                          \
3132       lval = (__typeof__(lval)) _res;                             \
3133    } while (0)
3134
3135 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3136    do {                                                           \
3137       volatile OrigFn        _orig = (orig);                      \
3138       volatile unsigned long _argvec[3+3];                        \
3139       volatile unsigned long _res;                                \
3140       /* _argvec[0] holds current r2 across the call */           \
3141       _argvec[1]   = (unsigned long)_orig.r2;                     \
3142       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3143       _argvec[2+1] = (unsigned long)arg1;                         \
3144       _argvec[2+2] = (unsigned long)arg2;                         \
3145       _argvec[2+3] = (unsigned long)arg3;                         \
3146       __asm__ volatile(                                           \
3147          "mr 11,%1\n\t"                                           \
3148          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3149          "std  2,-16(11)\n\t" /* save tocptr */                   \
3150          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3151          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3152          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3153          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3154          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3155          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3156          "mr 11,%1\n\t"                                           \
3157          "mr %0,3\n\t"                                            \
3158          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3159          VG_CONTRACT_FRAME_BY(512)                                \
3160          : /*out*/   "=r" (_res)                                  \
3161          : /*in*/    "r" (&_argvec[2])                            \
3162          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3163       );                                                          \
3164       lval = (__typeof__(lval)) _res;                             \
3165    } while (0)
3166
3167 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3168    do {                                                           \
3169       volatile OrigFn        _orig = (orig);                      \
3170       volatile unsigned long _argvec[3+4];                        \
3171       volatile unsigned long _res;                                \
3172       /* _argvec[0] holds current r2 across the call */           \
3173       _argvec[1]   = (unsigned long)_orig.r2;                     \
3174       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3175       _argvec[2+1] = (unsigned long)arg1;                         \
3176       _argvec[2+2] = (unsigned long)arg2;                         \
3177       _argvec[2+3] = (unsigned long)arg3;                         \
3178       _argvec[2+4] = (unsigned long)arg4;                         \
3179       __asm__ volatile(                                           \
3180          "mr 11,%1\n\t"                                           \
3181          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3182          "std  2,-16(11)\n\t" /* save tocptr */                   \
3183          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3184          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3185          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3186          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3187          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3188          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3189          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3190          "mr 11,%1\n\t"                                           \
3191          "mr %0,3\n\t"                                            \
3192          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3193          VG_CONTRACT_FRAME_BY(512)                                \
3194          : /*out*/   "=r" (_res)                                  \
3195          : /*in*/    "r" (&_argvec[2])                            \
3196          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3197       );                                                          \
3198       lval = (__typeof__(lval)) _res;                             \
3199    } while (0)
3200
3201 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3202    do {                                                           \
3203       volatile OrigFn        _orig = (orig);                      \
3204       volatile unsigned long _argvec[3+5];                        \
3205       volatile unsigned long _res;                                \
3206       /* _argvec[0] holds current r2 across the call */           \
3207       _argvec[1]   = (unsigned long)_orig.r2;                     \
3208       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3209       _argvec[2+1] = (unsigned long)arg1;                         \
3210       _argvec[2+2] = (unsigned long)arg2;                         \
3211       _argvec[2+3] = (unsigned long)arg3;                         \
3212       _argvec[2+4] = (unsigned long)arg4;                         \
3213       _argvec[2+5] = (unsigned long)arg5;                         \
3214       __asm__ volatile(                                           \
3215          "mr 11,%1\n\t"                                           \
3216          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3217          "std  2,-16(11)\n\t" /* save tocptr */                   \
3218          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3219          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3220          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3221          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3222          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3223          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3224          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3225          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3226          "mr 11,%1\n\t"                                           \
3227          "mr %0,3\n\t"                                            \
3228          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3229          VG_CONTRACT_FRAME_BY(512)                                \
3230          : /*out*/   "=r" (_res)                                  \
3231          : /*in*/    "r" (&_argvec[2])                            \
3232          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3233       );                                                          \
3234       lval = (__typeof__(lval)) _res;                             \
3235    } while (0)
3236
3237 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3238    do {                                                           \
3239       volatile OrigFn        _orig = (orig);                      \
3240       volatile unsigned long _argvec[3+6];                        \
3241       volatile unsigned long _res;                                \
3242       /* _argvec[0] holds current r2 across the call */           \
3243       _argvec[1]   = (unsigned long)_orig.r2;                     \
3244       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3245       _argvec[2+1] = (unsigned long)arg1;                         \
3246       _argvec[2+2] = (unsigned long)arg2;                         \
3247       _argvec[2+3] = (unsigned long)arg3;                         \
3248       _argvec[2+4] = (unsigned long)arg4;                         \
3249       _argvec[2+5] = (unsigned long)arg5;                         \
3250       _argvec[2+6] = (unsigned long)arg6;                         \
3251       __asm__ volatile(                                           \
3252          "mr 11,%1\n\t"                                           \
3253          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3254          "std  2,-16(11)\n\t" /* save tocptr */                   \
3255          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3256          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3257          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3258          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3259          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3260          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3261          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3262          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3263          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3264          "mr 11,%1\n\t"                                           \
3265          "mr %0,3\n\t"                                            \
3266          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3267          VG_CONTRACT_FRAME_BY(512)                                \
3268          : /*out*/   "=r" (_res)                                  \
3269          : /*in*/    "r" (&_argvec[2])                            \
3270          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3271       );                                                          \
3272       lval = (__typeof__(lval)) _res;                             \
3273    } while (0)
3274
3275 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3276                                  arg7)                            \
3277    do {                                                           \
3278       volatile OrigFn        _orig = (orig);                      \
3279       volatile unsigned long _argvec[3+7];                        \
3280       volatile unsigned long _res;                                \
3281       /* _argvec[0] holds current r2 across the call */           \
3282       _argvec[1]   = (unsigned long)_orig.r2;                     \
3283       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3284       _argvec[2+1] = (unsigned long)arg1;                         \
3285       _argvec[2+2] = (unsigned long)arg2;                         \
3286       _argvec[2+3] = (unsigned long)arg3;                         \
3287       _argvec[2+4] = (unsigned long)arg4;                         \
3288       _argvec[2+5] = (unsigned long)arg5;                         \
3289       _argvec[2+6] = (unsigned long)arg6;                         \
3290       _argvec[2+7] = (unsigned long)arg7;                         \
3291       __asm__ volatile(                                           \
3292          "mr 11,%1\n\t"                                           \
3293          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3294          "std  2,-16(11)\n\t" /* save tocptr */                   \
3295          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3296          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3297          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3298          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3299          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3300          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3301          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3302          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3303          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3304          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3305          "mr 11,%1\n\t"                                           \
3306          "mr %0,3\n\t"                                            \
3307          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3308          VG_CONTRACT_FRAME_BY(512)                                \
3309          : /*out*/   "=r" (_res)                                  \
3310          : /*in*/    "r" (&_argvec[2])                            \
3311          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3312       );                                                          \
3313       lval = (__typeof__(lval)) _res;                             \
3314    } while (0)
3315
3316 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3317                                  arg7,arg8)                       \
3318    do {                                                           \
3319       volatile OrigFn        _orig = (orig);                      \
3320       volatile unsigned long _argvec[3+8];                        \
3321       volatile unsigned long _res;                                \
3322       /* _argvec[0] holds current r2 across the call */           \
3323       _argvec[1]   = (unsigned long)_orig.r2;                     \
3324       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3325       _argvec[2+1] = (unsigned long)arg1;                         \
3326       _argvec[2+2] = (unsigned long)arg2;                         \
3327       _argvec[2+3] = (unsigned long)arg3;                         \
3328       _argvec[2+4] = (unsigned long)arg4;                         \
3329       _argvec[2+5] = (unsigned long)arg5;                         \
3330       _argvec[2+6] = (unsigned long)arg6;                         \
3331       _argvec[2+7] = (unsigned long)arg7;                         \
3332       _argvec[2+8] = (unsigned long)arg8;                         \
3333       __asm__ volatile(                                           \
3334          "mr 11,%1\n\t"                                           \
3335          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3336          "std  2,-16(11)\n\t" /* save tocptr */                   \
3337          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3338          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3339          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3340          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3341          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3342          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3343          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3344          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3345          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3346          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3347          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3348          "mr 11,%1\n\t"                                           \
3349          "mr %0,3\n\t"                                            \
3350          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3351          VG_CONTRACT_FRAME_BY(512)                                \
3352          : /*out*/   "=r" (_res)                                  \
3353          : /*in*/    "r" (&_argvec[2])                            \
3354          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3355       );                                                          \
3356       lval = (__typeof__(lval)) _res;                             \
3357    } while (0)
3358
3359 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3360                                  arg7,arg8,arg9)                  \
3361    do {                                                           \
3362       volatile OrigFn        _orig = (orig);                      \
3363       volatile unsigned long _argvec[3+9];                        \
3364       volatile unsigned long _res;                                \
3365       /* _argvec[0] holds current r2 across the call */           \
3366       _argvec[1]   = (unsigned long)_orig.r2;                     \
3367       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3368       _argvec[2+1] = (unsigned long)arg1;                         \
3369       _argvec[2+2] = (unsigned long)arg2;                         \
3370       _argvec[2+3] = (unsigned long)arg3;                         \
3371       _argvec[2+4] = (unsigned long)arg4;                         \
3372       _argvec[2+5] = (unsigned long)arg5;                         \
3373       _argvec[2+6] = (unsigned long)arg6;                         \
3374       _argvec[2+7] = (unsigned long)arg7;                         \
3375       _argvec[2+8] = (unsigned long)arg8;                         \
3376       _argvec[2+9] = (unsigned long)arg9;                         \
3377       __asm__ volatile(                                           \
3378          "mr 11,%1\n\t"                                           \
3379          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3380          "std  2,-16(11)\n\t" /* save tocptr */                   \
3381          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3382          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
3383          /* arg9 */                                               \
3384          "ld  3,72(11)\n\t"                                       \
3385          "std 3,112(1)\n\t"                                       \
3386          /* args1-8 */                                            \
3387          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3388          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3389          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3390          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3391          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3392          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3393          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3394          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3395          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3396          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3397          "mr 11,%1\n\t"                                           \
3398          "mr %0,3\n\t"                                            \
3399          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3400          VG_CONTRACT_FRAME_BY(128)                                \
3401          VG_CONTRACT_FRAME_BY(512)                                \
3402          : /*out*/   "=r" (_res)                                  \
3403          : /*in*/    "r" (&_argvec[2])                            \
3404          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3405       );                                                          \
3406       lval = (__typeof__(lval)) _res;                             \
3407    } while (0)
3408
3409 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3410                                   arg7,arg8,arg9,arg10)           \
3411    do {                                                           \
3412       volatile OrigFn        _orig = (orig);                      \
3413       volatile unsigned long _argvec[3+10];                       \
3414       volatile unsigned long _res;                                \
3415       /* _argvec[0] holds current r2 across the call */           \
3416       _argvec[1]   = (unsigned long)_orig.r2;                     \
3417       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3418       _argvec[2+1] = (unsigned long)arg1;                         \
3419       _argvec[2+2] = (unsigned long)arg2;                         \
3420       _argvec[2+3] = (unsigned long)arg3;                         \
3421       _argvec[2+4] = (unsigned long)arg4;                         \
3422       _argvec[2+5] = (unsigned long)arg5;                         \
3423       _argvec[2+6] = (unsigned long)arg6;                         \
3424       _argvec[2+7] = (unsigned long)arg7;                         \
3425       _argvec[2+8] = (unsigned long)arg8;                         \
3426       _argvec[2+9] = (unsigned long)arg9;                         \
3427       _argvec[2+10] = (unsigned long)arg10;                       \
3428       __asm__ volatile(                                           \
3429          "mr 11,%1\n\t"                                           \
3430          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3431          "std  2,-16(11)\n\t" /* save tocptr */                   \
3432          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3433          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
3434          /* arg10 */                                              \
3435          "ld  3,80(11)\n\t"                                       \
3436          "std 3,120(1)\n\t"                                       \
3437          /* arg9 */                                               \
3438          "ld  3,72(11)\n\t"                                       \
3439          "std 3,112(1)\n\t"                                       \
3440          /* args1-8 */                                            \
3441          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3442          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3443          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3444          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3445          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3446          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3447          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3448          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3449          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3450          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3451          "mr 11,%1\n\t"                                           \
3452          "mr %0,3\n\t"                                            \
3453          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3454          VG_CONTRACT_FRAME_BY(128)                                \
3455          VG_CONTRACT_FRAME_BY(512)                                \
3456          : /*out*/   "=r" (_res)                                  \
3457          : /*in*/    "r" (&_argvec[2])                            \
3458          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3459       );                                                          \
3460       lval = (__typeof__(lval)) _res;                             \
3461    } while (0)
3462
3463 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3464                                   arg7,arg8,arg9,arg10,arg11)     \
3465    do {                                                           \
3466       volatile OrigFn        _orig = (orig);                      \
3467       volatile unsigned long _argvec[3+11];                       \
3468       volatile unsigned long _res;                                \
3469       /* _argvec[0] holds current r2 across the call */           \
3470       _argvec[1]   = (unsigned long)_orig.r2;                     \
3471       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3472       _argvec[2+1] = (unsigned long)arg1;                         \
3473       _argvec[2+2] = (unsigned long)arg2;                         \
3474       _argvec[2+3] = (unsigned long)arg3;                         \
3475       _argvec[2+4] = (unsigned long)arg4;                         \
3476       _argvec[2+5] = (unsigned long)arg5;                         \
3477       _argvec[2+6] = (unsigned long)arg6;                         \
3478       _argvec[2+7] = (unsigned long)arg7;                         \
3479       _argvec[2+8] = (unsigned long)arg8;                         \
3480       _argvec[2+9] = (unsigned long)arg9;                         \
3481       _argvec[2+10] = (unsigned long)arg10;                       \
3482       _argvec[2+11] = (unsigned long)arg11;                       \
3483       __asm__ volatile(                                           \
3484          "mr 11,%1\n\t"                                           \
3485          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3486          "std  2,-16(11)\n\t" /* save tocptr */                   \
3487          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3488          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
3489          /* arg11 */                                              \
3490          "ld  3,88(11)\n\t"                                       \
3491          "std 3,128(1)\n\t"                                       \
3492          /* arg10 */                                              \
3493          "ld  3,80(11)\n\t"                                       \
3494          "std 3,120(1)\n\t"                                       \
3495          /* arg9 */                                               \
3496          "ld  3,72(11)\n\t"                                       \
3497          "std 3,112(1)\n\t"                                       \
3498          /* args1-8 */                                            \
3499          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3500          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3501          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3502          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3503          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3504          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3505          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3506          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3507          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3508          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3509          "mr 11,%1\n\t"                                           \
3510          "mr %0,3\n\t"                                            \
3511          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3512          VG_CONTRACT_FRAME_BY(144)                                \
3513          VG_CONTRACT_FRAME_BY(512)                                \
3514          : /*out*/   "=r" (_res)                                  \
3515          : /*in*/    "r" (&_argvec[2])                            \
3516          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3517       );                                                          \
3518       lval = (__typeof__(lval)) _res;                             \
3519    } while (0)
3520
3521 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3522                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3523    do {                                                           \
3524       volatile OrigFn        _orig = (orig);                      \
3525       volatile unsigned long _argvec[3+12];                       \
3526       volatile unsigned long _res;                                \
3527       /* _argvec[0] holds current r2 across the call */           \
3528       _argvec[1]   = (unsigned long)_orig.r2;                     \
3529       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3530       _argvec[2+1] = (unsigned long)arg1;                         \
3531       _argvec[2+2] = (unsigned long)arg2;                         \
3532       _argvec[2+3] = (unsigned long)arg3;                         \
3533       _argvec[2+4] = (unsigned long)arg4;                         \
3534       _argvec[2+5] = (unsigned long)arg5;                         \
3535       _argvec[2+6] = (unsigned long)arg6;                         \
3536       _argvec[2+7] = (unsigned long)arg7;                         \
3537       _argvec[2+8] = (unsigned long)arg8;                         \
3538       _argvec[2+9] = (unsigned long)arg9;                         \
3539       _argvec[2+10] = (unsigned long)arg10;                       \
3540       _argvec[2+11] = (unsigned long)arg11;                       \
3541       _argvec[2+12] = (unsigned long)arg12;                       \
3542       __asm__ volatile(                                           \
3543          "mr 11,%1\n\t"                                           \
3544          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3545          "std  2,-16(11)\n\t" /* save tocptr */                   \
3546          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3547          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
3548          /* arg12 */                                              \
3549          "ld  3,96(11)\n\t"                                       \
3550          "std 3,136(1)\n\t"                                       \
3551          /* arg11 */                                              \
3552          "ld  3,88(11)\n\t"                                       \
3553          "std 3,128(1)\n\t"                                       \
3554          /* arg10 */                                              \
3555          "ld  3,80(11)\n\t"                                       \
3556          "std 3,120(1)\n\t"                                       \
3557          /* arg9 */                                               \
3558          "ld  3,72(11)\n\t"                                       \
3559          "std 3,112(1)\n\t"                                       \
3560          /* args1-8 */                                            \
3561          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3562          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3563          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3564          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3565          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3566          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3567          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3568          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3569          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3570          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3571          "mr 11,%1\n\t"                                           \
3572          "mr %0,3\n\t"                                            \
3573          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3574          VG_CONTRACT_FRAME_BY(144)                                \
3575          VG_CONTRACT_FRAME_BY(512)                                \
3576          : /*out*/   "=r" (_res)                                  \
3577          : /*in*/    "r" (&_argvec[2])                            \
3578          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3579       );                                                          \
3580       lval = (__typeof__(lval)) _res;                             \
3581    } while (0)
3582
3583 #endif /* PLAT_ppc64_aix5 */
3584
3585
3586 /* ------------------------------------------------------------------ */
3587 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
3588 /*                                                                    */
3589 /* ------------------------------------------------------------------ */
3590
3591 /* Some request codes.  There are many more of these, but most are not
3592    exposed to end-user view.  These are the public ones, all of the
3593    form 0x1000 + small_number.
3594
3595    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
3596    ones start at 0x2000.
3597 */
3598
3599 /* These macros are used by tools -- they must be public, but don't
3600    embed them into other programs. */
3601 #define VG_USERREQ_TOOL_BASE(a,b) \
3602    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
3603 #define VG_IS_TOOL_USERREQ(a, b, v) \
3604    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
3605
3606 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
3607    This enum comprises an ABI exported by Valgrind to programs
3608    which use client requests.  DO NOT CHANGE THE ORDER OF THESE
3609    ENTRIES, NOR DELETE ANY -- add new ones at the end. */
3610 typedef
3611    enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
3612           VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
3613
3614           /* These allow any function to be called from the simulated
3615              CPU but run on the real CPU.  Nb: the first arg passed to
3616              the function is always the ThreadId of the running
3617              thread!  So CLIENT_CALL0 actually requires a 1 arg
3618              function, etc. */
3619           VG_USERREQ__CLIENT_CALL0 = 0x1101,
3620           VG_USERREQ__CLIENT_CALL1 = 0x1102,
3621           VG_USERREQ__CLIENT_CALL2 = 0x1103,
3622           VG_USERREQ__CLIENT_CALL3 = 0x1104,
3623
3624           /* Can be useful in regression testing suites -- eg. can
3625              send Valgrind's output to /dev/null and still count
3626              errors. */
3627           VG_USERREQ__COUNT_ERRORS = 0x1201,
3628
3629           /* These are useful and can be interpreted by any tool that
3630              tracks malloc() et al, by using vg_replace_malloc.c. */
3631           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
3632           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
3633           /* Memory pool support. */
3634           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
3635           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
3636           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
3637           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
3638           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
3639           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
3640           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
3641           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
3642
3643           /* Allow printfs to valgrind log. */
3644           VG_USERREQ__PRINTF           = 0x1401,
3645           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
3646
3647           /* Stack support. */
3648           VG_USERREQ__STACK_REGISTER   = 0x1501,
3649           VG_USERREQ__STACK_DEREGISTER = 0x1502,
3650           VG_USERREQ__STACK_CHANGE     = 0x1503,
3651
3652           /* Wine support */
3653           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601
3654    } Vg_ClientRequest;
3655
3656 #if !defined(__GNUC__)
3657 #  define __extension__ /* */
3658 #endif
3659
3660 /* Returns the number of Valgrinds this code is running under.  That
3661    is, 0 if running natively, 1 if running under Valgrind, 2 if
3662    running under Valgrind which is running under another Valgrind,
3663    etc. */
3664 #if defined(HOST_WIN32) || defined(TARGET_WIN32)
3665 #define RUNNING_ON_VALGRIND 0
3666 #else
3667 #define RUNNING_ON_VALGRIND  __extension__                        \
3668    ({unsigned int _qzz_res;                                       \
3669     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* if not */,          \
3670                                VG_USERREQ__RUNNING_ON_VALGRIND,   \
3671                                0, 0, 0, 0, 0);                    \
3672     _qzz_res;                                                     \
3673    })
3674 #endif
3675
3676 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
3677    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
3678    since it provides a way to make sure valgrind will retranslate the
3679    invalidated area.  Returns no value. */
3680 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)         \
3681    {unsigned int _qzz_res;                                        \
3682     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3683                                VG_USERREQ__DISCARD_TRANSLATIONS,  \
3684                                _qzz_addr, _qzz_len, 0, 0, 0);     \
3685    }
3686
3687
3688 /* These requests are for getting Valgrind itself to print something.
3689    Possibly with a backtrace.  This is a really ugly hack.  The return value
3690    is the number of characters printed, excluding the "**<pid>** " part at the
3691    start and the backtrace (if present). */
3692
3693 #if defined(NVALGRIND)
3694
3695 #  define VALGRIND_PRINTF(...)
3696 #  define VALGRIND_PRINTF_BACKTRACE(...)
3697
3698 #else /* NVALGRIND */
3699
3700 /* Modern GCC will optimize the static routine out if unused,
3701    and unused attribute will shut down warnings about it.  */
3702 static int VALGRIND_PRINTF(const char *format, ...)
3703    __attribute__((format(__printf__, 1, 2), __unused__));
3704 static int
3705 VALGRIND_PRINTF(const char *format, ...)
3706 {
3707    unsigned long _qzz_res;
3708    va_list vargs;
3709    va_start(vargs, format);
3710    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF,
3711                               (unsigned long)format, (unsigned long)vargs,
3712                               0, 0, 0);
3713    va_end(vargs);
3714    return (int)_qzz_res;
3715 }
3716
3717 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
3718    __attribute__((format(__printf__, 1, 2), __unused__));
3719 static int
3720 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
3721 {
3722    unsigned long _qzz_res;
3723    va_list vargs;
3724    va_start(vargs, format);
3725    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF_BACKTRACE,
3726                               (unsigned long)format, (unsigned long)vargs,
3727                               0, 0, 0);
3728    va_end(vargs);
3729    return (int)_qzz_res;
3730 }
3731
3732 #endif /* NVALGRIND */
3733
3734
3735 /* These requests allow control to move from the simulated CPU to the
3736    real CPU, calling an arbitary function.
3737
3738    Note that the current ThreadId is inserted as the first argument.
3739    So this call:
3740
3741      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
3742
3743    requires f to have this signature:
3744
3745      Word f(Word tid, Word arg1, Word arg2)
3746
3747    where "Word" is a word-sized type.
3748
3749    Note that these client requests are not entirely reliable.  For example,
3750    if you call a function with them that subsequently calls printf(),
3751    there's a high chance Valgrind will crash.  Generally, your prospects of
3752    these working are made higher if the called function does not refer to
3753    any global variables, and does not refer to any libc or other functions
3754    (printf et al).  Any kind of entanglement with libc or dynamic linking is
3755    likely to have a bad outcome, for tricky reasons which we've grappled
3756    with a lot in the past.
3757 */
3758 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
3759    __extension__                                                  \
3760    ({unsigned long _qyy_res;                                      \
3761     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3762                                VG_USERREQ__CLIENT_CALL0,          \
3763                                _qyy_fn,                           \
3764                                0, 0, 0, 0);                       \
3765     _qyy_res;                                                     \
3766    })
3767
3768 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)               \
3769    __extension__                                                  \
3770    ({unsigned long _qyy_res;                                      \
3771     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3772                                VG_USERREQ__CLIENT_CALL1,          \
3773                                _qyy_fn,                           \
3774                                _qyy_arg1, 0, 0, 0);               \
3775     _qyy_res;                                                     \
3776    })
3777
3778 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)    \
3779    __extension__                                                  \
3780    ({unsigned long _qyy_res;                                      \
3781     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3782                                VG_USERREQ__CLIENT_CALL2,          \
3783                                _qyy_fn,                           \
3784                                _qyy_arg1, _qyy_arg2, 0, 0);       \
3785     _qyy_res;                                                     \
3786    })
3787
3788 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
3789    __extension__                                                  \
3790    ({unsigned long _qyy_res;                                      \
3791     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3792                                VG_USERREQ__CLIENT_CALL3,          \
3793                                _qyy_fn,                           \
3794                                _qyy_arg1, _qyy_arg2,              \
3795                                _qyy_arg3, 0);                     \
3796     _qyy_res;                                                     \
3797    })
3798
3799
3800 /* Counts the number of errors that have been recorded by a tool.  Nb:
3801    the tool must record the errors with VG_(maybe_record_error)() or
3802    VG_(unique_error)() for them to be counted. */
3803 #define VALGRIND_COUNT_ERRORS                                     \
3804    __extension__                                                  \
3805    ({unsigned int _qyy_res;                                       \
3806     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
3807                                VG_USERREQ__COUNT_ERRORS,          \
3808                                0, 0, 0, 0, 0);                    \
3809     _qyy_res;                                                     \
3810    })
3811
3812 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
3813    when heap blocks are allocated in order to give accurate results.  This
3814    happens automatically for the standard allocator functions such as
3815    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
3816    delete[], etc.
3817
3818    But if your program uses a custom allocator, this doesn't automatically
3819    happen, and Valgrind will not do as well.  For example, if you allocate
3820    superblocks with mmap() and then allocates chunks of the superblocks, all
3821    Valgrind's observations will be at the mmap() level and it won't know that
3822    the chunks should be considered separate entities.  In Memcheck's case,
3823    that means you probably won't get heap block overrun detection (because
3824    there won't be redzones marked as unaddressable) and you definitely won't
3825    get any leak detection.
3826
3827    The following client requests allow a custom allocator to be annotated so
3828    that it can be handled accurately by Valgrind.
3829
3830    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
3831    by a malloc()-like function.  For Memcheck (an illustrative case), this
3832    does two things:
3833
3834    - It records that the block has been allocated.  This means any addresses
3835      within the block mentioned in error messages will be
3836      identified as belonging to the block.  It also means that if the block
3837      isn't freed it will be detected by the leak checker.
3838
3839    - It marks the block as being addressable and undefined (if 'is_zeroed' is
3840      not set), or addressable and defined (if 'is_zeroed' is set).  This
3841      controls how accesses to the block by the program are handled.
3842
3843    'addr' is the start of the usable block (ie. after any
3844    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
3845    can apply redzones -- these are blocks of padding at the start and end of
3846    each block.  Adding redzones is recommended as it makes it much more likely
3847    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
3848    zeroed (or filled with another predictable value), as is the case for
3849    calloc().
3850
3851    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
3852    heap block -- that will be used by the client program -- is allocated.
3853    It's best to put it at the outermost level of the allocator if possible;
3854    for example, if you have a function my_alloc() which calls
3855    internal_alloc(), and the client request is put inside internal_alloc(),
3856    stack traces relating to the heap block will contain entries for both
3857    my_alloc() and internal_alloc(), which is probably not what you want.
3858
3859    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
3860    custom blocks from within a heap block, B, that has been allocated with
3861    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
3862    -- the custom blocks will take precedence.
3863
3864    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
3865    Memcheck, it does two things:
3866
3867    - It records that the block has been deallocated.  This assumes that the
3868      block was annotated as having been allocated via
3869      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
3870
3871    - It marks the block as being unaddressable.
3872
3873    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
3874    heap block is deallocated.
3875
3876    In many cases, these two client requests will not be enough to get your
3877    allocator working well with Memcheck.  More specifically, if your allocator
3878    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
3879    will be necessary to mark the memory as addressable just before the zeroing
3880    occurs, otherwise you'll get a lot of invalid write errors.  For example,
3881    you'll need to do this if your allocator recycles freed blocks, but it
3882    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
3883    Alternatively, if your allocator reuses freed blocks for allocator-internal
3884    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
3885
3886    Really, what's happening is a blurring of the lines between the client
3887    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
3888    memory should be considered unaddressable to the client program, but the
3889    allocator knows more than the rest of the client program and so may be able
3890    to safely access it.  Extra client requests are necessary for Valgrind to
3891    understand the distinction between the allocator and the rest of the
3892    program.
3893
3894    Note: there is currently no VALGRIND_REALLOCLIKE_BLOCK client request;  it
3895    has to be emulated with MALLOCLIKE/FREELIKE and memory copying.
3896
3897    Ignored if addr == 0.
3898 */
3899 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)    \
3900    {unsigned int _qzz_res;                                        \
3901     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3902                                VG_USERREQ__MALLOCLIKE_BLOCK,      \
3903                                addr, sizeB, rzB, is_zeroed, 0);   \
3904    }
3905
3906 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
3907    Ignored if addr == 0.
3908 */
3909 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                        \
3910    {unsigned int _qzz_res;                                        \
3911     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3912                                VG_USERREQ__FREELIKE_BLOCK,        \
3913                                addr, rzB, 0, 0, 0);               \
3914    }
3915
3916 /* Create a memory pool. */
3917 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
3918    {unsigned int _qzz_res;                                        \
3919     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3920                                VG_USERREQ__CREATE_MEMPOOL,        \
3921                                pool, rzB, is_zeroed, 0, 0);       \
3922    }
3923
3924 /* Destroy a memory pool. */
3925 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
3926    {unsigned int _qzz_res;                                        \
3927     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3928                                VG_USERREQ__DESTROY_MEMPOOL,       \
3929                                pool, 0, 0, 0, 0);                 \
3930    }
3931
3932 /* Associate a piece of memory with a memory pool. */
3933 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
3934    {unsigned int _qzz_res;                                        \
3935     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3936                                VG_USERREQ__MEMPOOL_ALLOC,         \
3937                                pool, addr, size, 0, 0);           \
3938    }
3939
3940 /* Disassociate a piece of memory from a memory pool. */
3941 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
3942    {unsigned int _qzz_res;                                        \
3943     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3944                                VG_USERREQ__MEMPOOL_FREE,          \
3945                                pool, addr, 0, 0, 0);              \
3946    }
3947
3948 /* Disassociate any pieces outside a particular range. */
3949 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
3950    {unsigned int _qzz_res;                                        \
3951     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3952                                VG_USERREQ__MEMPOOL_TRIM,          \
3953                                pool, addr, size, 0, 0);           \
3954    }
3955
3956 /* Resize and/or move a piece associated with a memory pool. */
3957 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
3958    {unsigned int _qzz_res;                                        \
3959     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3960                                VG_USERREQ__MOVE_MEMPOOL,          \
3961                                poolA, poolB, 0, 0, 0);            \
3962    }
3963
3964 /* Resize and/or move a piece associated with a memory pool. */
3965 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
3966    {unsigned int _qzz_res;                                        \
3967     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3968                                VG_USERREQ__MEMPOOL_CHANGE,        \
3969                                pool, addrA, addrB, size, 0);      \
3970    }
3971
3972 /* Return 1 if a mempool exists, else 0. */
3973 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
3974    __extension__                                                  \
3975    ({unsigned int _qzz_res;                                       \
3976     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3977                                VG_USERREQ__MEMPOOL_EXISTS,        \
3978                                pool, 0, 0, 0, 0);                 \
3979     _qzz_res;                                                     \
3980    })
3981
3982 /* Mark a piece of memory as being a stack. Returns a stack id. */
3983 #define VALGRIND_STACK_REGISTER(start, end)                       \
3984    __extension__                                                  \
3985    ({unsigned int _qzz_res;                                       \
3986     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3987                                VG_USERREQ__STACK_REGISTER,        \
3988                                start, end, 0, 0, 0);              \
3989     _qzz_res;                                                     \
3990    })
3991
3992 /* Unmark the piece of memory associated with a stack id as being a
3993    stack. */
3994 #define VALGRIND_STACK_DEREGISTER(id)                             \
3995    {unsigned int _qzz_res;                                        \
3996     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
3997                                VG_USERREQ__STACK_DEREGISTER,      \
3998                                id, 0, 0, 0, 0);                   \
3999    }
4000
4001 /* Change the start and end address of the stack id. */
4002 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
4003    {unsigned int _qzz_res;                                        \
4004     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4005                                VG_USERREQ__STACK_CHANGE,          \
4006                                id, start, end, 0, 0);             \
4007    }
4008
4009 /* Load PDB debug info for Wine PE image_map. */
4010 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)   \
4011    {unsigned int _qzz_res;                                        \
4012     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4013                                VG_USERREQ__LOAD_PDB_DEBUGINFO,    \
4014                                fd, ptr, total_size, delta, 0);    \
4015    }
4016
4017
4018 #undef PLAT_x86_linux
4019 #undef PLAT_amd64_linux
4020 #undef PLAT_ppc32_linux
4021 #undef PLAT_ppc64_linux
4022 #undef PLAT_ppc32_aix5
4023 #undef PLAT_ppc64_aix5
4024
4025 #else /* __GNUC__ */
4026 #define RUNNING_ON_VALGRIND 0
4027 #endif
4028 #endif   /* __VALGRIND_H */