coregrind/m_libcproc.c

   1
   2 /*--------------------------------------------------------------------*/
   3 /*--- Process-related libc stuff.                     m_libcproc.c ---*/
   4 /*--------------------------------------------------------------------*/
   5
   6 /*
   7    This file is part of Valgrind, a dynamic binary instrumentation
   8    framework.
   9
  10    Copyright (C) 2000-2017 Julian Seward
  11       jseward@acm.org
  12
  13    This program is free software; you can redistribute it and/or
  14    modify it under the terms of the GNU General Public License as
  15    published by the Free Software Foundation; either version 2 of the
  16    License, or (at your option) any later version.
  17
  18    This program is distributed in the hope that it will be useful, but
  19    WITHOUT ANY WARRANTY; without even the implied warranty of
  20    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  21    General Public License for more details.
  22
  23    You should have received a copy of the GNU General Public License
  24    along with this program; if not, see <http://www.gnu.org/licenses/>.
  25
  26    The GNU General Public License is contained in the file COPYING.
  27 */
  28
  29 #include "pub_core_basics.h"
  30 #include "pub_core_machine.h"    // For VG_(machine_get_VexArchInfo)
  31 #include "pub_core_vki.h"
  32 #include "pub_core_vkiscnums.h"
  33 #include "pub_core_libcbase.h"
  34 #include "pub_core_libcassert.h"
  35 #include "pub_core_libcfile.h"
  36 #include "pub_core_libcprint.h"
  37 #include "pub_core_libcproc.h"
  38 #include "pub_core_libcsignal.h"
  39 #include "pub_core_seqmatch.h"
  40 #include "pub_core_mallocfree.h"
  41 #include "pub_core_syscall.h"
  42 #include "pub_core_xarray.h"
  43 #include "pub_core_clientstate.h"
  44
  45 #if defined(VGO_darwin)
  46 /* --- !!! --- EXTERNAL HEADERS start --- !!! --- */
  47 #include <mach/mach.h>   /* mach_thread_self */
  48 /* --- !!! --- EXTERNAL HEADERS end --- !!! --- */
  49 #endif
  50
  51 /* IMPORTANT: on Darwin it is essential to use the _nocancel versions
  52    of syscalls rather than the vanilla version, if a _nocancel version
  53    is available.  See docs/internals/Darwin-notes.txt for the reason
  54    why. */
  55
  56 /* ---------------------------------------------------------------------
  57    Command line and environment stuff
  58    ------------------------------------------------------------------ */
  59
  60 /* As deduced from sp_at_startup, the client's argc, argv[] and
  61    envp[] as extracted from the client's stack at startup-time. */
  62 HChar** VG_(client_envp) = NULL;
  63
  64 /* Path to library directory */
  65 const HChar *VG_(libdir) = VG_LIBDIR;
  66
  67 const HChar *VG_(LD_PRELOAD_var_name) =
  68 #if defined(VGO_linux) || defined(VGO_solaris)
  69    "LD_PRELOAD";
  70 #elif defined(VGO_darwin)
  71    "DYLD_INSERT_LIBRARIES";
  72 #else
  73 #  error Unknown OS
  74 #endif
  75
  76 /* We do getenv without libc's help by snooping around in
  77    VG_(client_envp) as determined at startup time. */
  78 HChar *VG_(getenv)(const HChar *varname)
  79 {
  80    Int i, n;
  81    vg_assert( VG_(client_envp) );
  82    n = VG_(strlen)(varname);
  83    for (i = 0; VG_(client_envp)[i] != NULL; i++) {
  84       HChar* s = VG_(client_envp)[i];
  85       if (VG_(strncmp)(varname, s, n) == 0 && s[n] == '=') {
  86          return & s[n+1];
  87       }
  88    }
  89    return NULL;
  90 }
  91
  92 /* If free_fn is not NULL, it is called on "unset" environment variable. */
  93 void  VG_(env_unsetenv) ( HChar **env, const HChar *varname,
  94                           void (*free_fn) (void *) )
  95 {
  96    HChar **from, **to;
  97    vg_assert(env);
  98    vg_assert(varname);
  99    to = NULL;
 100    Int len = VG_(strlen)(varname);
 101
 102    for (from = to = env; from && *from; from++) {
 103       if (!(VG_(strncmp)(varname, *from, len) == 0 && (*from)[len] == '=')) {
 104          *to = *from;
 105          to++;
 106       } else if (free_fn != NULL) {
 107          free_fn(*from);
 108       }
 109    }
 110    *to = *from;
 111 }
 112
 113 /* set the environment; returns the old env if a new one was allocated */
 114 HChar **VG_(env_setenv) ( HChar ***envp, const HChar* varname,
 115                           const HChar *val )
 116 {
 117    HChar **env = (*envp);
 118    HChar **cpp;
 119    Int len = VG_(strlen)(varname);
 120    HChar *valstr = VG_(malloc)("libcproc.es.1", len + VG_(strlen)(val) + 2);
 121    HChar **oldenv = NULL;
 122
 123    VG_(sprintf)(valstr, "%s=%s", varname, val);
 124
 125    for (cpp = env; cpp && *cpp; cpp++) {
 126       if (VG_(strncmp)(varname, *cpp, len) == 0 && (*cpp)[len] == '=') {
 127          *cpp = valstr;
 128          return oldenv;
 129       }
 130    }
 131
 132    if (env == NULL) {
 133       env = VG_(malloc)("libcproc.es.2", sizeof(HChar *) * 2);
 134       env[0] = valstr;
 135       env[1] = NULL;
 136
 137       *envp = env;
 138
 139    }  else {
 140       Int envlen = (cpp-env) + 2;
 141       HChar **newenv = VG_(malloc)("libcproc.es.3", envlen * sizeof(HChar *));
 142
 143       for (cpp = newenv; *env; )
 144          *cpp++ = *env++;
 145       *cpp++ = valstr;
 146       *cpp++ = NULL;
 147
 148       oldenv = *envp;
 149
 150       *envp = newenv;
 151    }
 152
 153    return oldenv;
 154 }
 155
 156
 157 /* Walk through a colon-separated environment variable, and remove the
 158    entries which match remove_pattern.  It slides everything down over
 159    the removed entries, and pads the remaining space with '\0'.  It
 160    modifies the entries in place (in the client address space), but it
 161    shouldn't matter too much, since we only do this just before an
 162    execve().
 163
 164    This is also careful to mop up any excess ':'s, since empty strings
 165    delimited by ':' are considered to be '.' in a path.
 166 */
 167 static void mash_colon_env(HChar *varp, const HChar *remove_pattern)
 168 {
 169    HChar *const start = varp;
 170    HChar *entry_start = varp;
 171    HChar *output = varp;
 172
 173    if (varp == NULL)
 174       return;
 175
 176    while(*varp) {
 177       if (*varp == ':') {
 178          HChar prev;
 179          Bool match;
 180
 181          /* This is a bit subtle: we want to match against the entry
 182             we just copied, because it may have overlapped with
 183             itself, junking the original. */
 184
 185          prev = *output;
 186          *output = '\0';
 187
 188          match = VG_(string_match)(remove_pattern, entry_start);
 189
 190          *output = prev;
 191
 192          if (match) {
 193             output = entry_start;
 194             varp++;                     /* skip ':' after removed entry */
 195          } else
 196             entry_start = output+1;     /* entry starts after ':' */
 197       }
 198
 199       if (*varp)
 200          *output++ = *varp++;
 201    }
 202
 203    /* make sure last entry is nul terminated */
 204    *output = '\0';
 205
 206    /* match against the last entry */
 207    if (VG_(string_match)(remove_pattern, entry_start)) {
 208       output = entry_start;
 209       if (output > start) {
 210          /* remove trailing ':' */
 211          output--;
 212          vg_assert(*output == ':');
 213       }
 214    }
 215
 216    /* pad out the left-overs with '\0' */
 217    while(output < varp)
 218       *output++ = '\0';
 219 }
 220
 221
 222 /* Removes all the Valgrind-added stuff from the passed environment.  Used
 223    when starting child processes, so they don't see that added stuff.
 224    If the ro_strings option is set to True then all strings referenced by envp
 225    are considered read-only, which means they will be duplicated before they
 226    are modified.
 227    If free_fn is not NULL, it is called on "unset" environment variables. */
 228 void VG_(env_remove_valgrind_env_stuff)(HChar** envp, Bool ro_strings,
 229                                         void (*free_fn) (void *) )
 230 {
 231    Int i;
 232    HChar* ld_preload_str = NULL;
 233    HChar* ld_library_path_str = NULL;
 234    HChar* dyld_insert_libraries_str = NULL;
 235    HChar* buf;
 236
 237    // Find LD_* variables
 238    // DDD: should probably conditionally compiled some of this:
 239    // - LD_LIBRARY_PATH is universal?
 240    // - LD_PRELOAD is on Linux, not on Darwin, not sure about AIX
 241    // - DYLD_INSERT_LIBRARIES and DYLD_SHARED_REGION are Darwin-only
 242    for (i = 0; envp[i] != NULL; i++) {
 243       if (VG_(strncmp)(envp[i], "LD_PRELOAD=", 11) == 0) {
 244          if (ro_strings)
 245             envp[i] = VG_(strdup)("libcproc.erves.1", envp[i]);
 246          ld_preload_str = &envp[i][11];
 247       }
 248       if (VG_(strncmp)(envp[i], "LD_LIBRARY_PATH=", 16) == 0) {
 249          if (ro_strings)
 250             envp[i] = VG_(strdup)("libcproc.erves.2", envp[i]);
 251          ld_library_path_str = &envp[i][16];
 252       }
 253       if (VG_(strncmp)(envp[i], "DYLD_INSERT_LIBRARIES=", 22) == 0) {
 254          if (ro_strings)
 255             envp[i] = VG_(strdup)("libcproc.erves.3", envp[i]);
 256          dyld_insert_libraries_str = &envp[i][22];
 257       }
 258    }
 259
 260    buf = VG_(malloc)("libcproc.erves.4", VG_(strlen)(VG_(libdir)) + 20);
 261
 262    // Remove Valgrind-specific entries from LD_*.
 263    VG_(sprintf)(buf, "%s*/vgpreload_*.so", VG_(libdir));
 264    mash_colon_env(ld_preload_str, buf);
 265    mash_colon_env(dyld_insert_libraries_str, buf);
 266    VG_(sprintf)(buf, "%s*", VG_(libdir));
 267    mash_colon_env(ld_library_path_str, buf);
 268
 269    // Remove VALGRIND_LAUNCHER variable.
 270    VG_(env_unsetenv)(envp, VALGRIND_LAUNCHER, free_fn);
 271
 272    // Remove DYLD_SHARED_REGION variable.
 273    VG_(env_unsetenv)(envp, "DYLD_SHARED_REGION", free_fn);
 274
 275    // XXX if variable becomes empty, remove it completely?
 276
 277    VG_(free)(buf);
 278 }
 279
 280 /* Resolves filename of VG_(cl_exec_fd) and copies it to the buffer.
 281    Buffer must not be NULL and buf_size must be at least 1.
 282    If buffer is not large enough it is terminated with '\0' only
 283    when 'terminate_with_NUL == True'. */
 284 void VG_(client_fname)(HChar *buffer, SizeT buf_size, Bool terminate_with_NUL)
 285 {
 286    vg_assert(buffer != NULL);
 287    vg_assert(buf_size >= 1);
 288
 289    const HChar *name;
 290    if (VG_(resolve_filename)(VG_(cl_exec_fd), &name)) {
 291       const HChar *n = name + VG_(strlen)(name) - 1;
 292
 293       while (n > name && *n != '/')
 294          n--;
 295       if (n != name)
 296          n++;
 297
 298       VG_(strncpy)(buffer, n, buf_size);
 299       if (terminate_with_NUL)
 300          buffer[buf_size - 1] = '\0';
 301    } else {
 302       buffer[0] = '\0';
 303    }
 304 }
 305
 306 static Bool add_string(HChar *buffer, SizeT *buf_size, const HChar *string)
 307 {
 308    SizeT len = VG_(strlen)(string);
 309    VG_(strncat)(buffer, string, *buf_size);
 310    if (len >= *buf_size - 1) {
 311       *buf_size = 0;
 312       return False;
 313    } else {
 314       *buf_size -= len;
 315       return True;
 316    }
 317 }
 318
 319 /* Concatenates client exename and command line arguments into
 320    the buffer. Buffer must not be NULL and buf_size must be
 321    at least 1. Buffer is always terminated with '\0'. */
 322 void VG_(client_cmd_and_args)(HChar *buffer, SizeT buf_size)
 323 {
 324    vg_assert(buffer != NULL);
 325    vg_assert(buf_size >= 1);
 326
 327    buffer[0] = '\0';
 328
 329    if (add_string(buffer, &buf_size, VG_(args_the_exename)) == False)
 330       return;
 331
 332    Int i;
 333    for (i = 0; i < VG_(sizeXA)(VG_(args_for_client)); i++) {
 334       if (add_string(buffer, &buf_size, " ") == False)
 335          return;
 336
 337       HChar *arg = *(HChar **) VG_(indexXA)(VG_(args_for_client), i);
 338       if (add_string(buffer, &buf_size, arg) == False)
 339          return;
 340    }
 341 }
 342
 343 /* ---------------------------------------------------------------------
 344    Various important syscall wrappers
 345    ------------------------------------------------------------------ */
 346
 347 Int VG_(waitpid)(Int pid, Int *status, Int options)
 348 {
 349 #  if defined(VGO_linux)
 350    SysRes res = VG_(do_syscall4)(__NR_wait4,
 351                                  pid, (UWord)status, options, 0);
 352    return sr_isError(res) ? -1 : sr_Res(res);
 353 #  elif defined(VGO_darwin)
 354    SysRes res = VG_(do_syscall4)(__NR_wait4_nocancel,
 355                                  pid, (UWord)status, options, 0);
 356    return sr_isError(res) ? -1 : sr_Res(res);
 357 #  elif defined(VGO_solaris)
 358    SysRes res;
 359    vki_idtype_t idtype;
 360    vki_id_t id;
 361    vki_siginfo_t info;
 362
 363    /* We need to do a lot of work here. */
 364
 365    if (pid > 0) {
 366       idtype = VKI_P_PID;
 367       id = pid;
 368    }
 369    else if (pid < -1) {
 370       idtype = VKI_P_PGID;
 371       id = -pid;
 372    }
 373    else if (pid == -1) {
 374       idtype = VKI_P_ALL;
 375       id = 0;
 376    }
 377    else {
 378       idtype = VKI_P_PGID;
 379       res = VG_(do_syscall0)(__NR_getpid);
 380       id = sr_ResHI(res);
 381    }
 382
 383    options |= VKI_WEXITED | VKI_WTRAPPED;
 384
 385    res = VG_(do_syscall4)(__NR_waitsys, idtype, id, (UWord)&info, options);
 386    if (sr_isError(res))
 387       return -1;
 388
 389    if (status) {
 390       Int s = info.si_status & 0xff;
 391
 392       switch (info.si_code) {
 393          case VKI_CLD_EXITED:
 394             s <<= 8;
 395             break;
 396          case VKI_CLD_DUMPED:
 397             s |= VKI_WCOREFLG;
 398             break;
 399          case VKI_CLD_KILLED:
 400             break;
 401          case VKI_CLD_TRAPPED:
 402          case VKI_CLD_STOPPED:
 403             s <<= 8;
 404             s |= VKI_WSTOPFLG;
 405             break;
 406          case VKI_CLD_CONTINUED:
 407             s = VKI_WCONTFLG;
 408             break;
 409       }
 410       *status = s;
 411    }
 412
 413    return info.si_pid;
 414 #  else
 415 #    error Unknown OS
 416 #  endif
 417 }
 418
 419 /* clone the environment */
 420 HChar **VG_(env_clone) ( HChar **oldenv )
 421 {
 422    HChar **oldenvp;
 423    HChar **newenvp;
 424    HChar **newenv;
 425    Int  envlen;
 426
 427    vg_assert(oldenv);
 428    for (oldenvp = oldenv; oldenvp && *oldenvp; oldenvp++);
 429
 430    envlen = oldenvp - oldenv + 1;
 431
 432    newenv = VG_(malloc)("libcproc.ec.1", envlen * sizeof(HChar *));
 433
 434    oldenvp = oldenv;
 435    newenvp = newenv;
 436
 437    while (oldenvp && *oldenvp) {
 438       *newenvp++ = *oldenvp++;
 439    }
 440
 441    *newenvp = *oldenvp;
 442
 443    return newenv;
 444 }
 445
 446 void VG_(execv) ( const HChar* filename, const HChar** argv )
 447 {
 448    HChar** envp;
 449    SysRes res;
 450
 451    envp = VG_(env_clone)(VG_(client_envp));
 452    VG_(env_remove_valgrind_env_stuff)( envp, True /*ro_strings*/, NULL );
 453
 454    res = VG_(do_syscall3)(__NR_execve,
 455                           (UWord)filename, (UWord)argv, (UWord)envp);
 456
 457    VG_(printf)("EXEC failed, errno = %lld\n", (Long)sr_Err(res));
 458 }
 459
 460 /* Spawns a new child. Uses either spawn syscall or fork+execv combo. */
 461 Int VG_(spawn) ( const HChar *filename, const HChar **argv )
 462 {
 463    vg_assert(filename != NULL);
 464    vg_assert(argv != NULL);
 465
 466 #  if defined(VGO_solaris) && defined(SOLARIS_SPAWN_SYSCALL)
 467    HChar **envp = VG_(env_clone)(VG_(client_envp));
 468    for (HChar **p = envp; *p != NULL; p++) {
 469       *p = VG_(strdup)("libcproc.s.1", *p);
 470    }
 471    VG_(env_remove_valgrind_env_stuff)(envp, /* ro_strings */ False, VG_(free));
 472
 473    /* Now combine argv and argp into argenv. */
 474    SizeT argenv_size = 1 + 1;
 475    for (const HChar **p = argv; *p != NULL; p++) {
 476       argenv_size += VG_(strlen)(*p) + 2;
 477    }
 478    for (HChar **p = envp; *p != NULL; p++) {
 479       argenv_size += VG_(strlen)(*p) + 2;
 480    }
 481
 482    HChar *argenv = VG_(malloc)("libcproc.s.2", argenv_size);
 483    HChar *current = argenv;
 484 #  define COPY_CHAR_TO_ARGENV(dst, character)  \
 485       do {                                     \
 486          *(dst) = character;                   \
 487          (dst) += 1;                           \
 488       } while (0)
 489 #  define COPY_STRING_TO_ARGENV(dst, src)        \
 490       do {                                       \
 491          COPY_CHAR_TO_ARGENV(dst, '\1');         \
 492          SizeT src_len = VG_(strlen)((src)) + 1; \
 493          VG_(memcpy)((dst), (src), src_len);     \
 494          (dst) += src_len;                       \
 495       } while (0)
 496
 497    for (const HChar **p = argv; *p != NULL; p++) {
 498       COPY_STRING_TO_ARGENV(current, *p);
 499    }
 500    COPY_CHAR_TO_ARGENV(current, '\0');
 501    for (HChar **p = envp; *p != NULL; p++) {
 502       COPY_STRING_TO_ARGENV(current, *p);
 503    }
 504    COPY_CHAR_TO_ARGENV(current, '\0');
 505    vg_assert(current == argenv + argenv_size);
 506 #  undef COPY_CHAR_TO_ARGENV
 507 #  undef COPY_STRING_TOARGENV
 508
 509    SysRes res = VG_(do_syscall5)(__NR_spawn, (UWord) filename, (UWord) NULL, 0,
 510                                  (UWord) argenv, argenv_size);
 511
 512    VG_(free)(argenv);
 513    for (HChar **p = envp; *p != NULL; p++) {
 514       VG_(free)(*p);
 515    }
 516    VG_(free)(envp);
 517
 518    if (sr_isError(res))
 519       return -1;
 520    return sr_Res(res);
 521
 522 #  else
 523
 524    Int pid = VG_(fork)();
 525    if (pid < 0)
 526       return -1;
 527    if (pid == 0) {
 528       /* child */
 529       VG_(execv)(argv[0], argv);
 530
 531       /* If we're still alive here, execv failed. */
 532       VG_(exit)(1);
 533    } else {
 534       return pid;
 535    }
 536 #  endif /* VGO_solaris && SOLARIS_SPAWN_SYSCALL */
 537 }
 538
 539 /* Return -1 if error, else 0.  NOTE does not indicate return code of
 540    child! */
 541 Int VG_(system) ( const HChar* cmd )
 542 {
 543    Int pid;
 544    if (cmd == NULL)
 545       return 1;
 546
 547    const HChar *argv[4] = { "/bin/sh", "-c", cmd, 0 };
 548    pid = VG_(spawn)(argv[0], argv);
 549    if (pid < 0)
 550       return -1;
 551
 552    vg_assert(pid > 0);
 553    /* parent */
 554    /* We have to set SIGCHLD to its default behaviour in order that
 555       VG_(waitpid) works (at least on AIX).  According to the Linux
 556       man page for waitpid:
 557
 558       POSIX.1-2001 specifies that if the disposition of SIGCHLD is
 559       set to SIG_IGN or the SA_NOCLDWAIT flag is set for SIGCHLD
 560       (see sigaction(2)), then children that terminate do not
 561       become zombies and a call to wait() or waitpid() will block
 562       until all children have terminated, and then fail with errno
 563       set to ECHILD.  (The original POSIX standard left the
 564       behaviour of setting SIGCHLD to SIG_IGN unspecified.)
 565    */
 566    Int ir, zzz;
 567    vki_sigaction_toK_t sa, sa2;
 568    vki_sigaction_fromK_t saved_sa;
 569    VG_(memset)( &sa, 0, sizeof(sa) );
 570    VG_(sigemptyset)(&sa.sa_mask);
 571    sa.ksa_handler = VKI_SIG_DFL;
 572    sa.sa_flags    = 0;
 573    ir = VG_(sigaction)(VKI_SIGCHLD, &sa, &saved_sa);
 574    vg_assert(ir == 0);
 575
 576    zzz = VG_(waitpid)(pid, NULL, 0);
 577
 578    VG_(convert_sigaction_fromK_to_toK)( &saved_sa, &sa2 );
 579    ir = VG_(sigaction)(VKI_SIGCHLD, &sa2, NULL);
 580    vg_assert(ir == 0);
 581    return zzz == -1 ? -1 : 0;
 582 }
 583
 584 Int VG_(sysctl)(Int *name, UInt namelen, void *oldp, SizeT *oldlenp, void *newp, SizeT newlen)
 585 {
 586    SysRes res;
 587 #  if defined(VGO_darwin)
 588    res = VG_(do_syscall6)(__NR___sysctl,
 589                            (UWord)name, namelen, (UWord)oldp, (UWord)oldlenp, (UWord)newp, newlen);
 590 #  else
 591    res = VG_(mk_SysRes_Error)(VKI_ENOSYS);
 592 #  endif
 593    return sr_isError(res) ? -1 : sr_Res(res);
 594 }
 595
 596 /* ---------------------------------------------------------------------
 597    Resource limits
 598    ------------------------------------------------------------------ */
 599
 600 /* Support for getrlimit. */
 601 Int VG_(getrlimit) (Int resource, struct vki_rlimit *rlim)
 602 {
 603    SysRes res;
 604    /* res = getrlimit( resource, rlim ); */
 605
 606 #  if defined(__NR_prlimit64) && defined(VKI_RLIM_INFINITY) && defined(VKI_RLIM64_INFINITY)
 607    struct vki_rlimit64 new_rlimit;
 608    res = VG_(do_syscall4)(__NR_prlimit64, 0, resource, 0, (UWord)&new_rlimit);
 609    if (!sr_isError(res)) {
 610       if (new_rlimit.rlim_cur == VKI_RLIM_INFINITY)
 611          new_rlimit.rlim_cur = VKI_RLIM64_INFINITY;
 612       if (new_rlimit.rlim_max == VKI_RLIM_INFINITY)
 613          new_rlimit.rlim_max = VKI_RLIM64_INFINITY;
 614       rlim->rlim_cur = new_rlimit.rlim_cur;
 615       rlim->rlim_max = new_rlimit.rlim_max;
 616       return sr_Res(res);
 617    }
 618    if (sr_Err(res) != VKI_ENOSYS) return -1;
 619 #  endif
 620
 621 #  ifdef __NR_ugetrlimit
 622    res = VG_(do_syscall2)(__NR_ugetrlimit, resource, (UWord)rlim);
 623    if (!sr_isError(res)) return sr_Res(res);
 624    if (sr_Err(res) != VKI_ENOSYS) return -1;
 625 #  endif
 626
 627 #  ifdef __NR_getrlimit
 628    res = VG_(do_syscall2)(__NR_getrlimit, resource, (UWord)rlim);
 629    if (!sr_isError(res)) return sr_Res(res);
 630 #  endif
 631
 632    return -1;
 633 }
 634
 635 /* Support for setrlimit. */
 636 Int VG_(setrlimit) (Int resource, const struct vki_rlimit *rlim)
 637 {
 638    SysRes res;
 639    /* res = setrlimit( resource, rlim ); */
 640
 641 #  ifdef __NR_prlimit64
 642    struct vki_rlimit64 new_rlimit;
 643    new_rlimit.rlim_cur = rlim->rlim_cur;
 644    new_rlimit.rlim_max = rlim->rlim_max;
 645    res = VG_(do_syscall4)(__NR_prlimit64, 0, resource, (UWord)&new_rlimit, 0);
 646    if (!sr_isError(res)) return sr_Res(res);
 647    if (sr_Err(res) != VKI_ENOSYS) return -1;
 648 #  endif
 649
 650 #  ifdef __NR_setrlimit
 651    res = VG_(do_syscall2)(__NR_setrlimit, resource, (UWord)rlim);
 652    if (!sr_isError(res)) return sr_Res(res);
 653    if (sr_Err(res) != VKI_ENOSYS) return -1;
 654 #  endif
 655
 656    return -1;
 657 }
 658
 659 /* Support for prctl. */
 660 Int VG_(prctl) (Int option,
 661                 ULong arg2, ULong arg3, ULong arg4, ULong arg5)
 662 {
 663    SysRes res = VG_(mk_SysRes_Error)(VKI_ENOSYS);
 664 #  if defined(VGO_linux)
 665    /* res = prctl( option, arg2, arg3, arg4, arg5 ); */
 666    res = VG_(do_syscall5)(__NR_prctl, (UWord) option,
 667                           (UWord) arg2, (UWord) arg3, (UWord) arg4,
 668                           (UWord) arg5);
 669 #  endif
 670
 671    return sr_isError(res) ? -1 : sr_Res(res);
 672 }
 673
 674 /* ---------------------------------------------------------------------
 675    pids, etc
 676    ------------------------------------------------------------------ */
 677
 678 Int VG_(gettid)(void)
 679 {
 680 #  if defined(VGO_linux)
 681    SysRes res = VG_(do_syscall0)(__NR_gettid);
 682
 683    if (sr_isError(res) && sr_Res(res) == VKI_ENOSYS) {
 684       HChar pid[16];
 685       /*
 686        * The gettid system call does not exist. The obvious assumption
 687        * to make at this point would be that we are running on an older
 688        * system where the getpid system call actually returns the ID of
 689        * the current thread.
 690        *
 691        * Unfortunately it seems that there are some systems with a kernel
 692        * where getpid has been changed to return the ID of the thread group
 693        * leader but where the gettid system call has not yet been added.
 694        *
 695        * So instead of calling getpid here we use readlink to see where
 696        * the /proc/self link is pointing...
 697        */
 698
 699 #     if defined(VGP_arm64_linux) || defined(VGP_nanomips_linux)
 700       res = VG_(do_syscall4)(__NR_readlinkat, VKI_AT_FDCWD,
 701                              (UWord)"/proc/self",
 702                              (UWord)pid, sizeof(pid));
 703 #     else
 704       res = VG_(do_syscall3)(__NR_readlink, (UWord)"/proc/self",
 705                              (UWord)pid, sizeof(pid));
 706 #     endif
 707       if (!sr_isError(res) && sr_Res(res) > 0) {
 708          HChar* s;
 709          pid[sr_Res(res)] = '\0';
 710          res = VG_(mk_SysRes_Success)(  VG_(strtoll10)(pid, &s) );
 711          if (*s != '\0') {
 712             VG_(message)(Vg_DebugMsg,
 713                "Warning: invalid file name linked to by /proc/self: %s\n",
 714                pid);
 715          }
 716       }
 717    }
 718
 719    return sr_Res(res);
 720
 721 #  elif defined(VGO_darwin)
 722    // Darwin's gettid syscall is something else.
 723    // Use Mach thread ports for lwpid instead.
 724    return mach_thread_self();
 725
 726 #  elif defined(VGO_solaris)
 727    SysRes res = VG_(do_syscall0)(__NR_lwp_self);
 728    return sr_Res(res);
 729
 730 #  else
 731 #    error "Unknown OS"
 732 #  endif
 733 }
 734
 735 /* You'd be amazed how many places need to know the current pid. */
 736 Int VG_(getpid) ( void )
 737 {
 738    /* ASSUMES SYSCALL ALWAYS SUCCEEDS */
 739    return sr_Res( VG_(do_syscall0)(__NR_getpid) );
 740 }
 741
 742 Int VG_(getpgrp) ( void )
 743 {
 744    /* ASSUMES SYSCALL ALWAYS SUCCEEDS */
 745 #  if defined(VGP_arm64_linux) || defined(VGP_nanomips_linux)
 746    return sr_Res( VG_(do_syscall1)(__NR_getpgid, 0) );
 747 #  elif defined(VGO_linux) || defined(VGO_darwin)
 748    return sr_Res( VG_(do_syscall0)(__NR_getpgrp) );
 749 #  elif defined(VGO_solaris)
 750    /* Uses the shared pgrpsys syscall, 0 for the getpgrp variant. */
 751    return sr_Res( VG_(do_syscall1)(__NR_pgrpsys, 0) );
 752 #  else
 753 #    error Unknown OS
 754 #  endif
 755 }
 756
 757 Int VG_(getppid) ( void )
 758 {
 759    /* ASSUMES SYSCALL ALWAYS SUCCEEDS */
 760 #  if defined(VGO_linux) || defined(VGO_darwin)
 761    return sr_Res( VG_(do_syscall0)(__NR_getppid) );
 762 #  elif defined(VGO_solaris)
 763    /* Uses the shared getpid/getppid syscall, val2 contains a parent pid. */
 764    return sr_ResHI( VG_(do_syscall0)(__NR_getpid) );
 765 #  else
 766 #    error Unknown OS
 767 #  endif
 768 }
 769
 770 Int VG_(geteuid) ( void )
 771 {
 772    /* ASSUMES SYSCALL ALWAYS SUCCEEDS */
 773 #  if defined(VGO_linux) || defined(VGO_darwin)
 774    {
 775 #     if defined(__NR_geteuid32)
 776       // We use the 32-bit version if it's supported.  Otherwise, IDs greater
 777       // than 65536 cause problems, as bug #151209 showed.
 778       return sr_Res( VG_(do_syscall0)(__NR_geteuid32) );
 779 #     else
 780       return sr_Res( VG_(do_syscall0)(__NR_geteuid) );
 781 #     endif
 782    }
 783 #  elif defined(VGO_solaris)
 784    /* Uses the shared getuid/geteuid syscall, val2 contains the effective
 785       uid. */
 786    return sr_ResHI( VG_(do_syscall0)(__NR_getuid) );
 787 #  else
 788 #    error Unknown OS
 789 #  endif
 790 }
 791
 792 Int VG_(getegid) ( void )
 793 {
 794 #  if defined(VGO_linux) || defined(VGO_darwin)
 795    /* ASSUMES SYSCALL ALWAYS SUCCEEDS */
 796 #    if defined(__NR_getegid32)
 797    // We use the 32-bit version if it's supported.  Otherwise, IDs greater
 798    // than 65536 cause problems, as bug #151209 showed.
 799    return sr_Res( VG_(do_syscall0)(__NR_getegid32) );
 800 #    else
 801    return sr_Res( VG_(do_syscall0)(__NR_getegid) );
 802 #    endif
 803
 804 #  elif defined(VGO_solaris)
 805    /* Uses the shared getgid/getegid syscall, val2 contains the effective
 806       gid. */
 807    return sr_ResHI( VG_(do_syscall0)(__NR_getgid) );
 808 #  else
 809 #    error Unknown OS
 810 #  endif
 811 }
 812
 813 /* Get supplementary groups into list[0 .. size-1].  Returns the
 814    number of groups written, or -1 if error.  Note that in order to be
 815    portable, the groups are 32-bit unsigned ints regardless of the
 816    platform.
 817    As a special case, if size == 0 the function returns the number of
 818    groups leaving list untouched. */
 819 Int VG_(getgroups)( Int size, UInt* list )
 820 {
 821    if (size < 0) return -1;
 822
 823 #  if defined(VGP_x86_linux) || defined(VGP_ppc32_linux) \
 824       || defined(VGP_mips64_linux)
 825    Int    i;
 826    SysRes sres;
 827    UShort list16[size];
 828    sres = VG_(do_syscall2)(__NR_getgroups, size, (Addr)list16);
 829    if (sr_isError(sres))
 830       return -1;
 831    if (size != 0) {
 832       for (i = 0; i < sr_Res(sres); i++)
 833          list[i] = (UInt)list16[i];
 834    }
 835    return sr_Res(sres);
 836
 837 #  elif defined(VGP_amd64_linux) || defined(VGP_arm_linux) \
 838         || defined(VGP_ppc64be_linux) || defined(VGP_ppc64le_linux)  \
 839         || defined(VGO_darwin) || defined(VGP_s390x_linux)    \
 840         || defined(VGP_mips32_linux) || defined(VGP_arm64_linux) \
 841         || defined(VGO_solaris) || defined(VGP_nanomips_linux)
 842    SysRes sres;
 843    sres = VG_(do_syscall2)(__NR_getgroups, size, (Addr)list);
 844    if (sr_isError(sres))
 845       return -1;
 846    return sr_Res(sres);
 847
 848 #  else
 849 #     error "VG_(getgroups): needs implementation on this platform"
 850 #  endif
 851 }
 852
 853 /* ---------------------------------------------------------------------
 854    Process tracing
 855    ------------------------------------------------------------------ */
 856
 857 Int VG_(ptrace) ( Int request, Int pid, void *addr, void *data )
 858 {
 859    SysRes res;
 860 #  if defined(VGO_linux) || defined(VGO_darwin)
 861    res = VG_(do_syscall4)(__NR_ptrace, request, pid, (UWord)addr, (UWord)data);
 862 #  elif defined(VGO_solaris)
 863    /* There is no ptrace syscall on Solaris.  Such requests has to be
 864       implemented using the /proc interface.  Callers of VG_(ptrace) should
 865       ensure that this function is not reached on Solaris, i.e. they must
 866       provide a special code for Solaris for whatever feature they provide. */
 867    I_die_here;
 868 #  else
 869 #    error Unknown OS
 870 #  endif
 871    if (sr_isError(res))
 872       return -1;
 873    return sr_Res(res);
 874 }
 875
 876 /* ---------------------------------------------------------------------
 877    Fork
 878    ------------------------------------------------------------------ */
 879
 880 Int VG_(fork) ( void )
 881 {
 882 #  if defined(VGP_arm64_linux) || defined(VGP_nanomips_linux)
 883    SysRes res;
 884    res = VG_(do_syscall5)(__NR_clone, VKI_SIGCHLD,
 885                           (UWord)NULL, (UWord)NULL, (UWord)NULL, (UWord)NULL);
 886    if (sr_isError(res))
 887       return -1;
 888    return sr_Res(res);
 889
 890 #  elif defined(VGO_linux)
 891    SysRes res;
 892    res = VG_(do_syscall0)(__NR_fork);
 893    if (sr_isError(res))
 894       return -1;
 895    return sr_Res(res);
 896
 897 #  elif defined(VGO_darwin)
 898    SysRes res;
 899    res = VG_(do_syscall0)(__NR_fork); /* __NR_fork is UX64 */
 900    if (sr_isError(res))
 901       return -1;
 902    /* on success: wLO = child pid; wHI = 1 for child, 0 for parent */
 903    if (sr_ResHI(res) != 0) {
 904       return 0;  /* this is child: return 0 instead of child pid */
 905    }
 906    return sr_Res(res);
 907
 908 #  elif defined(VGO_solaris)
 909    /* Using fork() on Solaris is not really the best thing to do. Solaris
 910       does not do memory overcommitment so fork() can fail if there is not
 911       enough memory to copy the current process into a new one.
 912       Prefer to use VG_(spawn)() over VG_(fork)() + VG_(execv)(). */
 913    SysRes res;
 914    res = VG_(do_syscall2)(__NR_forksys, 0 /*subcode (fork)*/, 0 /*flags*/);
 915    if (sr_isError(res))
 916       return -1;
 917    /* On success:
 918         val = a pid of the child in the parent, a pid of the parent in the
 919               child,
 920         val2 = 0 in the parent process, 1 in the child process. */
 921    if (sr_ResHI(res) != 0) {
 922       return 0;
 923    }
 924    return sr_Res(res);
 925
 926 #  else
 927 #    error "Unknown OS"
 928 #  endif
 929 }
 930
 931 /* ---------------------------------------------------------------------
 932    Timing stuff
 933    ------------------------------------------------------------------ */
 934
 935 UInt VG_(read_millisecond_timer) ( void )
 936 {
 937    /* 'now' and 'base' are in microseconds */
 938    static ULong base = 0;
 939    ULong  now;
 940
 941 #  if defined(VGO_linux) || defined(VGO_solaris)
 942    { SysRes res;
 943      struct vki_timespec ts_now;
 944      res = VG_(do_syscall2)(__NR_clock_gettime, VKI_CLOCK_MONOTONIC,
 945                             (UWord)&ts_now);
 946      if (sr_isError(res) == 0) {
 947         now = ts_now.tv_sec * 1000000ULL + ts_now.tv_nsec / 1000;
 948      } else {
 949        struct vki_timeval tv_now;
 950        /* Note: On Solaris, this syscall takes only one parameter but the
 951           extra dummy one does not cause any harm. */
 952        res = VG_(do_syscall2)(__NR_gettimeofday, (UWord)&tv_now, (UWord)NULL);
 953        vg_assert(! sr_isError(res));
 954        now = tv_now.tv_sec * 1000000ULL + tv_now.tv_usec;
 955      }
 956    }
 957
 958 #  elif defined(VGO_darwin)
 959    // Weird: it seems that gettimeofday() doesn't fill in the timeval, but
 960    // rather returns the tv_sec as the low 32 bits of the result and the
 961    // tv_usec as the high 32 bits of the result.  (But the timeval cannot be
 962    // NULL!)  See bug 200990.
 963    { SysRes res;
 964      struct vki_timeval tv_now = { 0, 0 };
 965      res = VG_(do_syscall2)(__NR_gettimeofday, (UWord)&tv_now, (UWord)NULL);
 966      vg_assert(! sr_isError(res));
 967      now = sr_Res(res) * 1000000ULL + sr_ResHI(res);
 968    }
 969
 970 #  else
 971 #    error "Unknown OS"
 972 #  endif
 973
 974    /* COMMON CODE */
 975    if (base == 0)
 976       base = now;
 977
 978    return (now - base) / 1000;
 979 }
 980
 981 #  if defined(VGO_linux) || defined(VGO_solaris)
 982 void VG_(clock_gettime) ( struct vki_timespec *ts, vki_clockid_t clk_id )
 983 {
 984     SysRes res;
 985     res = VG_(do_syscall2)(__NR_clock_gettime, clk_id,
 986                            (UWord)ts);
 987     vg_assert (sr_isError(res) == 0);
 988 }
 989 #  elif defined(VGO_darwin)
 990   /* See pub_tool_libcproc.h */
 991 #  else
 992 #    error "Unknown OS"
 993 #  endif
 994
 995 Int VG_(gettimeofday)(struct vki_timeval *tv, struct vki_timezone *tz)
 996 {
 997    SysRes res;
 998    res = VG_(do_syscall2)(__NR_gettimeofday, (UWord)tv, (UWord)tz);
 999
1000    if (! sr_isError(res)) return 0;
1001
1002    /* Make sure, argument values are deterministic upon failure */
1003    if (tv) *tv = (struct vki_timeval){ .tv_sec = 0, .tv_usec = 0 };
1004    if (tz) *tz = (struct vki_timezone){ .tz_minuteswest = 0, .tz_dsttime = 0 };
1005
1006    return -1;
1007 }
1008
1009 UInt VG_(get_user_milliseconds)(void)
1010 {
1011    UInt res = 0;
1012 #  if defined(VGO_linux)
1013    {
1014       struct vki_rusage ru;
1015       VG_(memset)(&ru, 0, sizeof(ru));
1016       SysRes sr = VG_(do_syscall2)(__NR_getrusage, VKI_RUSAGE_SELF, (UWord)&ru);
1017       if (!sr_isError(sr)) {
1018          res = ru.ru_utime.tv_sec * 1000 + ru.ru_utime.tv_usec / 1000;
1019       }
1020    }
1021
1022 #  elif defined(VGO_solaris)
1023    {
1024       struct vki_rusage ru;
1025       VG_(memset)(&ru, 0, sizeof(ru));
1026       SysRes sr = VG_(do_syscall2)(__NR_rusagesys, VKI__RUSAGESYS_GETRUSAGE,
1027                                    (UWord) &ru);
1028       if (!sr_isError(sr)) {
1029          res = ru.ru_utime.tv_sec * 1000 + ru.ru_utime.tv_usec / 1000;
1030       }
1031    }
1032
1033 #  elif defined(VGO_darwin)
1034    res = 0;
1035
1036 #  else
1037 #    error "Unknown OS"
1038 #  endif
1039
1040    return res;
1041 }
1042
1043
1044 /* ---------------------------------------------------------------------
1045    atfork()
1046    ------------------------------------------------------------------ */
1047
1048 struct atfork {
1049    vg_atfork_t  pre;
1050    vg_atfork_t  parent;
1051    vg_atfork_t  child;
1052 };
1053
1054 #define VG_MAX_ATFORK 10
1055
1056 static struct atfork atforks[VG_MAX_ATFORK];
1057 static Int n_atfork = 0;
1058
1059 void VG_(atfork)(vg_atfork_t pre, vg_atfork_t parent, vg_atfork_t child)
1060 {
1061    Int i;
1062
1063    for (i = 0; i < n_atfork; i++) {
1064       if (atforks[i].pre == pre &&
1065           atforks[i].parent == parent &&
1066           atforks[i].child == child)
1067          return;
1068    }
1069
1070    if (n_atfork >= VG_MAX_ATFORK)
1071       VG_(core_panic)(
1072          "Too many VG_(atfork) handlers requested: raise VG_MAX_ATFORK");
1073
1074    atforks[n_atfork].pre    = pre;
1075    atforks[n_atfork].parent = parent;
1076    atforks[n_atfork].child  = child;
1077
1078    n_atfork++;
1079 }
1080
1081 void VG_(do_atfork_pre)(ThreadId tid)
1082 {
1083    Int i;
1084
1085    for (i = 0; i < n_atfork; i++)
1086       if (atforks[i].pre != NULL)
1087          (*atforks[i].pre)(tid);
1088 }
1089
1090 void VG_(do_atfork_parent)(ThreadId tid)
1091 {
1092    Int i;
1093
1094    for (i = 0; i < n_atfork; i++)
1095       if (atforks[i].parent != NULL)
1096          (*atforks[i].parent)(tid);
1097 }
1098
1099 void VG_(do_atfork_child)(ThreadId tid)
1100 {
1101    Int i;
1102
1103    for (i = 0; i < n_atfork; i++)
1104       if (atforks[i].child != NULL)
1105          (*atforks[i].child)(tid);
1106 }
1107
1108
1109 /* ---------------------------------------------------------------------
1110    icache invalidation
1111    ------------------------------------------------------------------ */
1112
1113 void VG_(invalidate_icache) ( void *ptr, SizeT nbytes )
1114 {
1115    if (nbytes == 0) return;    // nothing to do
1116
1117    // Get cache info
1118    VexArchInfo vai;
1119    VG_(machine_get_VexArchInfo)(NULL, &vai);
1120
1121    // If I-caches are coherent, nothing needs to be done here
1122    if (vai.hwcache_info.icaches_maintain_coherence) return;
1123
1124 #  if defined(VGA_ppc32) || defined(VGA_ppc64be) || defined(VGA_ppc64le)
1125    Addr startaddr = (Addr) ptr;
1126    Addr endaddr   = startaddr + nbytes;
1127    Addr cls;
1128    Addr addr;
1129
1130    cls = vai.ppc_icache_line_szB;
1131
1132    /* Stay sane .. */
1133    vg_assert(cls == 16 || cls == 32 || cls == 64 || cls == 128);
1134
1135    startaddr &= ~(cls - 1);
1136    for (addr = startaddr; addr < endaddr; addr += cls) {
1137       __asm__ __volatile__("dcbst 0,%0" : : "r" (addr));
1138    }
1139    __asm__ __volatile__("sync");
1140    for (addr = startaddr; addr < endaddr; addr += cls) {
1141       __asm__ __volatile__("icbi 0,%0" : : "r" (addr));
1142    }
1143    __asm__ __volatile__("sync; isync");
1144
1145 #  elif defined(VGP_arm_linux)
1146    /* ARM cache flushes are privileged, so we must defer to the kernel. */
1147    Addr startaddr = (Addr) ptr;
1148    Addr endaddr   = startaddr + nbytes;
1149    VG_(do_syscall2)(__NR_ARM_cacheflush, startaddr, endaddr);
1150
1151 #  elif defined(VGP_arm64_linux)
1152    // This arm64_linux section of this function VG_(invalidate_icache)
1153    // is copied from
1154    // https://github.com/armvixl/vixl/blob/master/src/a64/cpu-a64.cc
1155    // which has the following copyright notice:
1156    /*
1157    Copyright 2013, ARM Limited
1158    All rights reserved.
1159
1160    Redistribution and use in source and binary forms, with or without
1161    modification, are permitted provided that the following conditions are met:
1162
1163    * Redistributions of source code must retain the above copyright notice,
1164      this list of conditions and the following disclaimer.
1165    * Redistributions in binary form must reproduce the above copyright notice,
1166      this list of conditions and the following disclaimer in the documentation
1167      and/or other materials provided with the distribution.
1168    * Neither the name of ARM Limited nor the names of its contributors may be
1169      used to endorse or promote products derived from this software without
1170      specific prior written permission.
1171
1172    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
1173    ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1174    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1175    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
1176    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
1177    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
1178    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
1179    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
1180    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
1181    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1182    */
1183
1184    // Ask what the I and D line sizes are
1185    UInt cache_type_register;
1186    // Copy the content of the cache type register to a core register.
1187    __asm__ __volatile__ ("mrs %[ctr], ctr_el0" // NOLINT
1188                          : [ctr] "=r" (cache_type_register));
1189
1190    const Int kDCacheLineSizeShift = 16;
1191    const Int kICacheLineSizeShift = 0;
1192    const UInt kDCacheLineSizeMask = 0xf << kDCacheLineSizeShift;
1193    const UInt kICacheLineSizeMask = 0xf << kICacheLineSizeShift;
1194
1195    // The cache type register holds the size of the I and D caches as a power of
1196    // two.
1197    const UInt dcache_line_size_power_of_two =
1198        (cache_type_register & kDCacheLineSizeMask) >> kDCacheLineSizeShift;
1199    const UInt icache_line_size_power_of_two =
1200        (cache_type_register & kICacheLineSizeMask) >> kICacheLineSizeShift;
1201
1202    const UInt dcache_line_size_ = 4 * (1 << dcache_line_size_power_of_two);
1203    const UInt icache_line_size_ = 4 * (1 << icache_line_size_power_of_two);
1204
1205    Addr start = (Addr)ptr;
1206    // Sizes will be used to generate a mask big enough to cover a pointer.
1207    Addr dsize = (Addr)dcache_line_size_;
1208    Addr isize = (Addr)icache_line_size_;
1209
1210    // Cache line sizes are always a power of 2.
1211    Addr dstart = start & ~(dsize - 1);
1212    Addr istart = start & ~(isize - 1);
1213    Addr end    = start + nbytes;
1214
1215    __asm__ __volatile__ (
1216      // Clean every line of the D cache containing the target data.
1217      "0: \n\t"
1218      // dc : Data Cache maintenance
1219      // c : Clean
1220      // va : by (Virtual) Address
1221      // u : to the point of Unification
1222      // The point of unification for a processor is the point by which the
1223      // instruction and data caches are guaranteed to see the same copy of a
1224      // memory location. See ARM DDI 0406B page B2-12 for more information.
1225      "dc cvau, %[dline] \n\t"
1226      "add %[dline], %[dline], %[dsize] \n\t"
1227      "cmp %[dline], %[end] \n\t"
1228      "b.lt 0b \n\t"
1229      // Barrier to make sure the effect of the code above is visible to the rest
1230      // of the world.
1231      // dsb : Data Synchronisation Barrier
1232      // ish : Inner SHareable domain
1233      // The point of unification for an Inner Shareable shareability domain is
1234      // the point by which the instruction and data caches of all the processors
1235      // in that Inner Shareable shareability domain are guaranteed to see the
1236      // same copy of a memory location. See ARM DDI 0406B page B2-12 for more
1237      // information.
1238      "dsb ish \n\t"
1239      // Invalidate every line of the I cache containing the target data.
1240      "1: \n\t"
1241      // ic : instruction cache maintenance
1242      // i : invalidate
1243      // va : by address
1244      // u : to the point of unification
1245      "ic ivau, %[iline] \n\t"
1246      "add %[iline], %[iline], %[isize] \n\t"
1247      "cmp %[iline], %[end] \n\t"
1248      "b.lt 1b \n\t"
1249      // Barrier to make sure the effect of the code above is visible to the rest
1250      // of the world.
1251      "dsb ish \n\t"
1252      // Barrier to ensure any prefetching which happened before this code is
1253      // discarded.
1254      // isb : Instruction Synchronisation Barrier
1255      "isb \n\t"
1256      : [dline] "+r" (dstart),
1257        [iline] "+r" (istart)
1258      : [dsize] "r" (dsize),
1259        [isize] "r" (isize),
1260        [end] "r" (end)
1261      // This code does not write to memory but without the dependency gcc might
1262      // move this code before the code is generated.
1263      : "cc", "memory"
1264    );
1265
1266 #  elif defined(VGA_mips32) || defined(VGA_mips64)
1267    SysRes sres = VG_(do_syscall3)(__NR_cacheflush, (UWord) ptr,
1268                                  (UWord) nbytes, (UWord) 3);
1269    vg_assert( !sr_isError(sres) );
1270
1271 # elif defined(VGA_nanomips)
1272
1273    __builtin___clear_cache(ptr, (char*)ptr + nbytes);
1274
1275 #  endif
1276 }
1277
1278
1279 /* ---------------------------------------------------------------------
1280    dcache flushing
1281    ------------------------------------------------------------------ */
1282
1283 void VG_(flush_dcache) ( void *ptr, SizeT nbytes )
1284 {
1285    /* Currently this is only required on ARM64. */
1286 #  if defined(VGA_arm64)
1287    Addr startaddr = (Addr) ptr;
1288    Addr endaddr   = startaddr + nbytes;
1289    Addr cls;
1290    Addr addr;
1291
1292    ULong ctr_el0;
1293    __asm__ __volatile__ ("mrs %0, ctr_el0" : "=r"(ctr_el0));
1294    cls = 4 * (1ULL << (0xF & (ctr_el0 >> 16)));
1295
1296    /* Stay sane .. */
1297    vg_assert(cls == 64 || cls == 128);
1298
1299    startaddr &= ~(cls - 1);
1300    for (addr = startaddr; addr < endaddr; addr += cls) {
1301       __asm__ __volatile__("dc cvau, %0" : : "r" (addr));
1302    }
1303    __asm__ __volatile__("dsb ish");
1304 #  endif
1305 }
1306
1307 /*--------------------------------------------------------------------*/
1308 /*--- end                                                          ---*/
1309 /*--------------------------------------------------------------------*/