lib/jit/cache.h

   1 /* JIT compiler - Flushing the instruction cache.
   2
   3    Copyright (C) 1995-2024 Free Software Foundation, Inc.
   4
   5    This file is free software: you can redistribute it and/or modify
   6    it under the terms of the GNU Lesser General Public License as
   7    published by the Free Software Foundation; either version 2.1 of the
   8    License, or (at your option) any later version.
   9
  10    This file is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public License
  16    along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
  17
  18 /* Written by Bruno Haible <bruno@clisp.org>, 2020.  */
  19
  20 #include <stdint.h>
  21 #if ENABLE_VALGRIND_SUPPORT
  22 # include <valgrind/valgrind.h>
  23 #endif
  24 #if defined _WIN32 && !defined __CYGWIN__
  25 # define WIN32_LEAN_AND_MEAN
  26 # include <windows.h>
  27 #endif
  28 #if defined __APPLE__ && defined __MACH__
  29 # include <libkern/OSCacheControl.h>
  30 #endif
  31 #if defined _AIX
  32 # include <sys/cache.h>
  33 #endif
  34 #if defined __sgi
  35 # include <sys/cachectl.h>
  36 #endif
  37 #if defined __sun
  38 # include <stddef.h>
  39 #endif
  40
  41 #ifdef __cplusplus
  42 extern "C" {
  43 #endif
  44
  45
  46 /* Clears the instruction cache for addresses
  47    start <= address < end.
  48    We need this because some CPUs have separate data cache and instruction
  49    cache. The freshly built trampoline is visible to the data cache, but
  50    maybe not to the instruction cache. This is hairy.  */
  51 static inline void
  52 clear_cache (void *start, void *end)
  53 {
  54 #if ENABLE_VALGRIND_SUPPORT
  55   /* Documentation:
  56      <https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq>  */
  57   VALGRIND_DISCARD_TRANSLATIONS (start, (char *) end - (char *) start);
  58 #endif
  59 #if (defined __x86_64__ || defined _M_X64) || (defined __i386 || defined _M_IX86)
  60   /* On this architecture, data cache and instruction cache are not separate.
  61      Therefore, nothing to do.
  62      For details, see
  63      <https://stackoverflow.com/questions/10989403/how-is-x86-instruction-cache-synchronized>  */
  64
  65 /* Use the operating system provided function, when available.  */
  66 #elif defined _WIN32 && !defined __CYGWIN__
  67   /* Native Windows.
  68      FlushInstructionCache
  69      <https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-flushinstructioncache>  */
  70   HANDLE process = GetCurrentProcess ();
  71   while (!FlushInstructionCache (process, start, (char *) end - (char *) start))
  72     ;
  73 #elif defined __APPLE__ && defined __MACH__
  74   /* macOS  */
  75   sys_icache_invalidate (start, (char *) end - (char *) start);
  76 #elif defined _AIX
  77   /* AIX.  */
  78   _sync_cache_range (start, (char *) end - (char *) start);
  79 #elif defined __sgi
  80   /* IRIX.  */
  81   cacheflush (start, (char *) end - (char *) start, ICACHE);
  82 #elif defined __sun
  83   /* Solaris.  */
  84   extern void sync_instruction_memory (char *, size_t);
  85   sync_instruction_memory (start, (char *) end - (char *) start);
  86
  87 /* No operating system provided function. Dispatch according to the CPU.  */
  88 #elif (defined __GNUC__ || defined __clang__) && defined __powerpc__
  89   /* XXX Is this enough, or do we also need the 'clf' instruction?  */
  90   uintptr_t addr = (uintptr_t) start & ~(intptr_t)3;
  91   uintptr_t end_addr = (uintptr_t) end;
  92   do
  93     {
  94       asm volatile ("icbi 0,%0; dcbf 0,%0" : : "r" (addr));
  95       addr += 4;
  96     }
  97   while (addr < end_addr);
  98   asm volatile ("sync; isync");
  99 #elif (defined __GNUC__ || defined __clang__) && defined __sparc
 100   /* Use inline assembly.  */
 101   /* The 'flush' instruction was earlier called 'iflush'.  */
 102   uintptr_t addr = (uintptr_t) start & ~(intptr_t)7;
 103   uintptr_t end_addr = (uintptr_t) end;
 104   do
 105     {
 106       asm volatile ("flush %0+0" : : "r" (addr));
 107       addr += 8;
 108     }
 109   while (addr < end_addr);
 110 #elif (defined __GNUC__ || defined __clang__) && defined __hppa
 111   /* Use inline assembly.  */
 112   /* The PA-RISC 1.1 Architecture and Instruction Set Reference Manual says:
 113      "A cache line can be 16, 32, or 64 bytes in length."  */
 114   /* XXX Is this good enough, or do we need the space register business
 115      like in gcc/gcc/config/pa/pa.md and libffcall/trampoline/cache-hppa.c?  */
 116   intptr_t cache_line_size = 16;
 117   uintptr_t addr = (uintptr_t) start & ~cache_line_size;
 118   uintptr_t end_addr = (uintptr_t) end;
 119   do
 120     {
 121       asm volatile ("fdc 0(0,%0)"
 122              "\n\t" "sync"
 123              "\n\t" "fic 0(0,%0)"
 124              "\n\t" "sync" : : "r" (addr));
 125       addr += cache_line_size;
 126     }
 127   while (addr < end_addr);
 128   asm volatile ("nop"
 129          "\n\t" "nop"
 130          "\n\t" "nop"
 131          "\n\t" "nop"
 132          "\n\t" "nop"
 133          "\n\t" "nop");
 134 #elif (defined __GNUC__ || defined __clang__) && defined __ia64
 135   /* Use inline assembly.  */
 136   /* The Intel IA-64 Architecture Software Developer's Manual volume 3 says:
 137      "The line size affected is at least 32 bytes."  */
 138   intptr_t cache_line_size = 32;
 139   uintptr_t addr = (uintptr_t) start & ~cache_line_size;
 140   uintptr_t end_addr = (uintptr_t) end;
 141   do
 142     {
 143       /* Flush a cache line.  */
 144       asm volatile ("fc %0" : : "r" (addr));
 145       addr += cache_line_size;
 146     }
 147   while (addr < end_addr);
 148   /* Ensure the preceding 'fc' instructions become effective in the local
 149      processor and all remote processors.  */
 150   asm volatile ("sync.i");
 151   /* Ensure the preceding 'sync.i' instruction becomes effective in the
 152      local processor's instruction cache.  */
 153   asm volatile ("srlz.i");
 154 #elif (defined __GNUC__ || defined __clang__) && defined __m68k__ && defined __linux__
 155   /* Use inline assembly to call the 'cacheflush' system call.
 156      sys_cacheflush (addr, scope, cache, len)
 157                       d1     d2     d3    d4
 158    */
 159   register uintptr_t addr __asm__ ("%d1") = (uintptr_t) start;
 160   register uintptr_t len __asm__ ("%d4") = (uintptr_t) end - addr;
 161   __asm__ __volatile__ (
 162            "move%.l %#123,%/d0" /* __NR_cacheflush */
 163     "\n\t" "move%.l %#1,%/d2"   /* FLUSH_SCOPE_LINE */
 164     "\n\t" "move%.l %#3,%/d3"   /* FLUSH_CACHE_BOTH */
 165     "\n\t" "trap %#0"
 166     :
 167     : "d" (addr), "d" (len)
 168     : "%d0", "%d2", "%d3"
 169     );
 170 #elif (__GNUC__ + (__GNUC_MINOR__ >= 3) > 4 && !defined __clang__) \
 171       || ((__clang_major__ + (__clang_minor__ >= 4) > 3) \
 172           && (defined __aarch64__ /* arm64 */ || defined __arm__))
 173   /* GCC >= 4.3 has a GCC built-in.
 174      <https://gcc.gnu.org/onlinedocs/gcc-4.3.6/gcc/Other-Builtins.html>
 175      But it's sometimes not correctly implemented.
 176      clang >= 3.4 has it as well, at least on ARM and ARM64.  */
 177   /* On ARM, cache flushing can only be done through a system call.
 178      GCC implements it for Linux with EABI, through an "swi 0" with code
 179      0xf0002.  For other systems, it may be an "swi 0x9f0002",
 180      an "swi 0xf00000", or similar.  */
 181   /* On ARM64, cache flushing is done through special instructions,
 182      and the length of the cache lines must be determined at runtime.
 183      See gcc/libgcc/config/aarch64/sync-cache.c.  */
 184   __builtin___clear_cache (start, end);
 185 #elif HAVE___CLEAR_CACHE
 186   /* Older versions of GCC have this libgcc function, but only on some
 187      platforms.  */
 188   extern void __clear_cache (char *, char *);
 189   __clear_cache (start, end);
 190 #else
 191 # error "Don't know how to implement clear_cache on this platform."
 192 #endif
 193 }
 194
 195
 196 #ifdef __cplusplus
 197 }
 198 #endif