libc/startup/linux/aarch64/start.cpp

   1 //===-- Implementation of crt for aarch64 ---------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "config/linux/app.h"
  10 #include "src/__support/OSUtil/syscall.h"
  11 #include "src/__support/threads/thread.h"
  12 #include "src/stdlib/atexit.h"
  13 #include "src/stdlib/exit.h"
  14 #include "src/string/memory_utils/inline_memcpy.h"
  15
  16 #include <arm_acle.h>
  17
  18 #include <linux/auxvec.h>
  19 #include <linux/elf.h>
  20 #include <stdint.h>
  21 #include <sys/mman.h>
  22 #include <sys/syscall.h>
  23 #include <unistd.h>
  24
  25 extern "C" int main(int, char **, char **);
  26
  27 // Source documentation:
  28 // https://github.com/ARM-software/abi-aa/tree/main/sysvabi64
  29
  30 namespace LIBC_NAMESPACE {
  31
  32 #ifdef SYS_mmap2
  33 static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
  34 #elif SYS_mmap
  35 static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
  36 #else
  37 #error "mmap and mmap2 syscalls not available."
  38 #endif
  39
  40 AppProperties app;
  41
  42 static ThreadAttributes main_thread_attrib;
  43
  44 void init_tls(TLSDescriptor &tls_descriptor) {
  45   if (app.tls.size == 0) {
  46     tls_descriptor.size = 0;
  47     tls_descriptor.tp = 0;
  48     return;
  49   }
  50
  51   // aarch64 follows the variant 1 TLS layout:
  52   //
  53   // 1. First entry is the dynamic thread vector pointer
  54   // 2. Second entry is a 8-byte reserved word.
  55   // 3. Padding for alignment.
  56   // 4. The TLS data from the ELF image.
  57   //
  58   // The thread pointer points to the first entry.
  59
  60   const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t);
  61   uintptr_t padding = 0;
  62   const uintptr_t ALIGNMENT_MASK = app.tls.align - 1;
  63   uintptr_t diff = size_of_pointers & ALIGNMENT_MASK;
  64   if (diff != 0)
  65     padding += (ALIGNMENT_MASK - diff) + 1;
  66
  67   uintptr_t alloc_size = size_of_pointers + padding + app.tls.size;
  68
  69   // We cannot call the mmap function here as the functions set errno on
  70   // failure. Since errno is implemented via a thread local variable, we cannot
  71   // use errno before TLS is setup.
  72   long mmap_ret_val = LIBC_NAMESPACE::syscall_impl<long>(
  73       MMAP_SYSCALL_NUMBER, nullptr, alloc_size, PROT_READ | PROT_WRITE,
  74       MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
  75   // We cannot check the return value with MAP_FAILED as that is the return
  76   // of the mmap function and not the mmap syscall.
  77   if (mmap_ret_val < 0 && static_cast<uintptr_t>(mmap_ret_val) > -app.pageSize)
  78     LIBC_NAMESPACE::syscall_impl<long>(SYS_exit, 1);
  79   uintptr_t thread_ptr = uintptr_t(reinterpret_cast<uintptr_t *>(mmap_ret_val));
  80   uintptr_t tls_addr = thread_ptr + size_of_pointers + padding;
  81   LIBC_NAMESPACE::inline_memcpy(reinterpret_cast<char *>(tls_addr),
  82                                 reinterpret_cast<const char *>(app.tls.address),
  83                                 app.tls.init_size);
  84   tls_descriptor.size = alloc_size;
  85   tls_descriptor.addr = thread_ptr;
  86   tls_descriptor.tp = thread_ptr;
  87 }
  88
  89 void cleanup_tls(uintptr_t addr, uintptr_t size) {
  90   if (size == 0)
  91     return;
  92   LIBC_NAMESPACE::syscall_impl<long>(SYS_munmap, addr, size);
  93 }
  94
  95 static void set_thread_ptr(uintptr_t val) { __arm_wsr64("tpidr_el0", val); }
  96
  97 using InitCallback = void(int, char **, char **);
  98 using FiniCallback = void(void);
  99
 100 extern "C" {
 101 // These arrays are present in the .init_array and .fini_array sections.
 102 // The symbols are inserted by linker when it sees references to them.
 103 extern uintptr_t __preinit_array_start[];
 104 extern uintptr_t __preinit_array_end[];
 105 extern uintptr_t __init_array_start[];
 106 extern uintptr_t __init_array_end[];
 107 extern uintptr_t __fini_array_start[];
 108 extern uintptr_t __fini_array_end[];
 109 }
 110
 111 static void call_init_array_callbacks(int argc, char **argv, char **env) {
 112   size_t preinit_array_size = __preinit_array_end - __preinit_array_start;
 113   for (size_t i = 0; i < preinit_array_size; ++i)
 114     reinterpret_cast<InitCallback *>(__preinit_array_start[i])(argc, argv, env);
 115   size_t init_array_size = __init_array_end - __init_array_start;
 116   for (size_t i = 0; i < init_array_size; ++i)
 117     reinterpret_cast<InitCallback *>(__init_array_start[i])(argc, argv, env);
 118 }
 119
 120 static void call_fini_array_callbacks() {
 121   size_t fini_array_size = __fini_array_end - __fini_array_start;
 122   for (size_t i = fini_array_size; i > 0; --i)
 123     reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
 124 }
 125
 126 } // namespace LIBC_NAMESPACE
 127
 128 using LIBC_NAMESPACE::app;
 129
 130 // TODO: Would be nice to use the aux entry structure from elf.h when available.
 131 struct AuxEntry {
 132   uint64_t type;
 133   uint64_t value;
 134 };
 135
 136 __attribute__((noinline)) static void do_start() {
 137   auto tid = LIBC_NAMESPACE::syscall_impl<long>(SYS_gettid);
 138   if (tid <= 0)
 139     LIBC_NAMESPACE::syscall_impl<long>(SYS_exit, 1);
 140   LIBC_NAMESPACE::main_thread_attrib.tid = static_cast<int>(tid);
 141
 142   // After the argv array, is a 8-byte long NULL value before the array of env
 143   // values. The end of the env values is marked by another 8-byte long NULL
 144   // value. We step over it (the "+ 1" below) to get to the env values.
 145   uint64_t *env_ptr = app.args->argv + app.args->argc + 1;
 146   uint64_t *env_end_marker = env_ptr;
 147   app.envPtr = env_ptr;
 148   while (*env_end_marker)
 149     ++env_end_marker;
 150
 151   // Initialize the POSIX global declared in unistd.h
 152   environ = reinterpret_cast<char **>(env_ptr);
 153
 154   // After the env array, is the aux-vector. The end of the aux-vector is
 155   // denoted by an AT_NULL entry.
 156   Elf64_Phdr *programHdrTable = nullptr;
 157   uintptr_t programHdrCount;
 158   for (AuxEntry *aux_entry = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
 159        aux_entry->type != AT_NULL; ++aux_entry) {
 160     switch (aux_entry->type) {
 161     case AT_PHDR:
 162       programHdrTable = reinterpret_cast<Elf64_Phdr *>(aux_entry->value);
 163       break;
 164     case AT_PHNUM:
 165       programHdrCount = aux_entry->value;
 166       break;
 167     case AT_PAGESZ:
 168       app.pageSize = aux_entry->value;
 169       break;
 170     default:
 171       break; // TODO: Read other useful entries from the aux vector.
 172     }
 173   }
 174
 175   app.tls.size = 0;
 176   for (uintptr_t i = 0; i < programHdrCount; ++i) {
 177     Elf64_Phdr *phdr = programHdrTable + i;
 178     if (phdr->p_type != PT_TLS)
 179       continue;
 180     // TODO: p_vaddr value has to be adjusted for static-pie executables.
 181     app.tls.address = phdr->p_vaddr;
 182     app.tls.size = phdr->p_memsz;
 183     app.tls.init_size = phdr->p_filesz;
 184     app.tls.align = phdr->p_align;
 185   }
 186
 187   LIBC_NAMESPACE::TLSDescriptor tls;
 188   LIBC_NAMESPACE::init_tls(tls);
 189   if (tls.size != 0)
 190     LIBC_NAMESPACE::set_thread_ptr(tls.tp);
 191
 192   LIBC_NAMESPACE::self.attrib = &LIBC_NAMESPACE::main_thread_attrib;
 193   LIBC_NAMESPACE::main_thread_attrib.atexit_callback_mgr =
 194       LIBC_NAMESPACE::internal::get_thread_atexit_callback_mgr();
 195
 196   // We want the fini array callbacks to be run after other atexit
 197   // callbacks are run. So, we register them before running the init
 198   // array callbacks as they can potentially register their own atexit
 199   // callbacks.
 200   LIBC_NAMESPACE::atexit(&LIBC_NAMESPACE::call_fini_array_callbacks);
 201
 202   LIBC_NAMESPACE::call_init_array_callbacks(
 203       static_cast<int>(app.args->argc),
 204       reinterpret_cast<char **>(app.args->argv),
 205       reinterpret_cast<char **>(env_ptr));
 206
 207   int retval = main(static_cast<int>(app.args->argc),
 208                     reinterpret_cast<char **>(app.args->argv),
 209                     reinterpret_cast<char **>(env_ptr));
 210
 211   // TODO: TLS cleanup should be done after all other atexit callbacks
 212   // are run. So, register a cleanup callback for it with atexit before
 213   // everything else.
 214   LIBC_NAMESPACE::cleanup_tls(tls.addr, tls.size);
 215   LIBC_NAMESPACE::exit(retval);
 216 }
 217
 218 extern "C" void _start() {
 219   // Skip the Frame Pointer and the Link Register
 220   // https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst
 221   // Section 6.2.3. Note that this only works if the current function
 222   // is not using any callee-saved registers (x19 to x28). If the
 223   // function uses such registers, then their value is pushed on to the
 224   // stack before the frame pointer an link register values. That breaks
 225   // the assumption that stepping over the frame pointer and link register
 226   // will take us to the previous stack pointer. That is the reason why the
 227   // actual business logic of the startup code is pushed into a non-inline
 228   // function do_start so that this function is free of any stack usage.
 229   app.args = reinterpret_cast<LIBC_NAMESPACE::Args *>(
 230       reinterpret_cast<uintptr_t *>(__builtin_frame_address(0)) + 2);
 231   do_start();
 232 }