?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[clang][modules] Don't prevent translation of FW_Private includes when explicitly...
[llvm-project.git] / compiler-rt / lib / tsan / rtl / tsan_platform.h
blobcfbb57d1d8d8d9beef05806892d96555f228e1d4
1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 //
11 // Platform-specific code.
12 //===----------------------------------------------------------------------===//
13
14 #ifndef TSAN_PLATFORM_H
15 #define TSAN_PLATFORM_H
16
17 #if !defined(__LP64__) && !defined(_WIN64)
18 # error "Only 64-bit is supported"
19 #endif
20
21 #include "sanitizer_common/sanitizer_common.h"
22 #include "tsan_defs.h"
23
24 namespace __tsan {
25
26 enum {
27 // App memory is not mapped onto shadow memory range.
28 kBrokenMapping = 1 << 0,
29 // Mapping app memory and back does not produce the same address,
30 // this can lead to wrong addresses in reports and potentially
31 // other bad consequences.
32 kBrokenReverseMapping = 1 << 1,
33 // Mapping is non-linear for linear user range.
34 // This is bad and can lead to unpredictable memory corruptions, etc
35 // because range access functions assume linearity.
36 kBrokenLinearity = 1 << 2,
37 // Meta for an app region overlaps with the meta of another app region.
38 // This is determined by recomputing the individual meta regions for
39 // each app region.
40 //
41 // N.B. There is no "kBrokenReverseMetaMapping" constant because there
42 // is no MetaToMem function. However, note that (!kBrokenLinearity
43 // && !kBrokenAliasedMetas) implies that MemToMeta is invertible.
44 kBrokenAliasedMetas = 1 << 3,
45 };
46
47 /*
48 C/C++ on linux/x86_64 and freebsd/x86_64
49 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
50 0040 0000 0000 - 0100 0000 0000: -
51 0100 0000 0000 - 1000 0000 0000: shadow
52 1000 0000 0000 - 3000 0000 0000: -
53 3000 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
54 3400 0000 0000 - 5500 0000 0000: -
55 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
56 5680 0000 0000 - 7d00 0000 0000: -
57 7b00 0000 0000 - 7c00 0000 0000: heap
58 7c00 0000 0000 - 7e80 0000 0000: -
59 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
60
61 C/C++ on netbsd/amd64 can reuse the same mapping:
62 * The address space starts from 0x1000 (option with 0x0) and ends with
63 0x7f7ffffff000.
64 * LoAppMem-kHeapMemEnd can be reused as it is.
65 * No VDSO support.
66 * No MidAppMem region.
67 * No additional HeapMem region.
68 * HiAppMem contains the stack, loader, shared libraries and heap.
69 * Stack on NetBSD/amd64 has prereserved 128MB.
70 * Heap grows downwards (top-down).
71 * ASLR must be disabled per-process or globally.
72 */
73 struct Mapping48AddressSpace {
74 static const uptr kMetaShadowBeg = 0x300000000000ull;
75 static const uptr kMetaShadowEnd = 0x340000000000ull;
76 static const uptr kShadowBeg = 0x010000000000ull;
77 static const uptr kShadowEnd = 0x100000000000ull;
78 static const uptr kHeapMemBeg = 0x7b0000000000ull;
79 static const uptr kHeapMemEnd = 0x7c0000000000ull;
80 static const uptr kLoAppMemBeg = 0x000000001000ull;
81 static const uptr kLoAppMemEnd = 0x008000000000ull;
82 static const uptr kMidAppMemBeg = 0x550000000000ull;
83 static const uptr kMidAppMemEnd = 0x568000000000ull;
84 static const uptr kHiAppMemBeg = 0x7e8000000000ull;
85 static const uptr kHiAppMemEnd = 0x800000000000ull;
86 static const uptr kShadowMsk = 0x780000000000ull;
87 static const uptr kShadowXor = 0x040000000000ull;
88 static const uptr kShadowAdd = 0x000000000000ull;
89 static const uptr kVdsoBeg = 0xf000000000000000ull;
90 };
91
92 /*
93 C/C++ on linux/mips64 (40-bit VMA)
94 0000 0000 00 - 0100 0000 00: - (4 GB)
95 0100 0000 00 - 0200 0000 00: main binary (4 GB)
96 0200 0000 00 - 1200 0000 00: - (64 GB)
97 1200 0000 00 - 2200 0000 00: shadow (64 GB)
98 2200 0000 00 - 4000 0000 00: - (120 GB)
99 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)
100 5000 0000 00 - aa00 0000 00: - (360 GB)
101 aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)
102 ab00 0000 00 - fe00 0000 00: - (332 GB)
103 fe00 0000 00 - ff00 0000 00: heap (4 GB)
104 ff00 0000 00 - ff80 0000 00: - (2 GB)
105 ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
106 */
107 struct MappingMips64_40 {
108 static const uptr kMetaShadowBeg = 0x4000000000ull;
109 static const uptr kMetaShadowEnd = 0x5000000000ull;
110 static const uptr kShadowBeg = 0x1200000000ull;
111 static const uptr kShadowEnd = 0x2200000000ull;
112 static const uptr kHeapMemBeg = 0xfe00000000ull;
113 static const uptr kHeapMemEnd = 0xff00000000ull;
114 static const uptr kLoAppMemBeg = 0x0100000000ull;
115 static const uptr kLoAppMemEnd = 0x0200000000ull;
116 static const uptr kMidAppMemBeg = 0xaa00000000ull;
117 static const uptr kMidAppMemEnd = 0xab00000000ull;
118 static const uptr kHiAppMemBeg = 0xff80000000ull;
119 static const uptr kHiAppMemEnd = 0xffffffffffull;
120 static const uptr kShadowMsk = 0xf800000000ull;
121 static const uptr kShadowXor = 0x0800000000ull;
122 static const uptr kShadowAdd = 0x0000000000ull;
123 static const uptr kVdsoBeg = 0xfffff00000ull;
124 };
125
126 /*
127 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
128 0000 0000 00 - 0100 0000 00: - (4 GB)
129 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
130 0200 0000 00 - 0300 0000 00: heap (4 GB)
131 0300 0000 00 - 0400 0000 00: - (4 GB)
132 0400 0000 00 - 0800 0000 00: shadow memory (16 GB)
133 0800 0000 00 - 0d00 0000 00: - (20 GB)
134 0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
135 0e00 0000 00 - 1000 0000 00: -
136 */
137 struct MappingAppleAarch64 {
138 static const uptr kLoAppMemBeg = 0x0100000000ull;
139 static const uptr kLoAppMemEnd = 0x0200000000ull;
140 static const uptr kHeapMemBeg = 0x0200000000ull;
141 static const uptr kHeapMemEnd = 0x0300000000ull;
142 static const uptr kShadowBeg = 0x0400000000ull;
143 static const uptr kShadowEnd = 0x0800000000ull;
144 static const uptr kMetaShadowBeg = 0x0d00000000ull;
145 static const uptr kMetaShadowEnd = 0x0e00000000ull;
146 static const uptr kHiAppMemBeg = 0x0fc0000000ull;
147 static const uptr kHiAppMemEnd = 0x0fc0000000ull;
148 static const uptr kShadowMsk = 0x0ull;
149 static const uptr kShadowXor = 0x0ull;
150 static const uptr kShadowAdd = 0x0200000000ull;
151 static const uptr kVdsoBeg = 0x7000000000000000ull;
152 static const uptr kMidAppMemBeg = 0;
153 static const uptr kMidAppMemEnd = 0;
154 };
155
156 /*
157 C/C++ on linux/aarch64 (39-bit VMA)
158 0000 0010 00 - 0500 0000 00: main binary (20 GB)
159 0100 0000 00 - 2000 0000 00: -
160 2000 0000 00 - 4000 0000 00: shadow memory (128 GB)
161 4000 0000 00 - 4800 0000 00: metainfo (32 GB)
162 4800 0000 00 - 5500 0000 00: -
163 5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB)
164 5600 0000 00 - 7c00 0000 00: -
165 7a00 0000 00 - 7d00 0000 00: heap (12 GB)
166 7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB)
167 */
168 struct MappingAarch64_39 {
169 static const uptr kLoAppMemBeg = 0x0000001000ull;
170 static const uptr kLoAppMemEnd = 0x0500000000ull;
171 static const uptr kShadowBeg = 0x2000000000ull;
172 static const uptr kShadowEnd = 0x4000000000ull;
173 static const uptr kMetaShadowBeg = 0x4000000000ull;
174 static const uptr kMetaShadowEnd = 0x4800000000ull;
175 static const uptr kMidAppMemBeg = 0x5500000000ull;
176 static const uptr kMidAppMemEnd = 0x5a00000000ull;
177 static const uptr kHeapMemBeg = 0x7a00000000ull;
178 static const uptr kHeapMemEnd = 0x7d00000000ull;
179 static const uptr kHiAppMemBeg = 0x7d00000000ull;
180 static const uptr kHiAppMemEnd = 0x7fffffffffull;
181 static const uptr kShadowMsk = 0x7000000000ull;
182 static const uptr kShadowXor = 0x1000000000ull;
183 static const uptr kShadowAdd = 0x0000000000ull;
184 static const uptr kVdsoBeg = 0x7f00000000ull;
185 };
186
187 /*
188 C/C++ on linux/aarch64 (42-bit VMA)
189 00000 0010 00 - 02000 0000 00: main binary (128 GB)
190 02000 0000 00 - 08000 0000 00: -
191 10000 0000 00 - 20000 0000 00: shadow memory (1024 GB)
192 20000 0000 00 - 24000 0000 00: metainfo (256 GB)
193 24000 0000 00 - 2aa00 0000 00: -
194 2aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB)
195 2c000 0000 00 - 3c000 0000 00: -
196 3c000 0000 00 - 3f000 0000 00: heap (192 GB)
197 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB)
198 */
199 struct MappingAarch64_42 {
200 static const uptr kLoAppMemBeg = 0x00000001000ull;
201 static const uptr kLoAppMemEnd = 0x02000000000ull;
202 static const uptr kShadowBeg = 0x10000000000ull;
203 static const uptr kShadowEnd = 0x20000000000ull;
204 static const uptr kMetaShadowBeg = 0x20000000000ull;
205 static const uptr kMetaShadowEnd = 0x24000000000ull;
206 static const uptr kMidAppMemBeg = 0x2aa00000000ull;
207 static const uptr kMidAppMemEnd = 0x2c000000000ull;
208 static const uptr kHeapMemBeg = 0x3c000000000ull;
209 static const uptr kHeapMemEnd = 0x3f000000000ull;
210 static const uptr kHiAppMemBeg = 0x3f000000000ull;
211 static const uptr kHiAppMemEnd = 0x3ffffffffffull;
212 static const uptr kShadowMsk = 0x38000000000ull;
213 static const uptr kShadowXor = 0x08000000000ull;
214 static const uptr kShadowAdd = 0x00000000000ull;
215 static const uptr kVdsoBeg = 0x37f00000000ull;
216 };
217
218 /*
219 C/C++ on linux/aarch64 (48-bit VMA)
220 0000 0000 1000 - 0a00 0000 0000: main binary (10240 GB)
221 0a00 0000 1000 - 1554 0000 0000: -
222 1554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB)
223 5400 0000 1000 - 8000 0000 0000: -
224 8000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB)
225 a000 0000 1000 - aaaa 0000 0000: -
226 aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB)
227 ac00 0000 1000 - fc00 0000 0000: -
228 fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB)
229
230 N.B. the shadow memory region has a strange start address, because it
231 contains the shadows for the mid, high and low app regions (in this
232 unusual order).
233 */
234 struct MappingAarch64_48 {
235 static const uptr kLoAppMemBeg = 0x0000000001000ull;
236 static const uptr kLoAppMemEnd = 0x00a0000000000ull;
237 static const uptr kShadowBeg = 0x0155400000000ull;
238 static const uptr kShadowEnd = 0x0540000000000ull;
239 static const uptr kMetaShadowBeg = 0x0800000000000ull;
240 static const uptr kMetaShadowEnd = 0x0a00000000000ull;
241 static const uptr kMidAppMemBeg = 0x0aaaa00000000ull;
242 static const uptr kMidAppMemEnd = 0x0ac0000000000ull;
243 static const uptr kHiAppMemBeg = 0x0fc0000000000ull;
244 static const uptr kHiAppMemEnd = 0x1000000000000ull;
245 static const uptr kHeapMemBeg = 0x0fc0000000000ull;
246 static const uptr kHeapMemEnd = 0x0fc0000000000ull;
247 static const uptr kShadowMsk = 0x0c00000000000ull;
248 static const uptr kShadowXor = 0x0200000000000ull;
249 static const uptr kShadowAdd = 0x0000000000000ull;
250 static const uptr kVdsoBeg = 0xffff000000000ull;
251 };
252
253 /* C/C++ on linux/loongarch64 (47-bit VMA)
254 0000 0000 4000 - 0080 0000 0000: main binary
255 0080 0000 0000 - 0100 0000 0000: -
256 0100 0000 0000 - 1000 0000 0000: shadow memory
257 1000 0000 0000 - 3000 0000 0000: -
258 3000 0000 0000 - 3400 0000 0000: metainfo
259 3400 0000 0000 - 5555 0000 0000: -
260 5555 0000 0000 - 5556 0000 0000: main binary (PIE)
261 5556 0000 0000 - 7ffe 0000 0000: -
262 7ffe 0000 0000 - 7fff 0000 0000: heap
263 7fff 0000 0000 - 7fff 8000 0000: -
264 7fff 8000 0000 - 8000 0000 0000: modules and main thread stack
265 */
266 struct MappingLoongArch64_47 {
267 static const uptr kMetaShadowBeg = 0x300000000000ull;
268 static const uptr kMetaShadowEnd = 0x340000000000ull;
269 static const uptr kShadowBeg = 0x010000000000ull;
270 static const uptr kShadowEnd = 0x100000000000ull;
271 static const uptr kHeapMemBeg = 0x7ffe00000000ull;
272 static const uptr kHeapMemEnd = 0x7fff00000000ull;
273 static const uptr kLoAppMemBeg = 0x000000004000ull;
274 static const uptr kLoAppMemEnd = 0x008000000000ull;
275 static const uptr kMidAppMemBeg = 0x555500000000ull;
276 static const uptr kMidAppMemEnd = 0x555600000000ull;
277 static const uptr kHiAppMemBeg = 0x7fff80000000ull;
278 static const uptr kHiAppMemEnd = 0x800000000000ull;
279 static const uptr kShadowMsk = 0x780000000000ull;
280 static const uptr kShadowXor = 0x040000000000ull;
281 static const uptr kShadowAdd = 0x000000000000ull;
282 static const uptr kVdsoBeg = 0x7fffffffc000ull;
283 };
284
285 /*
286 C/C++ on linux/powerpc64 (44-bit VMA)
287 0000 0000 0100 - 0001 0000 0000: main binary
288 0001 0000 0000 - 0001 0000 0000: -
289 0001 0000 0000 - 0b00 0000 0000: shadow
290 0b00 0000 0000 - 0b00 0000 0000: -
291 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
292 0d00 0000 0000 - 0f00 0000 0000: -
293 0f00 0000 0000 - 0f50 0000 0000: heap
294 0f50 0000 0000 - 0f60 0000 0000: -
295 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
296 */
297 struct MappingPPC64_44 {
298 static const uptr kBroken = kBrokenMapping | kBrokenReverseMapping |
299 kBrokenLinearity | kBrokenAliasedMetas;
300 static const uptr kMetaShadowBeg = 0x0b0000000000ull;
301 static const uptr kMetaShadowEnd = 0x0d0000000000ull;
302 static const uptr kShadowBeg = 0x000100000000ull;
303 static const uptr kShadowEnd = 0x0b0000000000ull;
304 static const uptr kLoAppMemBeg = 0x000000000100ull;
305 static const uptr kLoAppMemEnd = 0x000100000000ull;
306 static const uptr kHeapMemBeg = 0x0f0000000000ull;
307 static const uptr kHeapMemEnd = 0x0f5000000000ull;
308 static const uptr kHiAppMemBeg = 0x0f6000000000ull;
309 static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits
310 static const uptr kShadowMsk = 0x0f0000000000ull;
311 static const uptr kShadowXor = 0x002100000000ull;
312 static const uptr kShadowAdd = 0x000000000000ull;
313 static const uptr kVdsoBeg = 0x3c0000000000000ull;
314 static const uptr kMidAppMemBeg = 0;
315 static const uptr kMidAppMemEnd = 0;
316 };
317
318 /*
319 C/C++ on linux/powerpc64 (46-bit VMA)
320 0000 0000 1000 - 0100 0000 0000: main binary
321 0100 0000 0000 - 0200 0000 0000: -
322 0100 0000 0000 - 0800 0000 0000: shadow
323 0800 0000 0000 - 1000 0000 0000: -
324 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
325 1200 0000 0000 - 3d00 0000 0000: -
326 3d00 0000 0000 - 3e00 0000 0000: heap
327 3e00 0000 0000 - 3e80 0000 0000: -
328 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
329 */
330 struct MappingPPC64_46 {
331 static const uptr kMetaShadowBeg = 0x100000000000ull;
332 static const uptr kMetaShadowEnd = 0x120000000000ull;
333 static const uptr kShadowBeg = 0x010000000000ull;
334 static const uptr kShadowEnd = 0x080000000000ull;
335 static const uptr kHeapMemBeg = 0x3d0000000000ull;
336 static const uptr kHeapMemEnd = 0x3e0000000000ull;
337 static const uptr kLoAppMemBeg = 0x000000001000ull;
338 static const uptr kLoAppMemEnd = 0x010000000000ull;
339 static const uptr kHiAppMemBeg = 0x3e8000000000ull;
340 static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits
341 static const uptr kShadowMsk = 0x3c0000000000ull;
342 static const uptr kShadowXor = 0x020000000000ull;
343 static const uptr kShadowAdd = 0x000000000000ull;
344 static const uptr kVdsoBeg = 0x7800000000000000ull;
345 static const uptr kMidAppMemBeg = 0;
346 static const uptr kMidAppMemEnd = 0;
347 };
348
349 /*
350 C/C++ on linux/powerpc64 (47-bit VMA)
351 0000 0000 1000 - 0100 0000 0000: main binary
352 0100 0000 0000 - 0200 0000 0000: -
353 0100 0000 0000 - 0800 0000 0000: shadow
354 0800 0000 0000 - 1000 0000 0000: -
355 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
356 1200 0000 0000 - 7d00 0000 0000: -
357 7d00 0000 0000 - 7e00 0000 0000: heap
358 7e00 0000 0000 - 7e80 0000 0000: -
359 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
360 */
361 struct MappingPPC64_47 {
362 static const uptr kMetaShadowBeg = 0x100000000000ull;
363 static const uptr kMetaShadowEnd = 0x120000000000ull;
364 static const uptr kShadowBeg = 0x010000000000ull;
365 static const uptr kShadowEnd = 0x080000000000ull;
366 static const uptr kHeapMemBeg = 0x7d0000000000ull;
367 static const uptr kHeapMemEnd = 0x7e0000000000ull;
368 static const uptr kLoAppMemBeg = 0x000000001000ull;
369 static const uptr kLoAppMemEnd = 0x010000000000ull;
370 static const uptr kHiAppMemBeg = 0x7e8000000000ull;
371 static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits
372 static const uptr kShadowMsk = 0x7c0000000000ull;
373 static const uptr kShadowXor = 0x020000000000ull;
374 static const uptr kShadowAdd = 0x000000000000ull;
375 static const uptr kVdsoBeg = 0x7800000000000000ull;
376 static const uptr kMidAppMemBeg = 0;
377 static const uptr kMidAppMemEnd = 0;
378 };
379
380 /*
381 C/C++ on linux/riscv64 (39-bit VMA)
382 0000 0010 00 - 0200 0000 00: main binary ( 8 GB)
383 0200 0000 00 - 1000 0000 00: -
384 1000 0000 00 - 4000 0000 00: shadow memory (64 GB)
385 4000 0000 00 - 4800 0000 00: metainfo (16 GB)
386 4800 0000 00 - 5500 0000 00: -
387 5500 0000 00 - 5a00 0000 00: main binary (PIE) (~8 GB)
388 5600 0000 00 - 7c00 0000 00: -
389 7d00 0000 00 - 7fff ffff ff: libraries and main thread stack ( 8 GB)
390
391 mmap by default allocates from top downwards
392 VDSO sits below loader and above dynamic libraries, within HiApp region.
393 Heap starts after program region whose position depends on pie or non-pie.
394 Disable tracking them since their locations are not fixed.
395 */
396 struct MappingRiscv64_39 {
397 static const uptr kLoAppMemBeg = 0x0000001000ull;
398 static const uptr kLoAppMemEnd = 0x0200000000ull;
399 static const uptr kShadowBeg = 0x1000000000ull;
400 static const uptr kShadowEnd = 0x2000000000ull;
401 static const uptr kMetaShadowBeg = 0x2000000000ull;
402 static const uptr kMetaShadowEnd = 0x2400000000ull;
403 static const uptr kMidAppMemBeg = 0x2aaaaaa000ull;
404 static const uptr kMidAppMemEnd = 0x2c00000000ull;
405 static const uptr kHeapMemBeg = 0x2c00000000ull;
406 static const uptr kHeapMemEnd = 0x2c00000000ull;
407 static const uptr kHiAppMemBeg = 0x3c00000000ull;
408 static const uptr kHiAppMemEnd = 0x3fffffffffull;
409 static const uptr kShadowMsk = 0x3800000000ull;
410 static const uptr kShadowXor = 0x0800000000ull;
411 static const uptr kShadowAdd = 0x0000000000ull;
412 static const uptr kVdsoBeg = 0x4000000000ull;
413 };
414
415 /*
416 C/C++ on linux/riscv64 (48-bit VMA)
417 0000 0000 1000 - 0500 0000 0000: main binary ( 5 TB)
418 0500 0000 0000 - 2000 0000 0000: -
419 2000 0000 0000 - 4000 0000 0000: shadow memory (32 TB)
420 4000 0000 0000 - 4800 0000 0000: metainfo ( 8 TB)
421 4800 0000 0000 - 5555 5555 5000: -
422 5555 5555 5000 - 5a00 0000 0000: main binary (PIE) (~5 TB)
423 5a00 0000 0000 - 7a00 0000 0000: -
424 7a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack ( 5 TB)
425 */
426 struct MappingRiscv64_48 {
427 static const uptr kLoAppMemBeg = 0x000000001000ull;
428 static const uptr kLoAppMemEnd = 0x050000000000ull;
429 static const uptr kShadowBeg = 0x200000000000ull;
430 static const uptr kShadowEnd = 0x400000000000ull;
431 static const uptr kMetaShadowBeg = 0x400000000000ull;
432 static const uptr kMetaShadowEnd = 0x480000000000ull;
433 static const uptr kMidAppMemBeg = 0x555555555000ull;
434 static const uptr kMidAppMemEnd = 0x5a0000000000ull;
435 static const uptr kHeapMemBeg = 0x5a0000000000ull;
436 static const uptr kHeapMemEnd = 0x5a0000000000ull;
437 static const uptr kHiAppMemBeg = 0x7a0000000000ull;
438 static const uptr kHiAppMemEnd = 0x7fffffffffffull;
439 static const uptr kShadowMsk = 0x700000000000ull;
440 static const uptr kShadowXor = 0x100000000000ull;
441 static const uptr kShadowAdd = 0x000000000000ull;
442 static const uptr kVdsoBeg = 0x800000000000ull;
443 };
444
445 /*
446 C/C++ on linux/s390x
447 While the kernel provides a 64-bit address space, we have to restrict ourselves
448 to 48 bits due to how e.g. SyncVar::GetId() works.
449 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
450 0e00 0000 0000 - 2000 0000 0000: -
451 2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
452 4000 0000 0000 - 9000 0000 0000: -
453 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
454 9800 0000 0000 - be00 0000 0000: -
455 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
456 */
457 struct MappingS390x {
458 static const uptr kMetaShadowBeg = 0x900000000000ull;
459 static const uptr kMetaShadowEnd = 0x980000000000ull;
460 static const uptr kShadowBeg = 0x200000000000ull;
461 static const uptr kShadowEnd = 0x400000000000ull;
462 static const uptr kHeapMemBeg = 0xbe0000000000ull;
463 static const uptr kHeapMemEnd = 0xc00000000000ull;
464 static const uptr kLoAppMemBeg = 0x000000001000ull;
465 static const uptr kLoAppMemEnd = 0x0e0000000000ull;
466 static const uptr kHiAppMemBeg = 0xc00000004000ull;
467 static const uptr kHiAppMemEnd = 0xc00000004000ull;
468 static const uptr kShadowMsk = 0xb00000000000ull;
469 static const uptr kShadowXor = 0x100000000000ull;
470 static const uptr kShadowAdd = 0x000000000000ull;
471 static const uptr kVdsoBeg = 0xfffffffff000ull;
472 static const uptr kMidAppMemBeg = 0;
473 static const uptr kMidAppMemEnd = 0;
474 };
475
476 /* Go on linux, darwin and freebsd on x86_64
477 0000 0000 1000 - 0000 1000 0000: executable
478 0000 1000 0000 - 00c0 0000 0000: -
479 00c0 0000 0000 - 00e0 0000 0000: heap
480 00e0 0000 0000 - 2000 0000 0000: -
481 2000 0000 0000 - 21c0 0000 0000: shadow
482 21c0 0000 0000 - 3000 0000 0000: -
483 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
484 4000 0000 0000 - 8000 0000 0000: -
485 */
486
487 struct MappingGo48 {
488 static const uptr kMetaShadowBeg = 0x300000000000ull;
489 static const uptr kMetaShadowEnd = 0x400000000000ull;
490 static const uptr kShadowBeg = 0x200000000000ull;
491 static const uptr kShadowEnd = 0x21c000000000ull;
492 static const uptr kLoAppMemBeg = 0x000000001000ull;
493 static const uptr kLoAppMemEnd = 0x00e000000000ull;
494 static const uptr kMidAppMemBeg = 0;
495 static const uptr kMidAppMemEnd = 0;
496 static const uptr kHiAppMemBeg = 0;
497 static const uptr kHiAppMemEnd = 0;
498 static const uptr kHeapMemBeg = 0;
499 static const uptr kHeapMemEnd = 0;
500 static const uptr kVdsoBeg = 0;
501 static const uptr kShadowMsk = 0;
502 static const uptr kShadowXor = 0;
503 static const uptr kShadowAdd = 0x200000000000ull;
504 };
505
506 /* Go on windows
507 0000 0000 1000 - 0000 1000 0000: executable
508 0000 1000 0000 - 00f8 0000 0000: -
509 00c0 0000 0000 - 00e0 0000 0000: heap
510 00e0 0000 0000 - 0100 0000 0000: -
511 0100 0000 0000 - 0300 0000 0000: shadow
512 0300 0000 0000 - 0700 0000 0000: -
513 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
514 07d0 0000 0000 - 8000 0000 0000: -
515 PIE binaries currently not supported, but it should be theoretically possible.
516 */
517
518 struct MappingGoWindows {
519 static const uptr kMetaShadowBeg = 0x070000000000ull;
520 static const uptr kMetaShadowEnd = 0x077000000000ull;
521 static const uptr kShadowBeg = 0x010000000000ull;
522 static const uptr kShadowEnd = 0x030000000000ull;
523 static const uptr kLoAppMemBeg = 0x000000001000ull;
524 static const uptr kLoAppMemEnd = 0x00e000000000ull;
525 static const uptr kMidAppMemBeg = 0;
526 static const uptr kMidAppMemEnd = 0;
527 static const uptr kHiAppMemBeg = 0;
528 static const uptr kHiAppMemEnd = 0;
529 static const uptr kHeapMemBeg = 0;
530 static const uptr kHeapMemEnd = 0;
531 static const uptr kVdsoBeg = 0;
532 static const uptr kShadowMsk = 0;
533 static const uptr kShadowXor = 0;
534 static const uptr kShadowAdd = 0x010000000000ull;
535 };
536
537 /* Go on linux/powerpc64 (46-bit VMA)
538 0000 0000 1000 - 0000 1000 0000: executable
539 0000 1000 0000 - 00c0 0000 0000: -
540 00c0 0000 0000 - 00e0 0000 0000: heap
541 00e0 0000 0000 - 2000 0000 0000: -
542 2000 0000 0000 - 21c0 0000 0000: shadow
543 21c0 0000 0000 - 2400 0000 0000: -
544 2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
545 2470 0000 0000 - 4000 0000 0000: -
546 */
547
548 struct MappingGoPPC64_46 {
549 static const uptr kMetaShadowBeg = 0x240000000000ull;
550 static const uptr kMetaShadowEnd = 0x247000000000ull;
551 static const uptr kShadowBeg = 0x200000000000ull;
552 static const uptr kShadowEnd = 0x21c000000000ull;
553 static const uptr kLoAppMemBeg = 0x000000001000ull;
554 static const uptr kLoAppMemEnd = 0x00e000000000ull;
555 static const uptr kMidAppMemBeg = 0;
556 static const uptr kMidAppMemEnd = 0;
557 static const uptr kHiAppMemBeg = 0;
558 static const uptr kHiAppMemEnd = 0;
559 static const uptr kHeapMemBeg = 0;
560 static const uptr kHeapMemEnd = 0;
561 static const uptr kVdsoBeg = 0;
562 static const uptr kShadowMsk = 0;
563 static const uptr kShadowXor = 0;
564 static const uptr kShadowAdd = 0x200000000000ull;
565 };
566
567 /* Go on linux/powerpc64 (47-bit VMA)
568 0000 0000 1000 - 0000 1000 0000: executable
569 0000 1000 0000 - 00c0 0000 0000: -
570 00c0 0000 0000 - 00e0 0000 0000: heap
571 00e0 0000 0000 - 2000 0000 0000: -
572 2000 0000 0000 - 2800 0000 0000: shadow
573 2800 0000 0000 - 3000 0000 0000: -
574 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
575 3200 0000 0000 - 8000 0000 0000: -
576 */
577
578 struct MappingGoPPC64_47 {
579 static const uptr kMetaShadowBeg = 0x300000000000ull;
580 static const uptr kMetaShadowEnd = 0x320000000000ull;
581 static const uptr kShadowBeg = 0x200000000000ull;
582 static const uptr kShadowEnd = 0x280000000000ull;
583 static const uptr kLoAppMemBeg = 0x000000001000ull;
584 static const uptr kLoAppMemEnd = 0x00e000000000ull;
585 static const uptr kMidAppMemBeg = 0;
586 static const uptr kMidAppMemEnd = 0;
587 static const uptr kHiAppMemBeg = 0;
588 static const uptr kHiAppMemEnd = 0;
589 static const uptr kHeapMemBeg = 0;
590 static const uptr kHeapMemEnd = 0;
591 static const uptr kVdsoBeg = 0;
592 static const uptr kShadowMsk = 0;
593 static const uptr kShadowXor = 0;
594 static const uptr kShadowAdd = 0x200000000000ull;
595 };
596
597 /* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
598 0000 0000 1000 - 0000 1000 0000: executable
599 0000 1000 0000 - 00c0 0000 0000: -
600 00c0 0000 0000 - 00e0 0000 0000: heap
601 00e0 0000 0000 - 2000 0000 0000: -
602 2000 0000 0000 - 2800 0000 0000: shadow
603 2800 0000 0000 - 3000 0000 0000: -
604 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
605 3200 0000 0000 - 8000 0000 0000: -
606 */
607 struct MappingGoAarch64 {
608 static const uptr kMetaShadowBeg = 0x300000000000ull;
609 static const uptr kMetaShadowEnd = 0x320000000000ull;
610 static const uptr kShadowBeg = 0x200000000000ull;
611 static const uptr kShadowEnd = 0x280000000000ull;
612 static const uptr kLoAppMemBeg = 0x000000001000ull;
613 static const uptr kLoAppMemEnd = 0x00e000000000ull;
614 static const uptr kMidAppMemBeg = 0;
615 static const uptr kMidAppMemEnd = 0;
616 static const uptr kHiAppMemBeg = 0;
617 static const uptr kHiAppMemEnd = 0;
618 static const uptr kHeapMemBeg = 0;
619 static const uptr kHeapMemEnd = 0;
620 static const uptr kVdsoBeg = 0;
621 static const uptr kShadowMsk = 0;
622 static const uptr kShadowXor = 0;
623 static const uptr kShadowAdd = 0x200000000000ull;
624 };
625
626 /*
627 Go on linux/mips64 (47-bit VMA)
628 0000 0000 1000 - 0000 1000 0000: executable
629 0000 1000 0000 - 00c0 0000 0000: -
630 00c0 0000 0000 - 00e0 0000 0000: heap
631 00e0 0000 0000 - 2000 0000 0000: -
632 2000 0000 0000 - 2800 0000 0000: shadow
633 2800 0000 0000 - 3000 0000 0000: -
634 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
635 3200 0000 0000 - 8000 0000 0000: -
636 */
637 struct MappingGoMips64_47 {
638 static const uptr kMetaShadowBeg = 0x300000000000ull;
639 static const uptr kMetaShadowEnd = 0x320000000000ull;
640 static const uptr kShadowBeg = 0x200000000000ull;
641 static const uptr kShadowEnd = 0x280000000000ull;
642 static const uptr kLoAppMemBeg = 0x000000001000ull;
643 static const uptr kLoAppMemEnd = 0x00e000000000ull;
644 static const uptr kMidAppMemBeg = 0;
645 static const uptr kMidAppMemEnd = 0;
646 static const uptr kHiAppMemBeg = 0;
647 static const uptr kHiAppMemEnd = 0;
648 static const uptr kHeapMemBeg = 0;
649 static const uptr kHeapMemEnd = 0;
650 static const uptr kVdsoBeg = 0;
651 static const uptr kShadowMsk = 0;
652 static const uptr kShadowXor = 0;
653 static const uptr kShadowAdd = 0x200000000000ull;
654 };
655
656 /*
657 Go on linux/s390x
658 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
659 1000 0000 0000 - 4000 0000 0000: -
660 4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
661 6000 0000 0000 - 9000 0000 0000: -
662 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
663 */
664 struct MappingGoS390x {
665 static const uptr kMetaShadowBeg = 0x900000000000ull;
666 static const uptr kMetaShadowEnd = 0x980000000000ull;
667 static const uptr kShadowBeg = 0x400000000000ull;
668 static const uptr kShadowEnd = 0x600000000000ull;
669 static const uptr kLoAppMemBeg = 0x000000001000ull;
670 static const uptr kLoAppMemEnd = 0x100000000000ull;
671 static const uptr kMidAppMemBeg = 0;
672 static const uptr kMidAppMemEnd = 0;
673 static const uptr kHiAppMemBeg = 0;
674 static const uptr kHiAppMemEnd = 0;
675 static const uptr kHeapMemBeg = 0;
676 static const uptr kHeapMemEnd = 0;
677 static const uptr kVdsoBeg = 0;
678 static const uptr kShadowMsk = 0;
679 static const uptr kShadowXor = 0;
680 static const uptr kShadowAdd = 0x400000000000ull;
681 };
682
683 extern uptr vmaSize;
684
685 template <typename Func, typename Arg>
686 ALWAYS_INLINE auto SelectMapping(Arg arg) {
687 #if SANITIZER_GO
688 # if defined(__powerpc64__)
689 switch (vmaSize) {
690 case 46:
691 return Func::template Apply<MappingGoPPC64_46>(arg);
692 case 47:
693 return Func::template Apply<MappingGoPPC64_47>(arg);
694 }
695 # elif defined(__mips64)
696 return Func::template Apply<MappingGoMips64_47>(arg);
697 # elif defined(__s390x__)
698 return Func::template Apply<MappingGoS390x>(arg);
699 # elif defined(__aarch64__)
700 return Func::template Apply<MappingGoAarch64>(arg);
701 # elif SANITIZER_WINDOWS
702 return Func::template Apply<MappingGoWindows>(arg);
703 # else
704 return Func::template Apply<MappingGo48>(arg);
705 # endif
706 #else // SANITIZER_GO
707 # if SANITIZER_IOS && !SANITIZER_IOSSIM
708 return Func::template Apply<MappingAppleAarch64>(arg);
709 # elif defined(__x86_64__) || SANITIZER_APPLE
710 return Func::template Apply<Mapping48AddressSpace>(arg);
711 # elif defined(__aarch64__)
712 switch (vmaSize) {
713 case 39:
714 return Func::template Apply<MappingAarch64_39>(arg);
715 case 42:
716 return Func::template Apply<MappingAarch64_42>(arg);
717 case 48:
718 return Func::template Apply<MappingAarch64_48>(arg);
719 }
720 # elif SANITIZER_LOONGARCH64
721 return Func::template Apply<MappingLoongArch64_47>(arg);
722 # elif defined(__powerpc64__)
723 switch (vmaSize) {
724 case 44:
725 return Func::template Apply<MappingPPC64_44>(arg);
726 case 46:
727 return Func::template Apply<MappingPPC64_46>(arg);
728 case 47:
729 return Func::template Apply<MappingPPC64_47>(arg);
730 }
731 # elif defined(__mips64)
732 return Func::template Apply<MappingMips64_40>(arg);
733 # elif SANITIZER_RISCV64
734 switch (vmaSize) {
735 case 39:
736 return Func::template Apply<MappingRiscv64_39>(arg);
737 case 48:
738 return Func::template Apply<MappingRiscv64_48>(arg);
739 }
740 # elif defined(__s390x__)
741 return Func::template Apply<MappingS390x>(arg);
742 # else
743 # error "unsupported platform"
744 # endif
745 #endif
746 Die();
747 }
748
749 template <typename Func>
750 void ForEachMapping() {
751 Func::template Apply<Mapping48AddressSpace>();
752 Func::template Apply<MappingMips64_40>();
753 Func::template Apply<MappingAppleAarch64>();
754 Func::template Apply<MappingAarch64_39>();
755 Func::template Apply<MappingAarch64_42>();
756 Func::template Apply<MappingAarch64_48>();
757 Func::template Apply<MappingLoongArch64_47>();
758 Func::template Apply<MappingPPC64_44>();
759 Func::template Apply<MappingPPC64_46>();
760 Func::template Apply<MappingPPC64_47>();
761 Func::template Apply<MappingRiscv64_39>();
762 Func::template Apply<MappingRiscv64_48>();
763 Func::template Apply<MappingS390x>();
764 Func::template Apply<MappingGo48>();
765 Func::template Apply<MappingGoWindows>();
766 Func::template Apply<MappingGoPPC64_46>();
767 Func::template Apply<MappingGoPPC64_47>();
768 Func::template Apply<MappingGoAarch64>();
769 Func::template Apply<MappingGoMips64_47>();
770 Func::template Apply<MappingGoS390x>();
771 }
772
773 enum MappingType {
774 kLoAppMemBeg,
775 kLoAppMemEnd,
776 kHiAppMemBeg,
777 kHiAppMemEnd,
778 kMidAppMemBeg,
779 kMidAppMemEnd,
780 kHeapMemBeg,
781 kHeapMemEnd,
782 kShadowBeg,
783 kShadowEnd,
784 kMetaShadowBeg,
785 kMetaShadowEnd,
786 kVdsoBeg,
787 };
788
789 struct MappingField {
790 template <typename Mapping>
791 static uptr Apply(MappingType type) {
792 switch (type) {
793 case kLoAppMemBeg:
794 return Mapping::kLoAppMemBeg;
795 case kLoAppMemEnd:
796 return Mapping::kLoAppMemEnd;
797 case kMidAppMemBeg:
798 return Mapping::kMidAppMemBeg;
799 case kMidAppMemEnd:
800 return Mapping::kMidAppMemEnd;
801 case kHiAppMemBeg:
802 return Mapping::kHiAppMemBeg;
803 case kHiAppMemEnd:
804 return Mapping::kHiAppMemEnd;
805 case kHeapMemBeg:
806 return Mapping::kHeapMemBeg;
807 case kHeapMemEnd:
808 return Mapping::kHeapMemEnd;
809 case kVdsoBeg:
810 return Mapping::kVdsoBeg;
811 case kShadowBeg:
812 return Mapping::kShadowBeg;
813 case kShadowEnd:
814 return Mapping::kShadowEnd;
815 case kMetaShadowBeg:
816 return Mapping::kMetaShadowBeg;
817 case kMetaShadowEnd:
818 return Mapping::kMetaShadowEnd;
819 }
820 Die();
821 }
822 };
823
824 ALWAYS_INLINE
825 uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
826 ALWAYS_INLINE
827 uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
828
829 ALWAYS_INLINE
830 uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
831 ALWAYS_INLINE
832 uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
833
834 ALWAYS_INLINE
835 uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
836 ALWAYS_INLINE
837 uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
838
839 ALWAYS_INLINE
840 uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
841 ALWAYS_INLINE
842 uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
843
844 ALWAYS_INLINE
845 uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
846
847 ALWAYS_INLINE
848 uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
849 ALWAYS_INLINE
850 uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
851
852 ALWAYS_INLINE
853 uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
854 ALWAYS_INLINE
855 uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
856
857 struct IsAppMemImpl {
858 template <typename Mapping>
859 static bool Apply(uptr mem) {
860 return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
861 (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
862 (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
863 (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
864 }
865 };
866
867 ALWAYS_INLINE
868 bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
869
870 struct IsShadowMemImpl {
871 template <typename Mapping>
872 static bool Apply(uptr mem) {
873 return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
874 }
875 };
876
877 ALWAYS_INLINE
878 bool IsShadowMem(RawShadow *p) {
879 return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
880 }
881
882 struct IsMetaMemImpl {
883 template <typename Mapping>
884 static bool Apply(uptr mem) {
885 return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
886 }
887 };
888
889 ALWAYS_INLINE
890 bool IsMetaMem(const u32 *p) {
891 return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
892 }
893
894 struct MemToShadowImpl {
895 template <typename Mapping>
896 static uptr Apply(uptr x) {
897 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
898 return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
899 Mapping::kShadowXor) *
900 kShadowMultiplier +
901 Mapping::kShadowAdd;
902 }
903 };
904
905 ALWAYS_INLINE
906 RawShadow *MemToShadow(uptr x) {
907 return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
908 }
909
910 struct MemToMetaImpl {
911 template <typename Mapping>
912 static u32 *Apply(uptr x) {
913 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
914 return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
915 kMetaShadowCell * kMetaShadowSize) |
916 Mapping::kMetaShadowBeg);
917 }
918 };
919
920 ALWAYS_INLINE
921 u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
922
923 struct ShadowToMemImpl {
924 template <typename Mapping>
925 static uptr Apply(uptr sp) {
926 if (!IsShadowMemImpl::Apply<Mapping>(sp))
927 return 0;
928 // The shadow mapping is non-linear and we've lost some bits, so we don't
929 // have an easy way to restore the original app address. But the mapping is
930 // a bijection, so we try to restore the address as belonging to
931 // low/mid/high range consecutively and see if shadow->app->shadow mapping
932 // gives us the same address.
933 uptr p =
934 ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
935 if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
936 MemToShadowImpl::Apply<Mapping>(p) == sp)
937 return p;
938 if (Mapping::kMidAppMemBeg) {
939 uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
940 if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
941 MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
942 return p_mid;
943 }
944 return p | Mapping::kShadowMsk;
945 }
946 };
947
948 ALWAYS_INLINE
949 uptr ShadowToMem(RawShadow *s) {
950 return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
951 }
952
953 // Compresses addr to kCompressedAddrBits stored in least significant bits.
954 ALWAYS_INLINE uptr CompressAddr(uptr addr) {
955 return addr & ((1ull << kCompressedAddrBits) - 1);
956 }
957
958 struct RestoreAddrImpl {
959 typedef uptr Result;
960 template <typename Mapping>
961 static Result Apply(uptr addr) {
962 // To restore the address we go over all app memory ranges and check if top
963 // 3 bits of the compressed addr match that of the app range. If yes, we
964 // assume that the compressed address come from that range and restore the
965 // missing top bits to match the app range address.
966 const uptr ranges[] = {
967 Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
968 Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
969 Mapping::kHeapMemBeg, Mapping::kHeapMemEnd,
970 };
971 const uptr indicator = 0x0f0000000000ull;
972 const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
973 for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
974 uptr beg = ranges[i];
975 uptr end = ranges[i + 1];
976 if (beg == end)
977 continue;
978 for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
979 if ((addr & indicator) == (p & indicator))
980 return addr | (p & ~(ind_lsb - 1));
981 }
982 }
983 Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
984 Die();
985 }
986 };
987
988 // Restores compressed addr from kCompressedAddrBits to full representation.
989 // This is called only during reporting and is not performance-critical.
990 inline uptr RestoreAddr(uptr addr) {
991 return SelectMapping<RestoreAddrImpl>(addr);
992 }
993
994 void InitializePlatform();
995 void InitializePlatformEarly();
996 void CheckAndProtect();
997 void InitializeShadowMemoryPlatform();
998 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
999 int ExtractResolvFDs(void *state, int *fds, int nfd);
1000 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
1001 uptr ExtractLongJmpSp(uptr *env);
1002 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
1003
1004 int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
1005 void (*cleanup)(void *arg), void *arg);
1006
1007 void DestroyThreadState();
1008 void PlatformCleanUpThreadState(ThreadState *thr);
1009
1010 } // namespace __tsan
1011
1012 #endif // TSAN_PLATFORM_H
LLVM monorepo