?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'pull-loongarch-20241016' of https://gitlab.com/gaosong/qemu into staging
[qemu/armbru.git] / tcg / tcg-op-ldst.c
bloba318011229606de50f99ed0c7b6661b7a3c96ef2
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "tcg/tcg.h"
27 #include "tcg/tcg-temp-internal.h"
28 #include "tcg/tcg-op-common.h"
29 #include "tcg/tcg-mo.h"
30 #include "exec/translation-block.h"
31 #include "exec/plugin-gen.h"
32 #include "tcg-internal.h"
33
34
35 static void check_max_alignment(unsigned a_bits)
36 {
37 /*
38 * The requested alignment cannot overlap the TLB flags.
39 * FIXME: Must keep the count up-to-date with "exec/cpu-all.h".
40 */
41 if (tcg_use_softmmu) {
42 tcg_debug_assert(a_bits + 5 <= tcg_ctx->page_bits);
43 }
44 }
45
46 static MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
47 {
48 unsigned a_bits = memop_alignment_bits(op);
49
50 check_max_alignment(a_bits);
51
52 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
53 if (a_bits == (op & MO_SIZE)) {
54 op = (op & ~MO_AMASK) | MO_ALIGN;
55 }
56
57 switch (op & MO_SIZE) {
58 case MO_8:
59 op &= ~MO_BSWAP;
60 break;
61 case MO_16:
62 break;
63 case MO_32:
64 if (!is64) {
65 op &= ~MO_SIGN;
66 }
67 break;
68 case MO_64:
69 if (is64) {
70 op &= ~MO_SIGN;
71 break;
72 }
73 /* fall through */
74 default:
75 g_assert_not_reached();
76 }
77 if (st) {
78 op &= ~MO_SIGN;
79 }
80
81 /* In serial mode, reduce atomicity. */
82 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
83 op &= ~MO_ATOM_MASK;
84 op |= MO_ATOM_NONE;
85 }
86
87 return op;
88 }
89
90 static void gen_ldst(TCGOpcode opc, TCGTemp *vl, TCGTemp *vh,
91 TCGTemp *addr, MemOpIdx oi)
92 {
93 if (TCG_TARGET_REG_BITS == 64 || tcg_ctx->addr_type == TCG_TYPE_I32) {
94 if (vh) {
95 tcg_gen_op4(opc, temp_arg(vl), temp_arg(vh), temp_arg(addr), oi);
96 } else {
97 tcg_gen_op3(opc, temp_arg(vl), temp_arg(addr), oi);
98 }
99 } else {
100 /* See TCGV_LOW/HIGH. */
101 TCGTemp *al = addr + HOST_BIG_ENDIAN;
102 TCGTemp *ah = addr + !HOST_BIG_ENDIAN;
103
104 if (vh) {
105 tcg_gen_op5(opc, temp_arg(vl), temp_arg(vh),
106 temp_arg(al), temp_arg(ah), oi);
107 } else {
108 tcg_gen_op4(opc, temp_arg(vl), temp_arg(al), temp_arg(ah), oi);
109 }
110 }
111 }
112
113 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 v, TCGTemp *addr, MemOpIdx oi)
114 {
115 if (TCG_TARGET_REG_BITS == 32) {
116 TCGTemp *vl = tcgv_i32_temp(TCGV_LOW(v));
117 TCGTemp *vh = tcgv_i32_temp(TCGV_HIGH(v));
118 gen_ldst(opc, vl, vh, addr, oi);
119 } else {
120 gen_ldst(opc, tcgv_i64_temp(v), NULL, addr, oi);
121 }
122 }
123
124 static void tcg_gen_req_mo(TCGBar type)
125 {
126 type &= tcg_ctx->guest_mo;
127 type &= ~TCG_TARGET_DEFAULT_MO;
128 if (type) {
129 tcg_gen_mb(type | TCG_BAR_SC);
130 }
131 }
132
133 /* Only required for loads, where value might overlap addr. */
134 static TCGv_i64 plugin_maybe_preserve_addr(TCGTemp *addr)
135 {
136 #ifdef CONFIG_PLUGIN
137 if (tcg_ctx->plugin_insn != NULL) {
138 /* Save a copy of the vaddr for use after a load. */
139 TCGv_i64 temp = tcg_temp_ebb_new_i64();
140 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
141 tcg_gen_extu_i32_i64(temp, temp_tcgv_i32(addr));
142 } else {
143 tcg_gen_mov_i64(temp, temp_tcgv_i64(addr));
144 }
145 return temp;
146 }
147 #endif
148 return NULL;
149 }
150
151 #ifdef CONFIG_PLUGIN
152 static void
153 plugin_gen_mem_callbacks(TCGv_i64 copy_addr, TCGTemp *orig_addr, MemOpIdx oi,
154 enum qemu_plugin_mem_rw rw)
155 {
156 if (tcg_ctx->plugin_insn != NULL) {
157 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw);
158
159 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
160 if (!copy_addr) {
161 copy_addr = tcg_temp_ebb_new_i64();
162 tcg_gen_extu_i32_i64(copy_addr, temp_tcgv_i32(orig_addr));
163 }
164 tcg_gen_plugin_mem_cb(copy_addr, info);
165 tcg_temp_free_i64(copy_addr);
166 } else {
167 if (copy_addr) {
168 tcg_gen_plugin_mem_cb(copy_addr, info);
169 tcg_temp_free_i64(copy_addr);
170 } else {
171 tcg_gen_plugin_mem_cb(temp_tcgv_i64(orig_addr), info);
172 }
173 }
174 }
175 }
176 #endif
177
178 static void
179 plugin_gen_mem_callbacks_i32(TCGv_i32 val,
180 TCGv_i64 copy_addr, TCGTemp *orig_addr,
181 MemOpIdx oi, enum qemu_plugin_mem_rw rw)
182 {
183 #ifdef CONFIG_PLUGIN
184 if (tcg_ctx->plugin_insn != NULL) {
185 tcg_gen_st_i32(val, tcg_env,
186 offsetof(CPUState, neg.plugin_mem_value_low) -
187 sizeof(CPUState) + (HOST_BIG_ENDIAN * 4));
188 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw);
189 }
190 #endif
191 }
192
193 static void
194 plugin_gen_mem_callbacks_i64(TCGv_i64 val,
195 TCGv_i64 copy_addr, TCGTemp *orig_addr,
196 MemOpIdx oi, enum qemu_plugin_mem_rw rw)
197 {
198 #ifdef CONFIG_PLUGIN
199 if (tcg_ctx->plugin_insn != NULL) {
200 tcg_gen_st_i64(val, tcg_env,
201 offsetof(CPUState, neg.plugin_mem_value_low) -
202 sizeof(CPUState));
203 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw);
204 }
205 #endif
206 }
207
208 static void
209 plugin_gen_mem_callbacks_i128(TCGv_i128 val,
210 TCGv_i64 copy_addr, TCGTemp *orig_addr,
211 MemOpIdx oi, enum qemu_plugin_mem_rw rw)
212 {
213 #ifdef CONFIG_PLUGIN
214 if (tcg_ctx->plugin_insn != NULL) {
215 tcg_gen_st_i64(TCGV128_LOW(val), tcg_env,
216 offsetof(CPUState, neg.plugin_mem_value_low) -
217 sizeof(CPUState));
218 tcg_gen_st_i64(TCGV128_HIGH(val), tcg_env,
219 offsetof(CPUState, neg.plugin_mem_value_high) -
220 sizeof(CPUState));
221 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw);
222 }
223 #endif
224 }
225
226 static void tcg_gen_qemu_ld_i32_int(TCGv_i32 val, TCGTemp *addr,
227 TCGArg idx, MemOp memop)
228 {
229 MemOp orig_memop;
230 MemOpIdx orig_oi, oi;
231 TCGv_i64 copy_addr;
232 TCGOpcode opc;
233
234 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
235 orig_memop = memop = tcg_canonicalize_memop(memop, 0, 0);
236 orig_oi = oi = make_memop_idx(memop, idx);
237
238 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
239 memop &= ~MO_BSWAP;
240 /* The bswap primitive benefits from zero-extended input. */
241 if ((memop & MO_SSIZE) == MO_SW) {
242 memop &= ~MO_SIGN;
243 }
244 oi = make_memop_idx(memop, idx);
245 }
246
247 copy_addr = plugin_maybe_preserve_addr(addr);
248 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
249 opc = INDEX_op_qemu_ld_a32_i32;
250 } else {
251 opc = INDEX_op_qemu_ld_a64_i32;
252 }
253 gen_ldst(opc, tcgv_i32_temp(val), NULL, addr, oi);
254 plugin_gen_mem_callbacks_i32(val, copy_addr, addr, orig_oi,
255 QEMU_PLUGIN_MEM_R);
256
257 if ((orig_memop ^ memop) & MO_BSWAP) {
258 switch (orig_memop & MO_SIZE) {
259 case MO_16:
260 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
261 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
262 : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
263 break;
264 case MO_32:
265 tcg_gen_bswap32_i32(val, val);
266 break;
267 default:
268 g_assert_not_reached();
269 }
270 }
271 }
272
273 void tcg_gen_qemu_ld_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx,
274 MemOp memop, TCGType addr_type)
275 {
276 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
277 tcg_debug_assert((memop & MO_SIZE) <= MO_32);
278 tcg_gen_qemu_ld_i32_int(val, addr, idx, memop);
279 }
280
281 static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr,
282 TCGArg idx, MemOp memop)
283 {
284 TCGv_i32 swap = NULL;
285 MemOpIdx orig_oi, oi;
286 TCGOpcode opc;
287
288 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
289 memop = tcg_canonicalize_memop(memop, 0, 1);
290 orig_oi = oi = make_memop_idx(memop, idx);
291
292 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
293 swap = tcg_temp_ebb_new_i32();
294 switch (memop & MO_SIZE) {
295 case MO_16:
296 tcg_gen_bswap16_i32(swap, val, 0);
297 break;
298 case MO_32:
299 tcg_gen_bswap32_i32(swap, val);
300 break;
301 default:
302 g_assert_not_reached();
303 }
304 val = swap;
305 memop &= ~MO_BSWAP;
306 oi = make_memop_idx(memop, idx);
307 }
308
309 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
310 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
311 opc = INDEX_op_qemu_st8_a32_i32;
312 } else {
313 opc = INDEX_op_qemu_st8_a64_i32;
314 }
315 } else {
316 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
317 opc = INDEX_op_qemu_st_a32_i32;
318 } else {
319 opc = INDEX_op_qemu_st_a64_i32;
320 }
321 }
322 gen_ldst(opc, tcgv_i32_temp(val), NULL, addr, oi);
323 plugin_gen_mem_callbacks_i32(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W);
324
325 if (swap) {
326 tcg_temp_free_i32(swap);
327 }
328 }
329
330 void tcg_gen_qemu_st_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx,
331 MemOp memop, TCGType addr_type)
332 {
333 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
334 tcg_debug_assert((memop & MO_SIZE) <= MO_32);
335 tcg_gen_qemu_st_i32_int(val, addr, idx, memop);
336 }
337
338 static void tcg_gen_qemu_ld_i64_int(TCGv_i64 val, TCGTemp *addr,
339 TCGArg idx, MemOp memop)
340 {
341 MemOp orig_memop;
342 MemOpIdx orig_oi, oi;
343 TCGv_i64 copy_addr;
344 TCGOpcode opc;
345
346 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
347 tcg_gen_qemu_ld_i32_int(TCGV_LOW(val), addr, idx, memop);
348 if (memop & MO_SIGN) {
349 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
350 } else {
351 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
352 }
353 return;
354 }
355
356 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
357 orig_memop = memop = tcg_canonicalize_memop(memop, 1, 0);
358 orig_oi = oi = make_memop_idx(memop, idx);
359
360 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
361 memop &= ~MO_BSWAP;
362 /* The bswap primitive benefits from zero-extended input. */
363 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
364 memop &= ~MO_SIGN;
365 }
366 oi = make_memop_idx(memop, idx);
367 }
368
369 copy_addr = plugin_maybe_preserve_addr(addr);
370 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
371 opc = INDEX_op_qemu_ld_a32_i64;
372 } else {
373 opc = INDEX_op_qemu_ld_a64_i64;
374 }
375 gen_ldst_i64(opc, val, addr, oi);
376 plugin_gen_mem_callbacks_i64(val, copy_addr, addr, orig_oi,
377 QEMU_PLUGIN_MEM_R);
378
379 if ((orig_memop ^ memop) & MO_BSWAP) {
380 int flags = (orig_memop & MO_SIGN
381 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
382 : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
383 switch (orig_memop & MO_SIZE) {
384 case MO_16:
385 tcg_gen_bswap16_i64(val, val, flags);
386 break;
387 case MO_32:
388 tcg_gen_bswap32_i64(val, val, flags);
389 break;
390 case MO_64:
391 tcg_gen_bswap64_i64(val, val);
392 break;
393 default:
394 g_assert_not_reached();
395 }
396 }
397 }
398
399 void tcg_gen_qemu_ld_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx,
400 MemOp memop, TCGType addr_type)
401 {
402 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
403 tcg_debug_assert((memop & MO_SIZE) <= MO_64);
404 tcg_gen_qemu_ld_i64_int(val, addr, idx, memop);
405 }
406
407 static void tcg_gen_qemu_st_i64_int(TCGv_i64 val, TCGTemp *addr,
408 TCGArg idx, MemOp memop)
409 {
410 TCGv_i64 swap = NULL;
411 MemOpIdx orig_oi, oi;
412 TCGOpcode opc;
413
414 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
415 tcg_gen_qemu_st_i32_int(TCGV_LOW(val), addr, idx, memop);
416 return;
417 }
418
419 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
420 memop = tcg_canonicalize_memop(memop, 1, 1);
421 orig_oi = oi = make_memop_idx(memop, idx);
422
423 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
424 swap = tcg_temp_ebb_new_i64();
425 switch (memop & MO_SIZE) {
426 case MO_16:
427 tcg_gen_bswap16_i64(swap, val, 0);
428 break;
429 case MO_32:
430 tcg_gen_bswap32_i64(swap, val, 0);
431 break;
432 case MO_64:
433 tcg_gen_bswap64_i64(swap, val);
434 break;
435 default:
436 g_assert_not_reached();
437 }
438 val = swap;
439 memop &= ~MO_BSWAP;
440 oi = make_memop_idx(memop, idx);
441 }
442
443 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
444 opc = INDEX_op_qemu_st_a32_i64;
445 } else {
446 opc = INDEX_op_qemu_st_a64_i64;
447 }
448 gen_ldst_i64(opc, val, addr, oi);
449 plugin_gen_mem_callbacks_i64(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W);
450
451 if (swap) {
452 tcg_temp_free_i64(swap);
453 }
454 }
455
456 void tcg_gen_qemu_st_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx,
457 MemOp memop, TCGType addr_type)
458 {
459 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
460 tcg_debug_assert((memop & MO_SIZE) <= MO_64);
461 tcg_gen_qemu_st_i64_int(val, addr, idx, memop);
462 }
463
464 /*
465 * Return true if @mop, without knowledge of the pointer alignment,
466 * does not require 16-byte atomicity, and it would be adventagous
467 * to avoid a call to a helper function.
468 */
469 static bool use_two_i64_for_i128(MemOp mop)
470 {
471 /* Two softmmu tlb lookups is larger than one function call. */
472 if (tcg_use_softmmu) {
473 return false;
474 }
475
476 /*
477 * For user-only, two 64-bit operations may well be smaller than a call.
478 * Determine if that would be legal for the requested atomicity.
479 */
480 switch (mop & MO_ATOM_MASK) {
481 case MO_ATOM_NONE:
482 case MO_ATOM_IFALIGN_PAIR:
483 return true;
484 case MO_ATOM_IFALIGN:
485 case MO_ATOM_SUBALIGN:
486 case MO_ATOM_WITHIN16:
487 case MO_ATOM_WITHIN16_PAIR:
488 return false;
489 default:
490 g_assert_not_reached();
491 }
492 }
493
494 static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig)
495 {
496 MemOp mop_1 = orig, mop_2;
497
498 /* Reduce the size to 64-bit. */
499 mop_1 = (mop_1 & ~MO_SIZE) | MO_64;
500
501 /* Retain the alignment constraints of the original. */
502 switch (orig & MO_AMASK) {
503 case MO_UNALN:
504 case MO_ALIGN_2:
505 case MO_ALIGN_4:
506 mop_2 = mop_1;
507 break;
508 case MO_ALIGN_8:
509 /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */
510 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN;
511 mop_2 = mop_1;
512 break;
513 case MO_ALIGN:
514 /* Second has 8-byte alignment; first has 16-byte alignment. */
515 mop_2 = mop_1;
516 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16;
517 break;
518 case MO_ALIGN_16:
519 case MO_ALIGN_32:
520 case MO_ALIGN_64:
521 /* Second has 8-byte alignment; first retains original. */
522 mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN;
523 break;
524 default:
525 g_assert_not_reached();
526 }
527
528 /* Use a memory ordering implemented by the host. */
529 if ((orig & MO_BSWAP) && !tcg_target_has_memory_bswap(mop_1)) {
530 mop_1 &= ~MO_BSWAP;
531 mop_2 &= ~MO_BSWAP;
532 }
533
534 ret[0] = mop_1;
535 ret[1] = mop_2;
536 }
537
538 static TCGv_i64 maybe_extend_addr64(TCGTemp *addr)
539 {
540 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
541 TCGv_i64 a64 = tcg_temp_ebb_new_i64();
542 tcg_gen_extu_i32_i64(a64, temp_tcgv_i32(addr));
543 return a64;
544 }
545 return temp_tcgv_i64(addr);
546 }
547
548 static void maybe_free_addr64(TCGv_i64 a64)
549 {
550 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
551 tcg_temp_free_i64(a64);
552 }
553 }
554
555 static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr,
556 TCGArg idx, MemOp memop)
557 {
558 MemOpIdx orig_oi;
559 TCGv_i64 ext_addr = NULL;
560 TCGOpcode opc;
561
562 check_max_alignment(memop_alignment_bits(memop));
563 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
564
565 /* In serial mode, reduce atomicity. */
566 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
567 memop &= ~MO_ATOM_MASK;
568 memop |= MO_ATOM_NONE;
569 }
570 orig_oi = make_memop_idx(memop, idx);
571
572 /* TODO: For now, force 32-bit hosts to use the helper. */
573 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) {
574 TCGv_i64 lo, hi;
575 bool need_bswap = false;
576 MemOpIdx oi = orig_oi;
577
578 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
579 lo = TCGV128_HIGH(val);
580 hi = TCGV128_LOW(val);
581 oi = make_memop_idx(memop & ~MO_BSWAP, idx);
582 need_bswap = true;
583 } else {
584 lo = TCGV128_LOW(val);
585 hi = TCGV128_HIGH(val);
586 }
587
588 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
589 opc = INDEX_op_qemu_ld_a32_i128;
590 } else {
591 opc = INDEX_op_qemu_ld_a64_i128;
592 }
593 gen_ldst(opc, tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi);
594
595 if (need_bswap) {
596 tcg_gen_bswap64_i64(lo, lo);
597 tcg_gen_bswap64_i64(hi, hi);
598 }
599 } else if (use_two_i64_for_i128(memop)) {
600 MemOp mop[2];
601 TCGTemp *addr_p8;
602 TCGv_i64 x, y;
603 bool need_bswap;
604
605 canonicalize_memop_i128_as_i64(mop, memop);
606 need_bswap = (mop[0] ^ memop) & MO_BSWAP;
607
608 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
609 opc = INDEX_op_qemu_ld_a32_i64;
610 } else {
611 opc = INDEX_op_qemu_ld_a64_i64;
612 }
613
614 /*
615 * Since there are no global TCGv_i128, there is no visible state
616 * changed if the second load faults. Load directly into the two
617 * subwords.
618 */
619 if ((memop & MO_BSWAP) == MO_LE) {
620 x = TCGV128_LOW(val);
621 y = TCGV128_HIGH(val);
622 } else {
623 x = TCGV128_HIGH(val);
624 y = TCGV128_LOW(val);
625 }
626
627 gen_ldst_i64(opc, x, addr, make_memop_idx(mop[0], idx));
628
629 if (need_bswap) {
630 tcg_gen_bswap64_i64(x, x);
631 }
632
633 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
634 TCGv_i32 t = tcg_temp_ebb_new_i32();
635 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8);
636 addr_p8 = tcgv_i32_temp(t);
637 } else {
638 TCGv_i64 t = tcg_temp_ebb_new_i64();
639 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8);
640 addr_p8 = tcgv_i64_temp(t);
641 }
642
643 gen_ldst_i64(opc, y, addr_p8, make_memop_idx(mop[1], idx));
644 tcg_temp_free_internal(addr_p8);
645
646 if (need_bswap) {
647 tcg_gen_bswap64_i64(y, y);
648 }
649 } else {
650 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
651 ext_addr = tcg_temp_ebb_new_i64();
652 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr));
653 addr = tcgv_i64_temp(ext_addr);
654 }
655 gen_helper_ld_i128(val, tcg_env, temp_tcgv_i64(addr),
656 tcg_constant_i32(orig_oi));
657 }
658
659 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi,
660 QEMU_PLUGIN_MEM_R);
661 }
662
663 void tcg_gen_qemu_ld_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx,
664 MemOp memop, TCGType addr_type)
665 {
666 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
667 tcg_debug_assert((memop & MO_SIZE) == MO_128);
668 tcg_debug_assert((memop & MO_SIGN) == 0);
669 tcg_gen_qemu_ld_i128_int(val, addr, idx, memop);
670 }
671
672 static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr,
673 TCGArg idx, MemOp memop)
674 {
675 MemOpIdx orig_oi;
676 TCGv_i64 ext_addr = NULL;
677 TCGOpcode opc;
678
679 check_max_alignment(memop_alignment_bits(memop));
680 tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST);
681
682 /* In serial mode, reduce atomicity. */
683 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
684 memop &= ~MO_ATOM_MASK;
685 memop |= MO_ATOM_NONE;
686 }
687 orig_oi = make_memop_idx(memop, idx);
688
689 /* TODO: For now, force 32-bit hosts to use the helper. */
690
691 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) {
692 TCGv_i64 lo, hi;
693 MemOpIdx oi = orig_oi;
694 bool need_bswap = false;
695
696 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
697 lo = tcg_temp_ebb_new_i64();
698 hi = tcg_temp_ebb_new_i64();
699 tcg_gen_bswap64_i64(lo, TCGV128_HIGH(val));
700 tcg_gen_bswap64_i64(hi, TCGV128_LOW(val));
701 oi = make_memop_idx(memop & ~MO_BSWAP, idx);
702 need_bswap = true;
703 } else {
704 lo = TCGV128_LOW(val);
705 hi = TCGV128_HIGH(val);
706 }
707
708 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
709 opc = INDEX_op_qemu_st_a32_i128;
710 } else {
711 opc = INDEX_op_qemu_st_a64_i128;
712 }
713 gen_ldst(opc, tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi);
714
715 if (need_bswap) {
716 tcg_temp_free_i64(lo);
717 tcg_temp_free_i64(hi);
718 }
719 } else if (use_two_i64_for_i128(memop)) {
720 MemOp mop[2];
721 TCGTemp *addr_p8;
722 TCGv_i64 x, y, b = NULL;
723
724 canonicalize_memop_i128_as_i64(mop, memop);
725
726 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
727 opc = INDEX_op_qemu_st_a32_i64;
728 } else {
729 opc = INDEX_op_qemu_st_a64_i64;
730 }
731
732 if ((memop & MO_BSWAP) == MO_LE) {
733 x = TCGV128_LOW(val);
734 y = TCGV128_HIGH(val);
735 } else {
736 x = TCGV128_HIGH(val);
737 y = TCGV128_LOW(val);
738 }
739
740 if ((mop[0] ^ memop) & MO_BSWAP) {
741 b = tcg_temp_ebb_new_i64();
742 tcg_gen_bswap64_i64(b, x);
743 x = b;
744 }
745
746 gen_ldst_i64(opc, x, addr, make_memop_idx(mop[0], idx));
747
748 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
749 TCGv_i32 t = tcg_temp_ebb_new_i32();
750 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8);
751 addr_p8 = tcgv_i32_temp(t);
752 } else {
753 TCGv_i64 t = tcg_temp_ebb_new_i64();
754 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8);
755 addr_p8 = tcgv_i64_temp(t);
756 }
757
758 if (b) {
759 tcg_gen_bswap64_i64(b, y);
760 gen_ldst_i64(opc, b, addr_p8, make_memop_idx(mop[1], idx));
761 tcg_temp_free_i64(b);
762 } else {
763 gen_ldst_i64(opc, y, addr_p8, make_memop_idx(mop[1], idx));
764 }
765 tcg_temp_free_internal(addr_p8);
766 } else {
767 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
768 ext_addr = tcg_temp_ebb_new_i64();
769 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr));
770 addr = tcgv_i64_temp(ext_addr);
771 }
772 gen_helper_st_i128(tcg_env, temp_tcgv_i64(addr), val,
773 tcg_constant_i32(orig_oi));
774 }
775
776 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi,
777 QEMU_PLUGIN_MEM_W);
778 }
779
780 void tcg_gen_qemu_st_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx,
781 MemOp memop, TCGType addr_type)
782 {
783 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
784 tcg_debug_assert((memop & MO_SIZE) == MO_128);
785 tcg_debug_assert((memop & MO_SIGN) == 0);
786 tcg_gen_qemu_st_i128_int(val, addr, idx, memop);
787 }
788
789 void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
790 {
791 switch (opc & MO_SSIZE) {
792 case MO_SB:
793 tcg_gen_ext8s_i32(ret, val);
794 break;
795 case MO_UB:
796 tcg_gen_ext8u_i32(ret, val);
797 break;
798 case MO_SW:
799 tcg_gen_ext16s_i32(ret, val);
800 break;
801 case MO_UW:
802 tcg_gen_ext16u_i32(ret, val);
803 break;
804 case MO_UL:
805 case MO_SL:
806 tcg_gen_mov_i32(ret, val);
807 break;
808 default:
809 g_assert_not_reached();
810 }
811 }
812
813 void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
814 {
815 switch (opc & MO_SSIZE) {
816 case MO_SB:
817 tcg_gen_ext8s_i64(ret, val);
818 break;
819 case MO_UB:
820 tcg_gen_ext8u_i64(ret, val);
821 break;
822 case MO_SW:
823 tcg_gen_ext16s_i64(ret, val);
824 break;
825 case MO_UW:
826 tcg_gen_ext16u_i64(ret, val);
827 break;
828 case MO_SL:
829 tcg_gen_ext32s_i64(ret, val);
830 break;
831 case MO_UL:
832 tcg_gen_ext32u_i64(ret, val);
833 break;
834 case MO_UQ:
835 case MO_SQ:
836 tcg_gen_mov_i64(ret, val);
837 break;
838 default:
839 g_assert_not_reached();
840 }
841 }
842
843 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv_i64,
844 TCGv_i32, TCGv_i32, TCGv_i32);
845 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv_i64,
846 TCGv_i64, TCGv_i64, TCGv_i32);
847 typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv_i64,
848 TCGv_i128, TCGv_i128, TCGv_i32);
849 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv_i64,
850 TCGv_i32, TCGv_i32);
851 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv_i64,
852 TCGv_i64, TCGv_i32);
853
854 #ifdef CONFIG_ATOMIC64
855 # define WITH_ATOMIC64(X) X,
856 #else
857 # define WITH_ATOMIC64(X)
858 #endif
859 #if HAVE_CMPXCHG128
860 # define WITH_ATOMIC128(X) X,
861 #else
862 # define WITH_ATOMIC128(X)
863 #endif
864
865 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
866 [MO_8] = gen_helper_atomic_cmpxchgb,
867 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
868 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
869 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
870 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
871 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
872 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
873 WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le)
874 WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be)
875 };
876
877 static void tcg_gen_nonatomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr,
878 TCGv_i32 cmpv, TCGv_i32 newv,
879 TCGArg idx, MemOp memop)
880 {
881 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
882 TCGv_i32 t2 = tcg_temp_ebb_new_i32();
883
884 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
885
886 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop & ~MO_SIGN);
887 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
888 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop);
889 tcg_temp_free_i32(t2);
890
891 if (memop & MO_SIGN) {
892 tcg_gen_ext_i32(retv, t1, memop);
893 } else {
894 tcg_gen_mov_i32(retv, t1);
895 }
896 tcg_temp_free_i32(t1);
897 }
898
899 void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr,
900 TCGv_i32 cmpv, TCGv_i32 newv,
901 TCGArg idx, MemOp memop,
902 TCGType addr_type)
903 {
904 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
905 tcg_debug_assert((memop & MO_SIZE) <= MO_32);
906 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop);
907 }
908
909 static void tcg_gen_atomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr,
910 TCGv_i32 cmpv, TCGv_i32 newv,
911 TCGArg idx, MemOp memop)
912 {
913 gen_atomic_cx_i32 gen;
914 TCGv_i64 a64;
915 MemOpIdx oi;
916
917 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
918 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop);
919 return;
920 }
921
922 memop = tcg_canonicalize_memop(memop, 0, 0);
923 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
924 tcg_debug_assert(gen != NULL);
925
926 oi = make_memop_idx(memop & ~MO_SIGN, idx);
927 a64 = maybe_extend_addr64(addr);
928 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi));
929 maybe_free_addr64(a64);
930
931 if (memop & MO_SIGN) {
932 tcg_gen_ext_i32(retv, retv, memop);
933 }
934 }
935
936 void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr,
937 TCGv_i32 cmpv, TCGv_i32 newv,
938 TCGArg idx, MemOp memop,
939 TCGType addr_type)
940 {
941 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
942 tcg_debug_assert((memop & MO_SIZE) <= MO_32);
943 tcg_gen_atomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop);
944 }
945
946 static void tcg_gen_nonatomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr,
947 TCGv_i64 cmpv, TCGv_i64 newv,
948 TCGArg idx, MemOp memop)
949 {
950 TCGv_i64 t1, t2;
951
952 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
953 tcg_gen_nonatomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
954 TCGV_LOW(newv), idx, memop);
955 if (memop & MO_SIGN) {
956 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
957 } else {
958 tcg_gen_movi_i32(TCGV_HIGH(retv), 0);
959 }
960 return;
961 }
962
963 t1 = tcg_temp_ebb_new_i64();
964 t2 = tcg_temp_ebb_new_i64();
965
966 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
967
968 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop & ~MO_SIGN);
969 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
970 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop);
971 tcg_temp_free_i64(t2);
972
973 if (memop & MO_SIGN) {
974 tcg_gen_ext_i64(retv, t1, memop);
975 } else {
976 tcg_gen_mov_i64(retv, t1);
977 }
978 tcg_temp_free_i64(t1);
979 }
980
981 void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr,
982 TCGv_i64 cmpv, TCGv_i64 newv,
983 TCGArg idx, MemOp memop,
984 TCGType addr_type)
985 {
986 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
987 tcg_debug_assert((memop & MO_SIZE) <= MO_64);
988 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop);
989 }
990
991 static void tcg_gen_atomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr,
992 TCGv_i64 cmpv, TCGv_i64 newv,
993 TCGArg idx, MemOp memop)
994 {
995 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
996 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop);
997 return;
998 }
999
1000 if ((memop & MO_SIZE) == MO_64) {
1001 gen_atomic_cx_i64 gen;
1002
1003 memop = tcg_canonicalize_memop(memop, 1, 0);
1004 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
1005 if (gen) {
1006 MemOpIdx oi = make_memop_idx(memop, idx);
1007 TCGv_i64 a64 = maybe_extend_addr64(addr);
1008 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi));
1009 maybe_free_addr64(a64);
1010 return;
1011 }
1012
1013 gen_helper_exit_atomic(tcg_env);
1014
1015 /*
1016 * Produce a result for a well-formed opcode stream. This satisfies
1017 * liveness for set before used, which happens before this dead code
1018 * is removed.
1019 */
1020 tcg_gen_movi_i64(retv, 0);
1021 return;
1022 }
1023
1024 if (TCG_TARGET_REG_BITS == 32) {
1025 tcg_gen_atomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
1026 TCGV_LOW(newv), idx, memop);
1027 if (memop & MO_SIGN) {
1028 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
1029 } else {
1030 tcg_gen_movi_i32(TCGV_HIGH(retv), 0);
1031 }
1032 } else {
1033 TCGv_i32 c32 = tcg_temp_ebb_new_i32();
1034 TCGv_i32 n32 = tcg_temp_ebb_new_i32();
1035 TCGv_i32 r32 = tcg_temp_ebb_new_i32();
1036
1037 tcg_gen_extrl_i64_i32(c32, cmpv);
1038 tcg_gen_extrl_i64_i32(n32, newv);
1039 tcg_gen_atomic_cmpxchg_i32_int(r32, addr, c32, n32,
1040 idx, memop & ~MO_SIGN);
1041 tcg_temp_free_i32(c32);
1042 tcg_temp_free_i32(n32);
1043
1044 tcg_gen_extu_i32_i64(retv, r32);
1045 tcg_temp_free_i32(r32);
1046
1047 if (memop & MO_SIGN) {
1048 tcg_gen_ext_i64(retv, retv, memop);
1049 }
1050 }
1051 }
1052
1053 void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr,
1054 TCGv_i64 cmpv, TCGv_i64 newv,
1055 TCGArg idx, MemOp memop, TCGType addr_type)
1056 {
1057 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
1058 tcg_debug_assert((memop & MO_SIZE) <= MO_64);
1059 tcg_gen_atomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop);
1060 }
1061
1062 static void tcg_gen_nonatomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr,
1063 TCGv_i128 cmpv, TCGv_i128 newv,
1064 TCGArg idx, MemOp memop)
1065 {
1066 if (TCG_TARGET_REG_BITS == 32) {
1067 /* Inline expansion below is simply too large for 32-bit hosts. */
1068 MemOpIdx oi = make_memop_idx(memop, idx);
1069 TCGv_i64 a64 = maybe_extend_addr64(addr);
1070
1071 gen_helper_nonatomic_cmpxchgo(retv, tcg_env, a64, cmpv, newv,
1072 tcg_constant_i32(oi));
1073 maybe_free_addr64(a64);
1074 } else {
1075 TCGv_i128 oldv = tcg_temp_ebb_new_i128();
1076 TCGv_i128 tmpv = tcg_temp_ebb_new_i128();
1077 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1078 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1079 TCGv_i64 z = tcg_constant_i64(0);
1080
1081 tcg_gen_qemu_ld_i128_int(oldv, addr, idx, memop);
1082
1083 /* Compare i128 */
1084 tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv));
1085 tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv));
1086 tcg_gen_or_i64(t0, t0, t1);
1087
1088 /* tmpv = equal ? newv : oldv */
1089 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z,
1090 TCGV128_LOW(newv), TCGV128_LOW(oldv));
1091 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z,
1092 TCGV128_HIGH(newv), TCGV128_HIGH(oldv));
1093
1094 /* Unconditional writeback. */
1095 tcg_gen_qemu_st_i128_int(tmpv, addr, idx, memop);
1096 tcg_gen_mov_i128(retv, oldv);
1097
1098 tcg_temp_free_i64(t0);
1099 tcg_temp_free_i64(t1);
1100 tcg_temp_free_i128(tmpv);
1101 tcg_temp_free_i128(oldv);
1102 }
1103 }
1104
1105 void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr,
1106 TCGv_i128 cmpv, TCGv_i128 newv,
1107 TCGArg idx, MemOp memop,
1108 TCGType addr_type)
1109 {
1110 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
1111 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128);
1112 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop);
1113 }
1114
1115 static void tcg_gen_atomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr,
1116 TCGv_i128 cmpv, TCGv_i128 newv,
1117 TCGArg idx, MemOp memop)
1118 {
1119 gen_atomic_cx_i128 gen;
1120
1121 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
1122 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop);
1123 return;
1124 }
1125
1126 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
1127 if (gen) {
1128 MemOpIdx oi = make_memop_idx(memop, idx);
1129 TCGv_i64 a64 = maybe_extend_addr64(addr);
1130 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi));
1131 maybe_free_addr64(a64);
1132 return;
1133 }
1134
1135 gen_helper_exit_atomic(tcg_env);
1136
1137 /*
1138 * Produce a result for a well-formed opcode stream. This satisfies
1139 * liveness for set before used, which happens before this dead code
1140 * is removed.
1141 */
1142 tcg_gen_movi_i64(TCGV128_LOW(retv), 0);
1143 tcg_gen_movi_i64(TCGV128_HIGH(retv), 0);
1144 }
1145
1146 void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr,
1147 TCGv_i128 cmpv, TCGv_i128 newv,
1148 TCGArg idx, MemOp memop,
1149 TCGType addr_type)
1150 {
1151 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
1152 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128);
1153 tcg_gen_atomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop);
1154 }
1155
1156 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val,
1157 TCGArg idx, MemOp memop, bool new_val,
1158 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
1159 {
1160 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1161 TCGv_i32 t2 = tcg_temp_ebb_new_i32();
1162
1163 memop = tcg_canonicalize_memop(memop, 0, 0);
1164
1165 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop);
1166 tcg_gen_ext_i32(t2, val, memop);
1167 gen(t2, t1, t2);
1168 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop);
1169
1170 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
1171 tcg_temp_free_i32(t1);
1172 tcg_temp_free_i32(t2);
1173 }
1174
1175 static void do_atomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val,
1176 TCGArg idx, MemOp memop, void * const table[])
1177 {
1178 gen_atomic_op_i32 gen;
1179 TCGv_i64 a64;
1180 MemOpIdx oi;
1181
1182 memop = tcg_canonicalize_memop(memop, 0, 0);
1183
1184 gen = table[memop & (MO_SIZE | MO_BSWAP)];
1185 tcg_debug_assert(gen != NULL);
1186
1187 oi = make_memop_idx(memop & ~MO_SIGN, idx);
1188 a64 = maybe_extend_addr64(addr);
1189 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi));
1190 maybe_free_addr64(a64);
1191
1192 if (memop & MO_SIGN) {
1193 tcg_gen_ext_i32(ret, ret, memop);
1194 }
1195 }
1196
1197 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val,
1198 TCGArg idx, MemOp memop, bool new_val,
1199 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
1200 {
1201 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1202 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
1203
1204 memop = tcg_canonicalize_memop(memop, 1, 0);
1205
1206 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop);
1207 tcg_gen_ext_i64(t2, val, memop);
1208 gen(t2, t1, t2);
1209 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop);
1210
1211 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
1212 tcg_temp_free_i64(t1);
1213 tcg_temp_free_i64(t2);
1214 }
1215
1216 static void do_atomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val,
1217 TCGArg idx, MemOp memop, void * const table[])
1218 {
1219 memop = tcg_canonicalize_memop(memop, 1, 0);
1220
1221 if ((memop & MO_SIZE) == MO_64) {
1222 gen_atomic_op_i64 gen = table[memop & (MO_SIZE | MO_BSWAP)];
1223
1224 if (gen) {
1225 MemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx);
1226 TCGv_i64 a64 = maybe_extend_addr64(addr);
1227 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi));
1228 maybe_free_addr64(a64);
1229 return;
1230 }
1231
1232 gen_helper_exit_atomic(tcg_env);
1233 /* Produce a result, so that we have a well-formed opcode stream
1234 with respect to uses of the result in the (dead) code following. */
1235 tcg_gen_movi_i64(ret, 0);
1236 } else {
1237 TCGv_i32 v32 = tcg_temp_ebb_new_i32();
1238 TCGv_i32 r32 = tcg_temp_ebb_new_i32();
1239
1240 tcg_gen_extrl_i64_i32(v32, val);
1241 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
1242 tcg_temp_free_i32(v32);
1243
1244 tcg_gen_extu_i32_i64(ret, r32);
1245 tcg_temp_free_i32(r32);
1246
1247 if (memop & MO_SIGN) {
1248 tcg_gen_ext_i64(ret, ret, memop);
1249 }
1250 }
1251 }
1252
1253 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
1254 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
1255 [MO_8] = gen_helper_atomic_##NAME##b, \
1256 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
1257 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
1258 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
1259 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
1260 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
1261 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
1262 }; \
1263 void tcg_gen_atomic_##NAME##_i32_chk(TCGv_i32 ret, TCGTemp *addr, \
1264 TCGv_i32 val, TCGArg idx, \
1265 MemOp memop, TCGType addr_type) \
1266 { \
1267 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \
1268 tcg_debug_assert((memop & MO_SIZE) <= MO_32); \
1269 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
1270 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
1271 } else { \
1272 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
1273 tcg_gen_##OP##_i32); \
1274 } \
1275 } \
1276 void tcg_gen_atomic_##NAME##_i64_chk(TCGv_i64 ret, TCGTemp *addr, \
1277 TCGv_i64 val, TCGArg idx, \
1278 MemOp memop, TCGType addr_type) \
1279 { \
1280 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \
1281 tcg_debug_assert((memop & MO_SIZE) <= MO_64); \
1282 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
1283 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
1284 } else { \
1285 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
1286 tcg_gen_##OP##_i64); \
1287 } \
1288 }
1289
1290 GEN_ATOMIC_HELPER(fetch_add, add, 0)
1291 GEN_ATOMIC_HELPER(fetch_and, and, 0)
1292 GEN_ATOMIC_HELPER(fetch_or, or, 0)
1293 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
1294 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
1295 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
1296 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
1297 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
1298
1299 GEN_ATOMIC_HELPER(add_fetch, add, 1)
1300 GEN_ATOMIC_HELPER(and_fetch, and, 1)
1301 GEN_ATOMIC_HELPER(or_fetch, or, 1)
1302 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
1303 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
1304 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
1305 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
1306 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
1307
1308 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
1309 {
1310 tcg_gen_mov_i32(r, b);
1311 }
1312
1313 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
1314 {
1315 tcg_gen_mov_i64(r, b);
1316 }
1317
1318 GEN_ATOMIC_HELPER(xchg, mov2, 0)
1319
1320 #undef GEN_ATOMIC_HELPER
armbru's QEMU hackery