| blob | e3fea3db493879c0f7e95132df4b328e6d541d12 |
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
19 #define frame_offs(x) ((ssize_t)offsetof(struct frame_struct, x) - (ssize_t)frame_offset)
21 #if defined(C_LITTLE_ENDIAN)
22 #define lo_word(size) (0)
23 #define hi_word(size) ((size_t)1 << (size))
24 #elif defined(C_BIG_ENDIAN)
25 #define lo_word(size) ((size_t)1 << (size))
26 #define hi_word(size) (0)
27 #else
28 endian not defined
29 #endif
32 static bool attr_w gen_frame_load_raw(struct codegen_context *ctx, unsigned size, enum extend ex, frame_t slot, int64_t offset, bool dual, unsigned reg);
33 static bool attr_w gen_frame_store_raw(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg);
36 static const struct type *get_type_of_local(struct codegen_context *ctx, frame_t pos)
37 {
38 const struct type *t;
39 const struct data *function = ctx->fn;
40 t = da(function,function)->local_variables[pos].type;
41 if (t)
42 TYPE_TAG_VALIDATE(t->tag);
43 return t;
44 }
46 static unsigned real_type_to_op_size(unsigned real_type)
47 {
48 switch (real_type) {
49 case 0: return OP_SIZE_2;
50 case 1: return OP_SIZE_4;
51 case 2: return OP_SIZE_8;
52 case 3: return OP_SIZE_10;
53 case 4: return OP_SIZE_16;
54 default:
55 internal(file_line, "real_type_to_op_size: invalid type %u", real_type);
56 return 0;
57 }
58 }
60 static unsigned spill_size(const struct type *t)
61 {
62 if (!TYPE_TAG_IS_BUILTIN(t->tag)) {
63 return OP_SIZE_SLOT;
64 } else if (TYPE_TAG_IS_REAL(t->tag)) {
65 return real_type_to_op_size(TYPE_TAG_IDX_REAL(t->tag));
66 } else {
67 return log_2(t->size);
68 }
69 }
71 static bool attr_w spill(struct codegen_context *ctx, frame_t v)
72 {
73 const struct type *t = get_type_of_local(ctx, v);
74 g(gen_frame_store_raw(ctx, spill_size(t), v, 0, ctx->registers[v]));
75 return true;
76 }
78 static bool attr_w unspill(struct codegen_context *ctx, frame_t v)
79 {
80 const struct type *t = get_type_of_local(ctx, v);
81 enum extend ex = garbage;
82 if (t->tag == TYPE_TAG_flat_option)
83 ex = zero;
84 else if (!TYPE_IS_FLAT(t))
85 ex = native;
86 g(gen_frame_load_raw(ctx, spill_size(t), ex, v, 0, false, ctx->registers[v]));
87 return true;
88 }
91 static bool attr_w gen_frame_address(struct codegen_context *ctx, frame_t slot, int64_t offset, unsigned reg)
92 {
93 offset += (size_t)slot * slot_size;
94 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, reg, R_FRAME, offset, 0));
95 return true;
96 }
98 static int64_t get_constant(unsigned size, enum extend ex, frame_t slot, int64_t offset, bool dual)
99 {
100 if (ex == garbage)
101 ex = sign_x;
102 if (ex == native)
103 ex = ARCH_PREFERS_SX(size) ? sign_x : zero_x;
104 if (!dual) {
105 int64_t c;
106 if (unlikely(offset != 0))
107 goto crash;
108 c = frame_t_get_const(slot);
109 if (ex == zero_x) {
110 if (size == OP_SIZE_1) c &= 0xFFUL;
111 if (size == OP_SIZE_2) c &= 0xFFFFUL;
112 if (size == OP_SIZE_4) c &= 0xFFFFFFFFUL;
113 }
114 return c;
115 }
116 if (offset == (int64_t)lo_word(size))
117 return frame_t_get_const(slot);
118 if (offset == (int64_t)hi_word(size))
119 return frame_t_get_const(slot) < 0 ? -1 : 0;
120 crash:
121 internal(file_line, "get_constant: invalid offset %"PRIdMAX"", (intmax_t)offset);
122 }
124 static bool attr_w gen_frame_load_raw(struct codegen_context *ctx, unsigned size, enum extend ex, frame_t slot, int64_t offset, bool dual, unsigned reg)
125 {
126 int64_t x_offset;
127 if (frame_t_is_const(slot)) {
128 g(gen_load_constant(ctx, reg, get_constant(size, ex, slot, offset, dual)));
129 return true;
130 }
131 if (ex == garbage || ex == native) {
132 if (!reg_is_fp(reg))
133 ex = ARCH_PREFERS_SX(size) ? sign_x : zero_x;
134 else
135 ex = zero_x;
136 }
137 x_offset = offset + (size_t)slot * slot_size;
138 if (!ARCH_HAS_BWX && size < OP_SIZE_4) {
139 g(gen_address(ctx, R_FRAME, x_offset, reg_is_fp(reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_LDR_SX_OFFSET, OP_SIZE_4));
140 gen_insn(INSN_MOVSX, OP_SIZE_4, 0, 0);
141 gen_one(reg);
142 gen_address_offset();
144 g(gen_extend(ctx, size, ex, reg, reg));
146 return true;
147 }
148 #if defined(ARCH_ALPHA)
149 if (size < OP_SIZE_4) {
150 g(gen_address(ctx, R_FRAME, x_offset, reg_is_fp(reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_LDR_OFFSET, size));
151 gen_insn(INSN_MOV, size, 0, 0);
152 gen_one(reg);
153 gen_address_offset();
155 if (ex != zero_x)
156 g(gen_extend(ctx, size, ex, reg, reg));
158 return true;
159 }
160 if (size == OP_SIZE_4 && !reg_is_fp(reg) && ex == zero_x) {
161 g(gen_frame_load_raw(ctx, size, sign_x, slot, offset, false, reg));
162 g(gen_extend(ctx, size, ex, reg, reg));
164 return true;
165 }
166 #endif
167 #if defined(ARCH_MIPS)
168 if (reg_is_fp(reg) && size == OP_SIZE_8 && !MIPS_HAS_LS_DOUBLE) {
169 #if defined(C_LITTLE_ENDIAN)
170 g(gen_frame_load_raw(ctx, OP_SIZE_4, zero_x, slot, offset, true, reg));
171 g(gen_frame_load_raw(ctx, OP_SIZE_4, zero_x, slot, offset + 4, true, reg + 1));
172 #else
173 g(gen_frame_load_raw(ctx, OP_SIZE_4, zero_x, slot, offset, true, reg + 1));
174 g(gen_frame_load_raw(ctx, OP_SIZE_4, zero_x, slot, offset + 4, true, reg));
175 #endif
176 return true;
177 }
178 #endif
179 #if defined(ARCH_IA64) || defined(ARCH_PARISC)
180 if (ex == sign_x) {
181 g(gen_address(ctx, R_FRAME, x_offset, IMM_PURPOSE_LDR_OFFSET, size));
182 gen_insn(INSN_MOV, size, 0, 0);
183 gen_one(reg);
184 gen_address_offset();
186 g(gen_extend(ctx, size, ex, reg, reg));
188 return true;
189 }
190 #endif
191 #if defined(ARCH_POWER)
192 if (size == OP_SIZE_1 && ex == sign_x) {
193 g(gen_address(ctx, R_FRAME, x_offset, IMM_PURPOSE_LDR_OFFSET, size));
194 gen_insn(INSN_MOV, size, 0, 0);
195 gen_one(reg);
196 gen_address_offset();
198 g(gen_extend(ctx, size, ex, reg, reg));
200 return true;
201 }
202 #endif
203 #if defined(ARCH_S390)
204 if (size == OP_SIZE_1 && !cpu_test_feature(CPU_FEATURE_long_displacement)) {
205 g(gen_address(ctx, R_FRAME, x_offset, IMM_PURPOSE_LDR_OFFSET, size));
206 gen_insn(INSN_MOV_MASK, OP_SIZE_NATIVE, MOV_MASK_0_8, 0);
207 gen_one(reg);
208 gen_one(reg);
209 gen_address_offset();
211 g(gen_extend(ctx, size, ex, reg, reg));
213 return true;
214 }
215 if (size == OP_SIZE_16 && reg_is_fp(reg)) {
216 g(gen_frame_load_raw(ctx, OP_SIZE_8, zero_x, 0, x_offset, true, reg));
217 g(gen_frame_load_raw(ctx, OP_SIZE_8, zero_x, 0, x_offset + 8, true, reg + 2));
219 return true;
220 }
221 #endif
222 g(gen_address(ctx, R_FRAME, x_offset, reg_is_fp(reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : ex ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, size));
223 gen_insn(unlikely(ex == sign_x) ? INSN_MOVSX : INSN_MOV, size, 0, 0);
224 gen_one(reg);
225 gen_address_offset();
227 return true;
228 }
230 static bool attr_w gen_frame_load(struct codegen_context *ctx, unsigned size, enum extend ex, frame_t slot, int64_t offset, bool dual, unsigned reg)
231 {
232 ajla_assert_lo((slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn)) || frame_t_is_const(slot), (file_line, "gen_frame_load: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
233 if (slot_is_register(ctx, slot)) {
234 if (unlikely(offset != 0))
235 internal(file_line, "gen_frame_load: offset is non-zero: %"PRIdMAX"", (intmax_t)offset);
236 if (ex != garbage && size < OP_SIZE_NATIVE && !reg_is_fp(reg)) {
237 g(gen_extend(ctx, size, ex, reg, ctx->registers[slot]));
238 return true;
239 }
240 g(gen_mov(ctx, !reg_is_fp(reg) ? OP_SIZE_NATIVE : size, reg, ctx->registers[slot]));
241 goto ret;
242 }
244 g(gen_frame_load_raw(ctx, size, ex, slot, offset, dual, reg));
245 ret:
246 #ifdef DEBUG_GARBAGE
247 if (size < OP_SIZE_NATIVE && ex == garbage) {
248 uint64_t mask;
249 g(gen_extend(ctx, size, zero_x, reg, reg));
250 mask = (rand()) | ((uint64_t)rand() << 31) | ((uint64_t)rand() << 62);
251 mask <<= 8ULL << size;
252 g(gen_imm(ctx, mask, IMM_PURPOSE_OR, OP_SIZE_NATIVE));
253 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_OR, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_OR, false, is_imm(), ctx->const_imm));
254 gen_one(reg);
255 gen_one(reg);
256 gen_imm_offset();
257 }
258 #endif
259 return true;
260 }
262 static bool attr_w gen_frame_get(struct codegen_context *ctx, unsigned size, enum extend ex, frame_t slot, unsigned reg, unsigned *dest)
263 {
264 const struct type *t;
265 if (frame_t_is_const(slot)) {
266 g(gen_frame_load(ctx, size, ex, slot, 0, false, reg));
267 *dest = reg;
268 return true;
269 }
270 t = get_type_of_local(ctx, slot);
271 ajla_assert_lo(slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn), (file_line, "gen_frame_get: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
272 if (slot_is_register(ctx, slot)) {
273 unsigned reg = ctx->registers[slot];
274 if (ex != garbage && size < OP_SIZE_NATIVE && !reg_is_fp(reg)) {
275 if (t->tag == TYPE_TAG_flat_option && size <= ARCH_BOOL_SIZE)
276 goto skip_extend;
277 g(gen_extend(ctx, size, ex, reg, reg));
278 }
279 skip_extend:
280 *dest = reg;
281 goto ret;
282 }
283 *dest = reg;
284 g(gen_frame_load(ctx, size, ex, slot, 0, false, reg));
285 ret:
286 #ifdef DEBUG_GARBAGE
287 if (size < OP_SIZE_NATIVE && ex == garbage && t->tag != TYPE_TAG_flat_option) {
288 uint64_t mask;
289 g(gen_extend(ctx, size, zero_x, *dest, *dest));
290 mask = (rand()) | ((uint64_t)rand() << 31) | ((uint64_t)rand() << 62);
291 mask <<= 8ULL << size;
292 g(gen_imm(ctx, mask, IMM_PURPOSE_OR, OP_SIZE_NATIVE));
293 gen_insn(INSN_ALU, OP_SIZE_NATIVE, ALU_OR, ALU_WRITES_FLAGS(OP_SIZE_NATIVE, ALU_OR, false, is_imm(), ctx->const_imm));
294 gen_one(*dest);
295 gen_one(*dest);
296 gen_imm_offset();
297 }
298 #endif
299 return true;
300 }
302 #if defined(ARCH_X86)
303 static bool attr_w gen_frame_load_x87(struct codegen_context *ctx, unsigned insn, unsigned size, unsigned alu, frame_t slot)
304 {
305 if (slot_is_register(ctx, slot))
306 g(spill(ctx,slot));
307 g(gen_address(ctx, R_FRAME, (size_t)slot * slot_size, IMM_PURPOSE_LDR_OFFSET, size));
308 gen_insn(insn, size, alu, 0);
309 gen_address_offset();
310 return true;
311 }
313 static bool attr_w gen_frame_store_x87(struct codegen_context *ctx, unsigned insn, unsigned size, frame_t slot)
314 {
315 g(gen_address(ctx, R_FRAME, (size_t)slot * slot_size, IMM_PURPOSE_STR_OFFSET, size));
316 gen_insn(insn, size, 0, 0);
317 gen_address_offset();
318 if (slot_is_register(ctx, slot))
319 g(unspill(ctx,slot));
320 return true;
321 }
322 #endif
324 static bool attr_w gen_frame_load_op(struct codegen_context *ctx, unsigned size, enum extend ex, unsigned alu, unsigned writes_flags, frame_t slot, int64_t offset, bool dual, unsigned reg)
325 {
326 ajla_assert_lo((slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn)) || frame_t_is_const(slot), (file_line, "gen_frame_load_op: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
327 if (slot_is_register(ctx, slot)) {
328 if (size != i_size(size) + (unsigned)zero && ex != garbage)
329 goto fallback;
330 g(gen_3address_alu(ctx, i_size(size), alu, reg, reg, ctx->registers[slot], writes_flags));
331 return true;
332 }
333 if (frame_t_is_const(slot)) {
334 g(gen_imm(ctx, get_constant(size, ex, slot, offset, dual), alu_purpose(alu), i_size(size)));
335 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), i_size(size), alu, ALU_WRITES_FLAGS(i_size(size), alu, false, is_imm(), ctx->const_imm) | writes_flags);
336 gen_one(reg);
337 gen_one(reg);
338 gen_imm_offset();
339 return true;
340 }
341 #if defined(ARCH_X86) || defined(ARCH_S390)
342 #if defined(ARCH_S390)
343 if (size >= OP_SIZE_4)
344 #endif
345 {
346 offset += (size_t)slot * slot_size;
347 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, size));
348 gen_insn(INSN_ALU + ARCH_PARTIAL_ALU(size), size, alu, ALU_WRITES_FLAGS(size, alu, true, false, 0) | writes_flags);
349 gen_one(reg);
350 gen_one(reg);
351 gen_address_offset();
352 return true;
353 }
354 #endif
355 fallback:
356 #if defined(R_SCRATCH_NA_1)
357 g(gen_frame_load(ctx, size, ex, slot, offset, dual, R_SCRATCH_NA_1));
358 g(gen_3address_alu(ctx, i_size(size), alu, reg, reg, R_SCRATCH_NA_1, writes_flags));
359 #endif
360 return true;
361 }
363 static bool attr_w attr_unused gen_frame_load_op1(struct codegen_context *ctx, unsigned size, unsigned alu, unsigned writes_flags, frame_t slot, unsigned reg)
364 {
365 ajla_assert_lo(slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn), (file_line, "gen_frame_load_op1: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
366 if (slot_is_register(ctx, slot)) {
367 g(gen_2address_alu1(ctx, size, alu, reg, ctx->registers[slot], writes_flags));
368 return true;
369 }
370 #if defined(ARCH_X86)
371 {
372 int64_t offset = (size_t)slot * slot_size;
373 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, size));
374 gen_insn(INSN_ALU1 + ARCH_PARTIAL_ALU(size), size, alu, ALU1_WRITES_FLAGS(alu) | writes_flags);
375 gen_one(reg);
376 gen_address_offset();
377 return true;
378 }
379 #endif
380 #if !defined(ARCH_X86)
381 g(gen_frame_load(ctx, size, garbage, slot, 0, false, reg));
382 g(gen_2address_alu1(ctx, size, alu, reg, reg, writes_flags));
383 return true;
384 #endif
385 }
387 #if ARCH_HAS_FLAGS
388 static bool attr_w gen_frame_load_cmp(struct codegen_context *ctx, unsigned size, bool logical, enum extend attr_unused ex, bool swap, frame_t slot, int64_t offset, bool dual, unsigned reg)
389 {
390 if (slot_is_register(ctx, slot)) {
391 if (size != i_size_cmp(size) + (unsigned)zero && ex != garbage)
392 goto fallback;
393 gen_insn(INSN_CMP, i_size_cmp(size), 0, 1 + logical);
394 if (!swap) {
395 gen_one(reg);
396 gen_one(ctx->registers[slot]);
397 } else {
398 gen_one(ctx->registers[slot]);
399 gen_one(reg);
400 }
401 return true;
402 }
403 if (frame_t_is_const(slot)) {
404 if (!swap) {
405 g(gen_imm(ctx, get_constant(size, ex, slot, offset, dual), !logical ? IMM_PURPOSE_CMP : IMM_PURPOSE_CMP_LOGICAL, size));
406 gen_insn(INSN_CMP, i_size_cmp(size), 0, 1 + logical);
407 gen_one(reg);
408 gen_imm_offset();
409 } else {
410 g(gen_load_constant(ctx, R_CONST_IMM, get_constant(size, ex, slot, offset, dual)));
411 gen_insn(INSN_CMP, i_size_cmp(size), 0, 1 + logical);
412 gen_one(R_CONST_IMM);
413 gen_one(reg);
414 }
415 return true;
416 }
417 #if defined(ARCH_S390) || defined(ARCH_X86)
418 #if defined(ARCH_S390)
419 if (size < OP_SIZE_4)
420 goto fallback;
421 #endif
422 offset += (size_t)slot * slot_size;
423 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDR_OFFSET, size));
424 gen_insn(INSN_CMP, size, 0, 1 + logical);
425 if (!swap) {
426 gen_one(reg);
427 gen_address_offset();
428 } else {
429 gen_address_offset();
430 gen_one(reg);
431 }
432 return true;
433 #endif
434 fallback:
435 #if defined(R_SCRATCH_NA_1)
436 g(gen_frame_load(ctx, size, ex, slot, offset, false, R_SCRATCH_NA_1));
437 gen_insn(INSN_CMP, i_size_cmp(size), 0, 1 + logical);
438 if (!swap) {
439 gen_one(reg);
440 gen_one(R_SCRATCH_NA_1);
441 } else {
442 gen_one(R_SCRATCH_NA_1);
443 gen_one(reg);
444 }
445 #endif
446 return true;
447 }
449 static bool attr_w gen_frame_load_cmp_imm(struct codegen_context *ctx, unsigned size, bool logical, enum extend attr_unused ex, frame_t slot, int64_t offset, int64_t value)
450 {
451 if (slot_is_register(ctx, slot)) {
452 #if defined(ARCH_X86)
453 g(gen_imm(ctx, value, logical ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, size));
454 gen_insn(INSN_CMP, size, 0, 1 + logical);
455 gen_one(ctx->registers[slot]);
456 gen_imm_offset();
457 #else
458 if (size != i_size(size) + (unsigned)zero && size < OP_SIZE_4 && ex != garbage)
459 goto fallback;
460 g(gen_imm(ctx, value, logical ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, size));
461 gen_insn(INSN_CMP, i_size_cmp(size), 0, 1 + logical);
462 gen_one(ctx->registers[slot]);
463 gen_imm_offset();
464 #endif
465 return true;
466 }
467 #if defined(ARCH_X86)
468 offset += (size_t)slot * slot_size;
469 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_MVI_CLI_OFFSET, size));
470 g(gen_imm(ctx, value, logical ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, size));
471 gen_insn(INSN_CMP, size, 0, 1 + logical);
472 gen_address_offset();
473 gen_imm_offset();
474 return true;
475 #endif
476 #if defined(ARCH_S390)
477 if (size != OP_SIZE_1 || !logical)
478 goto fallback;
479 offset += (size_t)slot * slot_size;
480 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_MVI_CLI_OFFSET, size));
481 gen_insn(INSN_CMP, size, 0, 1 + logical);
482 gen_address_offset();
483 gen_one(ARG_IMM);
484 gen_eight((int8_t)value);
485 return true;
486 #endif
487 #if defined(R_SCRATCH_NA_1)
488 goto fallback;
489 fallback:
490 g(gen_frame_load(ctx, size, ex, slot, offset, false, R_SCRATCH_NA_1));
491 g(gen_imm(ctx, value, logical ? IMM_PURPOSE_CMP_LOGICAL : IMM_PURPOSE_CMP, size));
492 gen_insn(INSN_CMP, i_size(size), 0, 1 + logical);
493 gen_one(R_SCRATCH_NA_1);
494 gen_imm_offset();
495 return true;
496 #endif
497 }
498 #endif
500 static bool attr_w gen_frame_load_2(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg1, unsigned reg2)
501 {
502 if (frame_t_is_const(slot))
503 goto skip_ldd;
504 #if defined(ARCH_ARM64)
505 offset += (size_t)slot * slot_size;
506 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDP_STP_OFFSET, size));
507 gen_insn(INSN_LDP, size, 0, 0);
508 gen_one(reg1);
509 gen_one(reg2);
510 gen_address_offset();
511 return true;
512 #endif
513 #if defined(ARCH_ARM32)
514 if (likely(!(reg1 & 1)) && likely(reg2 == reg1 + 1) && likely(cpu_test_feature(CPU_FEATURE_armv6)))
515 #elif defined(ARCH_SPARC32)
516 if (likely(!(reg2 & 1)) && likely(reg1 == reg2 + 1))
517 #elif defined(ARCH_S390)
518 if (likely(reg1 == reg2 + 1))
519 #else
520 if (0)
521 #endif
522 {
523 offset += (size_t)slot * slot_size;
524 if (UNALIGNED_TRAP) {
525 if (unlikely((offset & ((2U << size) - 1)) != 0)) {
526 offset -= (size_t)slot * slot_size;
527 goto skip_ldd;
528 }
529 }
530 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDP_STP_OFFSET, size));
531 gen_insn(INSN_LDP, size, 0, 0);
532 gen_one(reg1);
533 gen_one(reg2);
534 gen_address_offset();
535 return true;
536 }
537 goto skip_ldd;
538 skip_ldd:
539 g(gen_frame_load(ctx, size, garbage, slot, offset + lo_word(size), true, reg1));
540 g(gen_frame_load(ctx, size, garbage, slot, offset + hi_word(size), true, reg2));
541 return true;
542 }
544 static bool attr_w gen_frame_store_raw(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg)
545 {
546 offset += (size_t)slot * slot_size;
547 if (!ARCH_HAS_BWX)
548 size = maximum(OP_SIZE_4, size);
549 #if defined(ARCH_MIPS)
550 if (reg_is_fp(reg) && size == OP_SIZE_8 && !MIPS_HAS_LS_DOUBLE) {
551 #if defined(C_LITTLE_ENDIAN)
552 g(gen_frame_store_raw(ctx, OP_SIZE_4, 0, offset, reg));
553 g(gen_frame_store_raw(ctx, OP_SIZE_4, 0, offset + 4, reg + 1));
554 #else
555 g(gen_frame_store_raw(ctx, OP_SIZE_4, 0, offset, reg + 1));
556 g(gen_frame_store_raw(ctx, OP_SIZE_4, 0, offset + 4, reg));
557 #endif
558 return true;
559 }
560 #endif
561 #if defined(ARCH_S390)
562 if (size == OP_SIZE_16 && reg_is_fp(reg)) {
563 g(gen_frame_store_raw(ctx, OP_SIZE_8, 0, offset, reg));
564 g(gen_frame_store_raw(ctx, OP_SIZE_8, 0, offset + 8, reg + 2));
565 return true;
566 }
567 #endif
568 g(gen_address(ctx, R_FRAME, offset, reg_is_fp(reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_STR_OFFSET, size));
569 gen_insn(INSN_MOV, size, 0, 0);
570 gen_address_offset();
571 gen_one(reg);
572 return true;
573 }
575 static bool attr_w gen_frame_store(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg)
576 {
577 ajla_assert_lo(slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn), (file_line, "gen_frame_store: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
578 if (slot_is_register(ctx, slot)) {
579 if (unlikely(offset != 0))
580 internal(file_line, "gen_frame_store: offset is non-zero: %"PRIdMAX"", (intmax_t)offset);
581 g(gen_mov(ctx, !reg_is_fp(reg) ? OP_SIZE_NATIVE : size, ctx->registers[slot], reg));
582 return true;
583 }
584 return gen_frame_store_raw(ctx, size, slot, offset, reg);
585 }
587 static unsigned gen_frame_target(struct codegen_context *ctx, frame_t slot_r, frame_t slot_na_1, frame_t slot_na_2, unsigned reg)
588 {
589 if (slot_is_register(ctx, slot_r)) {
590 short d = ctx->registers[slot_r];
591 if (slot_na_1 != NO_FRAME_T && slot_is_register(ctx, slot_na_1) && ctx->registers[slot_na_1] == d)
592 return reg;
593 if (slot_na_2 != NO_FRAME_T && slot_is_register(ctx, slot_na_2) && ctx->registers[slot_na_2] == d)
594 return reg;
595 return d;
596 }
597 return reg;
598 }
600 static bool attr_w gen_frame_store_2(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, unsigned reg1, unsigned reg2)
601 {
602 #if defined(ARCH_ARM64)
603 offset += (size_t)slot * slot_size;
604 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDP_STP_OFFSET, size));
605 gen_insn(INSN_STP, size, 0, 0);
606 gen_address_offset();
607 gen_one(reg1);
608 gen_one(reg2);
609 return true;
610 #endif
611 #if defined(ARCH_ARM32)
612 if (likely(!(reg1 & 1)) && likely(reg2 == reg1 + 1) && likely(cpu_test_feature(CPU_FEATURE_armv6)))
613 #elif defined(ARCH_SPARC32)
614 if (likely(!(reg2 & 1)) && likely(reg1 == reg2 + 1))
615 #elif defined(ARCH_S390)
616 if (likely(reg1 == reg2 + 1))
617 #else
618 if (0)
619 #endif
620 {
621 offset += (size_t)slot * slot_size;
622 if (UNALIGNED_TRAP) {
623 if (unlikely((offset & ((2U << size) - 1)) != 0)) {
624 offset -= (size_t)slot * slot_size;
625 goto skip_ldd;
626 }
627 }
628 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_LDP_STP_OFFSET, size));
629 gen_insn(INSN_STP, size, 0, 0);
630 gen_address_offset();
631 gen_one(reg1);
632 gen_one(reg2);
633 return true;
634 }
635 goto skip_ldd;
636 skip_ldd:
637 g(gen_frame_store(ctx, size, slot, offset + lo_word(size), reg1));
638 g(gen_frame_store(ctx, size, slot, offset + hi_word(size), reg2));
639 return true;
640 }
642 static bool attr_w gen_frame_store_imm_raw(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, int64_t imm)
643 {
644 offset += (size_t)slot * slot_size;
645 if (!ARCH_HAS_BWX)
646 size = maximum(OP_SIZE_4, size);
647 g(gen_address(ctx, R_FRAME, offset, size == OP_SIZE_1 ? IMM_PURPOSE_MVI_CLI_OFFSET : IMM_PURPOSE_STR_OFFSET, size));
648 g(gen_imm(ctx, imm, IMM_PURPOSE_STORE_VALUE, size));
649 gen_insn(INSN_MOV, size, 0, 0);
650 gen_address_offset();
651 gen_imm_offset();
652 return true;
653 }
655 static bool attr_w gen_frame_store_imm(struct codegen_context *ctx, unsigned size, frame_t slot, int64_t offset, int64_t imm)
656 {
657 ajla_assert_lo(slot >= MIN_USEABLE_SLOT && slot < function_n_variables(ctx->fn), (file_line, "gen_frame_store_imm: invalid slot: %lu >= %lu", (unsigned long)slot, (unsigned long)function_n_variables(ctx->fn)));
658 if (slot_is_register(ctx, slot)) {
659 if (unlikely(offset != 0))
660 internal(file_line, "gen_frame_store_imm: offset is non-zero: %"PRIdMAX"", (intmax_t)offset);
661 if (size == OP_SIZE_1)
662 imm = ARCH_PREFERS_SX(size) ? (int64_t)(int8_t)imm : (int64_t)(uint8_t)imm;
663 if (size == OP_SIZE_2)
664 imm = ARCH_PREFERS_SX(size) ? (int64_t)(int16_t)imm : (int64_t)(uint16_t)imm;
665 if (size == OP_SIZE_4)
666 imm = ARCH_PREFERS_SX(size) ? (int64_t)(int32_t)imm : (int64_t)(uint32_t)imm;
667 g(gen_load_constant(ctx, ctx->registers[slot], imm));
668 return true;
669 }
670 return gen_frame_store_imm_raw(ctx, size, slot, offset, imm);
671 }
673 static bool attr_w gen_frame_clear_raw(struct codegen_context *ctx, unsigned size, frame_t slot)
674 {
675 g(gen_frame_store_imm_raw(ctx, size, slot, 0, 0));
676 return true;
677 }
679 static bool attr_w gen_frame_clear(struct codegen_context *ctx, unsigned size, frame_t slot)
680 {
681 g(gen_frame_store_imm(ctx, size, slot, 0, 0));
682 if (slot_is_register(ctx, slot))
683 g(spill(ctx, slot));
684 return true;
685 }
687 #if defined(ARCH_X86)
688 static bool attr_w gen_frame_set_cond(struct codegen_context *ctx, unsigned attr_unused size, bool attr_unused logical, unsigned cond, frame_t slot)
689 {
690 size_t offset;
691 if (slot_is_register(ctx, slot)) {
692 unsigned reg = ctx->registers[slot];
693 #if defined(ARCH_X86_32)
694 if (reg >= 4) {
695 gen_insn(INSN_SET_COND_PARTIAL, OP_SIZE_1, cond, 0);
696 gen_one(R_SCRATCH_1);
697 gen_one(R_SCRATCH_1);
699 g(gen_mov(ctx, OP_SIZE_1, reg, R_SCRATCH_1));
700 return true;
701 }
702 #endif
703 gen_insn(INSN_SET_COND_PARTIAL, OP_SIZE_1, cond, 0);
704 gen_one(reg);
705 gen_one(reg);
707 if (sizeof(ajla_flat_option_t) > 1) {
708 g(gen_mov(ctx, OP_SIZE_1, reg, reg));
709 }
711 return true;
712 }
713 offset = (size_t)slot * slot_size;
714 if (sizeof(ajla_flat_option_t) > 1) {
715 gen_insn(INSN_SET_COND_PARTIAL, OP_SIZE_1, cond, 0);
716 gen_one(R_SCRATCH_1);
717 gen_one(R_SCRATCH_1);
719 g(gen_mov(ctx, OP_SIZE_1, R_SCRATCH_1, R_SCRATCH_1));
721 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot, 0, R_SCRATCH_1));
722 } else {
723 g(gen_address(ctx, R_FRAME, offset, IMM_PURPOSE_STR_OFFSET, OP_SIZE_1));
724 gen_insn(INSN_SET_COND, OP_SIZE_1, cond, 0);
725 gen_address_offset();
726 }
727 return true;
728 }
729 #elif defined(ARCH_ARM64)
730 static bool attr_w gen_frame_set_cond(struct codegen_context *ctx, unsigned attr_unused size, bool attr_unused logical, unsigned cond, frame_t slot)
731 {
732 if (slot_is_register(ctx, slot)) {
733 gen_insn(INSN_SET_COND, OP_SIZE_4, cond, 0);
734 gen_one(ctx->registers[slot]);
735 } else {
736 gen_insn(INSN_SET_COND, OP_SIZE_4, cond, 0);
737 gen_one(R_SCRATCH_1);
738 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot, 0, R_SCRATCH_1));
739 }
740 return true;
741 }
742 #elif ARCH_HAS_FLAGS
743 static bool attr_w gen_frame_set_cond(struct codegen_context *ctx, unsigned size, bool logical, unsigned cond, frame_t slot)
744 {
745 unsigned target = gen_frame_target(ctx, slot, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_1);
746 #if defined(ARCH_POWER)
747 if (!cpu_test_feature(CPU_FEATURE_v203))
748 #elif defined(ARCH_S390)
749 if (!cpu_test_feature(CPU_FEATURE_misc_45))
750 #elif defined(ARCH_SPARC32)
751 if (!SPARC_9)
752 #else
753 if (0)
754 #endif
755 {
756 uint32_t label;
757 g(gen_load_constant(ctx, target, 1));
758 label = alloc_label(ctx);
759 if (unlikely(!label))
760 return false;
761 gen_insn(!logical ? INSN_JMP_COND : INSN_JMP_COND_LOGICAL, i_size_cmp(size), cond, 0);
762 gen_four(label);
763 g(gen_load_constant(ctx, target, 0));
764 gen_label(label);
765 goto do_store;
766 }
767 g(gen_load_constant(ctx, target, 1));
768 g(gen_imm(ctx, 0, IMM_PURPOSE_CMOV, OP_SIZE_NATIVE));
769 if (cond & COND_FP) {
770 gen_insn(INSN_CMOV, OP_SIZE_NATIVE, cond ^ 1, 0);
771 } else {
772 #if defined(ARCH_S390)
773 gen_insn(logical ? INSN_CMOV_XCC : INSN_CMOV, OP_SIZE_NATIVE, cond ^ 1, 0);
774 #else
775 gen_insn(size == OP_SIZE_8 ? INSN_CMOV_XCC : INSN_CMOV, OP_SIZE_NATIVE, cond ^ 1, 0);
776 #endif
777 }
778 gen_one(target);
779 gen_one(target);
780 gen_imm_offset();
781 do_store:
782 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot, 0, target));
783 return true;
784 }
785 #endif
787 #if !ARCH_HAS_FLAGS
788 static bool attr_w gen_frame_cmp_imm_set_cond_reg(struct codegen_context *ctx, unsigned size, unsigned reg, int64_t imm, unsigned cond, frame_t slot_r)
789 {
790 unsigned dest_reg;
791 dest_reg = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_CMP_RESULT);
792 g(gen_cmp_dest_reg(ctx, size, reg, (unsigned)-1, dest_reg, imm, cond));
793 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, dest_reg));
795 return true;
796 }
797 #endif
799 static bool attr_w gen_frame_load_cmp_set_cond(struct codegen_context *ctx, unsigned size, enum extend ex, frame_t slot, unsigned reg, unsigned cond, frame_t slot_r)
800 {
801 #if ARCH_HAS_FLAGS
802 bool logical = COND_IS_LOGICAL(cond);
803 g(gen_frame_load_cmp(ctx, size, logical, ex, false, slot, 0, false, reg));
804 g(gen_frame_set_cond(ctx, size, logical, cond, slot_r));
805 #else
806 unsigned src_reg, dest_reg;
807 g(gen_frame_get(ctx, size, ex, slot, R_SCRATCH_NA_1, &src_reg));
808 dest_reg = gen_frame_target(ctx, slot_r, NO_FRAME_T, NO_FRAME_T, R_SCRATCH_NA_1);
809 g(gen_cmp_dest_reg(ctx, size, reg, src_reg, dest_reg, 0, cond));
810 g(gen_frame_store(ctx, log_2(sizeof(ajla_flat_option_t)), slot_r, 0, dest_reg));
811 #endif
812 return true;
813 }
815 static bool attr_w gen_frame_load_cmp_imm_set_cond(struct codegen_context *ctx, unsigned size, enum extend ex, frame_t slot, int64_t value, unsigned cond, frame_t slot_r)
816 {
817 #if ARCH_HAS_FLAGS
818 bool logical = COND_IS_LOGICAL(cond);
819 #if defined(ARCH_S390)
820 if (cond == COND_E)
821 logical = true;
822 #endif
823 g(gen_frame_load_cmp_imm(ctx, size, logical, ex, slot, 0, value));
824 g(gen_frame_set_cond(ctx, size, false, cond, slot_r));
825 #else
826 unsigned src_reg;
827 g(gen_frame_get(ctx, size, ex, slot, R_SCRATCH_NA_1, &src_reg));
828 g(gen_frame_cmp_imm_set_cond_reg(ctx, size, src_reg, value, cond, slot_r));
829 #endif
830 return true;
831 }
833 static bool attr_w gen_upcall_start(struct codegen_context *ctx, unsigned offset, unsigned args)
834 {
835 size_t i;
836 ajla_assert_lo(ctx->upcall_args == -1, (file_line, "gen_upcall_start: gen_upcall_end not called"));
837 ajla_assert_lo(ctx->upcall_offset == -1, (file_line, "gen_upcall_start: gen_upcall_end not called"));
838 ctx->upcall_args = (int)args;
839 ctx->upcall_offset = (int)offset;
840 ctx->upcall_hacked_abi = false;
842 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
843 if (asm_generated_upcalls && (
844 offset == offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference) ||
845 offset == offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned) ||
846 offset == offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_copy_variable_to_pointer))) {
847 ctx->upcall_hacked_abi = true;
848 }
850 for (i = 0; i < ctx->need_spill_l; i++) {
851 unsigned reg = ctx->registers[ctx->need_spill[i]];
852 if (reg_is_fp(reg))
853 g(spill(ctx, ctx->need_spill[i]));
854 }
855 ctx->n_pushes = 0;
856 for (i = 0; i < ctx->need_spill_l; i++) {
857 unsigned reg = ctx->registers[ctx->need_spill[i]];
858 if (!reg_is_fp(reg)) {
859 if (ctx->upcall_hacked_abi && !reg_is_saved(reg))
860 continue;
861 gen_insn(INSN_PUSH, OP_SIZE_8, 0, 0);
862 gen_one(reg);
863 ctx->n_pushes++;
864 }
865 }
866 if (ctx->n_pushes & 1) {
867 gen_insn(INSN_PUSH, OP_SIZE_8, 0, 0);
868 gen_one(R_AX);
869 }
870 #else
871 for (i = 0; i < ctx->need_spill_l; i++)
872 g(spill(ctx, ctx->need_spill[i]));
873 #endif
874 return true;
875 }
877 static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned offset, unsigned args)
878 {
879 size_t i;
880 ajla_assert_lo(ctx->upcall_args == (int)args, (file_line, "gen_upcall_end: gen_upcall_start args mismatch: %d != %d", ctx->upcall_args, (int)args));
881 ajla_assert_lo(ctx->upcall_offset == (int)offset, (file_line, "gen_upcall_end: gen_upcall_start offset mismatch: %d != %d", ctx->upcall_offset, (int)offset));
883 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
884 if (ctx->n_pushes & 1) {
885 gen_insn(INSN_POP, OP_SIZE_8, 0, 0);
886 gen_one(R_CX);
887 }
888 for (i = ctx->need_spill_l; i;) {
889 unsigned reg;
890 i--;
891 reg = ctx->registers[ctx->need_spill[i]];
892 if (!reg_is_fp(reg)) {
893 if (ctx->upcall_hacked_abi && !reg_is_saved(reg))
894 continue;
895 gen_insn(INSN_POP, OP_SIZE_8, 0, 0);
896 gen_one(reg);
897 }
898 }
899 for (i = 0; i < ctx->need_spill_l; i++) {
900 unsigned reg = ctx->registers[ctx->need_spill[i]];
901 if (reg_is_fp(reg))
902 g(unspill(ctx, ctx->need_spill[i]));
903 }
904 #else
905 for (i = 0; i < ctx->need_spill_l; i++)
906 g(unspill(ctx, ctx->need_spill[i]));
907 #endif
909 ctx->upcall_args = -1;
910 ctx->upcall_offset = -1;
911 return true;
912 }
914 static bool attr_w gen_memcpy_raw(struct codegen_context *ctx, unsigned dest_base, int64_t dest_offset, unsigned src_base, int64_t src_offset, size_t size, size_t attr_unused align)
915 {
916 if (!size)
917 return true;
918 if (!ARCH_HAS_BWX) {
919 if (align < 4 || (size & 3))
920 goto call_memcpy;
921 }
922 #if defined(ARCH_S390)
923 if (size <= 0x10) {
924 if (!(size & 3) || (cpu_test_feature(CPU_FEATURE_long_displacement) && cpu_test_feature(CPU_FEATURE_extended_imm)))
925 goto do_explicit_copy;
926 }
927 if (size <= 0x100 && dest_offset >= 0 && dest_offset < 0x1000 && src_offset >= 0 && src_offset < 0x1000) {
928 gen_insn(INSN_MEMCPY, 0, 0, 0);
929 gen_one(ARG_ADDRESS_1);
930 gen_one(dest_base);
931 gen_eight(dest_offset);
932 gen_one(ARG_ADDRESS_1);
933 gen_one(src_base);
934 gen_eight(src_offset);
935 gen_one(ARG_IMM);
936 gen_eight(size);
937 return true;
938 }
939 goto call_memcpy;
940 do_explicit_copy:
941 #endif
942 if (size <= INLINE_COPY_SIZE) {
943 while (size) {
944 unsigned this_step;
945 unsigned this_op_size;
946 #if defined(ARCH_ARM)
947 if (size >= 2U << OP_SIZE_NATIVE
948 #if defined(ARCH_ARM32)
949 && align >= 1U << OP_SIZE_NATIVE
950 #endif
951 ) {
952 g(gen_address(ctx, src_base, src_offset, IMM_PURPOSE_LDP_STP_OFFSET, OP_SIZE_NATIVE));
953 gen_insn(INSN_LDP, OP_SIZE_NATIVE, 0, 0);
954 gen_one(R_SCRATCH_NA_1);
955 gen_one(R_SCRATCH_NA_2);
956 gen_address_offset();
958 g(gen_address(ctx, dest_base, dest_offset, IMM_PURPOSE_LDP_STP_OFFSET, OP_SIZE_NATIVE));
959 gen_insn(INSN_STP, OP_SIZE_NATIVE, 0, 0);
960 gen_address_offset();
961 gen_one(R_SCRATCH_NA_1);
962 gen_one(R_SCRATCH_NA_2);
964 size -= 2U << OP_SIZE_NATIVE;
965 src_offset += 2U << OP_SIZE_NATIVE;
966 dest_offset += 2U << OP_SIZE_NATIVE;
968 continue;
969 }
970 #endif
971 if (size >= 8 && OP_SIZE_NATIVE >= OP_SIZE_8)
972 this_step = 8;
973 else if (size >= 4)
974 this_step = 4;
975 else if (size >= 2)
976 this_step = 2;
977 else
978 this_step = 1;
979 if (UNALIGNED_TRAP)
980 this_step = minimum(this_step, align);
981 this_op_size = log_2(this_step);
983 g(gen_address(ctx, src_base, src_offset, ARCH_PREFERS_SX(this_op_size) ? IMM_PURPOSE_LDR_SX_OFFSET : IMM_PURPOSE_LDR_OFFSET, this_op_size));
984 gen_insn(ARCH_PREFERS_SX(this_op_size) ? INSN_MOVSX : INSN_MOV, this_op_size, 0, 0);
985 gen_one(R_SCRATCH_1);
986 gen_address_offset();
988 g(gen_address(ctx, dest_base, dest_offset, IMM_PURPOSE_STR_OFFSET, this_op_size));
989 gen_insn(INSN_MOV, this_op_size, 0, 0);
990 gen_address_offset();
991 gen_one(R_SCRATCH_1);
993 size -= this_step;
994 src_offset += this_step;
995 dest_offset += this_step;
996 }
997 return true;
998 }
1000 call_memcpy:
1001 g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, mem_copy), 3));
1002 if (unlikely(R_ARG0 == src_base)) {
1003 if (unlikely(R_ARG1 == dest_base))
1004 internal(file_line, "gen_memcpy_raw: swapped registers: %u, %u", src_base, dest_base);
1005 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG1, src_base, src_offset, 0));
1006 g(gen_upcall_argument(ctx, 1));
1007 }
1009 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG0, dest_base, dest_offset, 0));
1010 g(gen_upcall_argument(ctx, 0));
1012 if (R_ARG0 != src_base) {
1013 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG1, src_base, src_offset, 0));
1014 g(gen_upcall_argument(ctx, 1));
1015 }
1017 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
1018 if (cpu_test_feature(CPU_FEATURE_erms)) {
1019 g(gen_load_constant(ctx, R_CX, size));
1021 gen_insn(INSN_MEMCPY, 0, 0, 0);
1022 gen_one(ARG_ADDRESS_1_POST_I);
1023 gen_one(R_DI);
1024 gen_eight(0);
1025 gen_one(ARG_ADDRESS_1_POST_I);
1026 gen_one(R_SI);
1027 gen_eight(0);
1028 gen_one(R_CX);
1029 g(gen_upcall_end(ctx, offsetof(struct cg_upcall_vector_s, mem_copy), 3));
1030 return true;
1031 }
1032 #endif
1034 g(gen_load_constant(ctx, R_ARG2, size));
1035 g(gen_upcall_argument(ctx, 2));
1037 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, mem_copy), 3));
1039 return true;
1040 }
1042 static bool attr_w gen_memcpy_to_slot(struct codegen_context *ctx, frame_t dest_slot, unsigned src_base, int64_t src_offset)
1043 {
1044 const struct type *t = get_type_of_local(ctx, dest_slot);
1045 unsigned size = spill_size(t);
1046 short dest_reg = ctx->registers[dest_slot];
1047 if (dest_reg >= 0) {
1048 if (ARCH_PREFERS_SX(size) && !reg_is_fp(dest_reg)) {
1049 #if defined(ARCH_S390)
1050 if (size == OP_SIZE_1 && !cpu_test_feature(CPU_FEATURE_long_displacement)) {
1051 g(gen_address(ctx, src_base, src_offset, IMM_PURPOSE_LDR_OFFSET, size));
1052 gen_insn(INSN_MOV_MASK, OP_SIZE_NATIVE, MOV_MASK_0_8, 0);
1053 gen_one(dest_reg);
1054 gen_one(dest_reg);
1055 gen_address_offset();
1056 g(gen_extend(ctx, size, sign_x, dest_reg, dest_reg));
1057 return true;
1058 }
1059 #endif
1060 g(gen_address(ctx, src_base, src_offset, IMM_PURPOSE_LDR_SX_OFFSET, size));
1061 gen_insn(INSN_MOVSX, size, 0, 0);
1062 } else {
1063 g(gen_address(ctx, src_base, src_offset, reg_is_fp(dest_reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_LDR_OFFSET, size));
1064 gen_insn(INSN_MOV, size, 0, 0);
1065 }
1066 gen_one(dest_reg);
1067 gen_address_offset();
1068 return true;
1069 }
1070 g(gen_memcpy_raw(ctx, R_FRAME, (size_t)dest_slot * slot_size, src_base, src_offset, t->size, t->align));
1071 return true;
1072 }
1074 static bool attr_w gen_memcpy_from_slot(struct codegen_context *ctx, unsigned dest_base, int64_t dest_offset, frame_t src_slot)
1075 {
1076 const struct type *t = get_type_of_local(ctx, src_slot);
1077 short src_reg = ctx->registers[src_slot];
1078 if (src_reg >= 0) {
1079 unsigned size = spill_size(t);
1080 g(gen_address(ctx, dest_base, dest_offset, reg_is_fp(src_reg) ? IMM_PURPOSE_VLDR_VSTR_OFFSET : IMM_PURPOSE_STR_OFFSET, size));
1081 gen_insn(INSN_MOV, size, 0, 0);
1082 gen_address_offset();
1083 gen_one(src_reg);
1084 return true;
1085 }
1086 g(gen_memcpy_raw(ctx, dest_base, dest_offset, R_FRAME, (size_t)src_slot * slot_size, t->size, t->align));
1087 return true;
1088 }
1090 static bool attr_w gen_memcpy_slots(struct codegen_context *ctx, frame_t dest_slot, frame_t src_slot)
1091 {
1092 const struct type *t = get_type_of_local(ctx, src_slot);
1093 short dest_reg = ctx->registers[dest_slot];
1094 short src_reg = ctx->registers[src_slot];
1095 if (dest_reg >= 0 && src_reg >= 0) {
1096 unsigned size = spill_size(t);
1097 g(gen_mov(ctx, reg_is_fp(src_reg) ? size : OP_SIZE_NATIVE, dest_reg, src_reg));
1098 return true;
1099 }
1100 if (dest_reg >= 0) {
1101 unsigned size = spill_size(t);
1102 g(gen_frame_load(ctx, size, garbage, src_slot, 0, false, dest_reg));
1103 return true;
1104 }
1105 if (src_reg >= 0) {
1106 unsigned size = spill_size(t);
1107 g(gen_frame_store(ctx, size, dest_slot, 0, src_reg));
1108 return true;
1109 }
1110 g(gen_memcpy_raw(ctx, R_FRAME, (size_t)dest_slot * slot_size, R_FRAME, (size_t)src_slot * slot_size, t->size, maximum(slot_size, t->align)));
1111 return true;
1112 }
1114 static bool attr_w gen_clear_bitmap(struct codegen_context *ctx, unsigned additional_offset, unsigned dest_base, int64_t dest_offset, frame_t bitmap_slots)
1115 {
1116 if (bitmap_slots <= INLINE_BITMAP_SLOTS) {
1117 bool attr_unused scratch_2_zeroed = false;
1118 size_t bitmap_length = (size_t)bitmap_slots * slot_size;
1119 size_t clear_offset = 0;
1120 additional_offset += (unsigned)dest_offset;
1121 #if defined(ARCH_X86)
1122 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_XOR, R_SCRATCH_1, R_SCRATCH_1, R_SCRATCH_1, 0));
1123 #endif
1124 #if defined(ARCH_ARM32) || defined(ARCH_S390)
1125 g(gen_load_constant(ctx, R_SCRATCH_1, 0));
1126 #endif
1127 while (clear_offset < bitmap_length) {
1128 size_t len = bitmap_length - clear_offset;
1129 if (len > frame_align)
1130 len = frame_align;
1131 if (additional_offset)
1132 len = minimum(len, additional_offset & -additional_offset);
1133 #if defined(ARCH_ARM32) || defined(ARCH_S390)
1134 len = minimum(len, 2U << OP_SIZE_NATIVE);
1135 if (len == 2U << OP_SIZE_NATIVE) {
1136 if (!scratch_2_zeroed) {
1137 g(gen_load_constant(ctx, R_SCRATCH_2, 0));
1138 scratch_2_zeroed = true;
1139 }
1140 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_LDP_STP_OFFSET, OP_SIZE_NATIVE));
1141 gen_insn(INSN_STP, OP_SIZE_NATIVE, 0, 0);
1142 gen_address_offset();
1143 gen_one(R_SCRATCH_1);
1144 gen_one(R_SCRATCH_2);
1145 goto next_loop;
1146 }
1147 #elif defined(ARCH_ARM64)
1148 len = minimum(len, 1U << OP_SIZE_16);
1149 if (len == 1U << OP_SIZE_16) {
1150 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_LDP_STP_OFFSET, OP_SIZE_8));
1151 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, OP_SIZE_8));
1152 gen_insn(INSN_STP, OP_SIZE_NATIVE, 0, 0);
1153 gen_address_offset();
1154 gen_imm_offset();
1155 gen_imm_offset();
1156 goto next_loop;
1157 }
1158 #elif defined(ARCH_X86)
1159 len = minimum(len, 1U << OP_SIZE_16);
1160 if (len == 1U << OP_SIZE_16 && cpu_test_feature(CPU_FEATURE_sse)) {
1161 if (!scratch_2_zeroed) {
1162 g(gen_3address_alu(ctx, OP_SIZE_16, ALU_XOR, R_XMM0, R_XMM0, R_XMM0, 0));
1163 scratch_2_zeroed = true;
1164 }
1165 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_VLDR_VSTR_OFFSET, OP_SIZE_16));
1166 gen_insn(INSN_MOV, OP_SIZE_16, 0, 0);
1167 gen_address_offset();
1168 gen_one(R_XMM0);
1169 goto next_loop;
1170 }
1171 #endif
1172 len = minimum(len, 1U << OP_SIZE_NATIVE);
1173 len = (size_t)1 << high_bit(len);
1174 #if defined(ARCH_X86) || defined(ARCH_ARM32) || defined(ARCH_S390)
1175 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_STR_OFFSET, log_2(len)));
1176 gen_insn(INSN_MOV, log_2(len), 0, 0);
1177 gen_address_offset();
1178 gen_one(R_SCRATCH_1);
1179 #else
1180 g(gen_address(ctx, dest_base, dest_offset + clear_offset, IMM_PURPOSE_STR_OFFSET, log_2(len)));
1181 g(gen_imm(ctx, 0, IMM_PURPOSE_STORE_VALUE, log_2(len)));
1182 gen_insn(INSN_MOV, log_2(len), 0, 0);
1183 gen_address_offset();
1184 gen_imm_offset();
1185 #endif
1186 goto next_loop;
1187 next_loop:
1188 clear_offset += len;
1189 additional_offset += len;
1190 }
1191 return true;
1192 }
1193 #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI)
1194 if (cpu_test_feature(CPU_FEATURE_erms)) {
1195 gen_insn(INSN_PUSH, OP_SIZE_8, 0, 0);
1196 gen_one(R_DI);
1198 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_DI, dest_base, dest_offset, 0));
1200 g(gen_load_constant(ctx, R_CX, (size_t)bitmap_slots * slot_size));
1202 g(gen_3address_alu(ctx, OP_SIZE_4, ALU_XOR, R_AX, R_AX, R_AX, 0));
1204 gen_insn(INSN_MEMSET, 0, 0, 0);
1205 gen_one(ARG_ADDRESS_1_POST_I);
1206 gen_one(R_DI);
1207 gen_eight(0);
1208 gen_one(R_CX);
1209 gen_one(R_AX);
1211 gen_insn(INSN_POP, OP_SIZE_8, 0, 0);
1212 gen_one(R_DI);
1214 return true;
1215 }
1216 #endif
1217 g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, mem_clear), 2));
1219 g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG0, dest_base, dest_offset, 0));
1220 g(gen_upcall_argument(ctx, 0));
1222 g(gen_load_constant(ctx, R_ARG1, (size_t)bitmap_slots * slot_size));
1223 g(gen_upcall_argument(ctx, 1));
1225 g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, mem_clear), 2));
1227 return true;
1228 }
1230 static bool attr_w load_function_offset(struct codegen_context *ctx, unsigned dest, size_t fn_offset)
1231 {
1232 g(gen_frame_load_raw(ctx, OP_SIZE_ADDRESS, zero_x, 0, frame_offs(function), false, dest));
1234 g(gen_address(ctx, dest, fn_offset, IMM_PURPOSE_LDR_OFFSET, OP_SIZE_ADDRESS));
1235 gen_insn(INSN_MOV, OP_SIZE_ADDRESS, 0, 0);
1236 gen_one(dest);
1237 gen_address_offset();
1239 return true;
1240 }