From 204d7f813eb610bc66274a361f7c532b3578663f Mon Sep 17 00:00:00 2001 From: Mikulas Patocka Date: Tue, 19 Nov 2024 20:35:33 +0100 Subject: [PATCH] x86-64: remove pushes and pops arounf pointer_dereference and pointer_reference_owned --- c1-alpha.inc | 2 +- c1-arm.inc | 2 +- c1-arm64.inc | 2 +- c1-hppa.inc | 2 +- c1-ia64.inc | 2 +- c1-loong.inc | 2 +- c1-mips.inc | 2 +- c1-power.inc | 2 +- c1-riscv.inc | 2 +- c1-s390.inc | 2 +- c1-sparc.inc | 2 +- c1-x86.inc | 2 +- cg-alu.inc | 17 +++++++++++------ cg-frame.inc | 46 +++++++++++++++++++++++++++------------------- cg-ops.inc | 56 ++++++++++++++++++++++++++++---------------------------- cg-ptr.inc | 6 +++--- codegen.c | 12 ++++++++---- 17 files changed, 89 insertions(+), 72 deletions(-) diff --git a/c1-alpha.inc b/c1-alpha.inc index 28f1deb..4ed9be3 100644 --- a/c1-alpha.inc +++ b/c1-alpha.inc @@ -479,7 +479,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_ADDRESS, 0, 0); gen_one(R_T12); - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-arm.inc b/c1-arm.inc index 3c2f48b..8f94a24 100644 --- a/c1-arm.inc +++ b/c1-arm.inc @@ -403,7 +403,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_4, 0, 0); gen_one(R_SCRATCH_NA_1); - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-arm64.inc b/c1-arm64.inc index c1a1c13..2c25463 100644 --- a/c1-arm64.inc +++ b/c1-arm64.inc @@ -507,7 +507,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_8, 0, 0); gen_one(R_SCRATCH_NA_1); - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-hppa.inc b/c1-hppa.inc index 9599959..28ef475 100644 --- a/c1-hppa.inc +++ b/c1-hppa.inc @@ -511,7 +511,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_8, 0, 0); gen_one(R_DP); #endif - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-ia64.inc b/c1-ia64.inc index 77c4d0b..4ebed30 100644 --- a/c1-ia64.inc +++ b/c1-ia64.inc @@ -581,7 +581,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_8, 0, 0); gen_one(R_SCRATCH_B); - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-loong.inc b/c1-loong.inc index 88ca894..a67c3ab 100644 --- a/c1-loong.inc +++ b/c1-loong.inc @@ -430,7 +430,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_ADDRESS, 0, 0); gen_one(R_SCRATCH_NA_1); - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-mips.inc b/c1-mips.inc index c5e8ad1..a4faf4c 100644 --- a/c1-mips.inc +++ b/c1-mips.inc @@ -580,7 +580,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_ADDRESS, 0, 0); gen_one(R_T9); - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-power.inc b/c1-power.inc index 2fe8c99..3c60764 100644 --- a/c1-power.inc +++ b/c1-power.inc @@ -592,7 +592,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_NATIVE, 0, 0); gen_one(R_CTR); #endif - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-riscv.inc b/c1-riscv.inc index 876d040..091f651 100644 --- a/c1-riscv.inc +++ b/c1-riscv.inc @@ -523,7 +523,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_ADDRESS, 0, 0); gen_one(R_SCRATCH_NA_1); - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-s390.inc b/c1-s390.inc index ddda36e..f021ce8 100644 --- a/c1-s390.inc +++ b/c1-s390.inc @@ -439,7 +439,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_ADDRESS, 0, 0); gen_one(R_OFFSET_IMM); - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-sparc.inc b/c1-sparc.inc index b9ead55..1be2ffb 100644 --- a/c1-sparc.inc +++ b/c1-sparc.inc @@ -433,7 +433,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_insn(INSN_CALL_INDIRECT, OP_SIZE_NATIVE, 0, 0); gen_one(R_SCRATCH_NA_1); - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/c1-x86.inc b/c1-x86.inc index 82cbc01..74f6a90 100644 --- a/c1-x86.inc +++ b/c1-x86.inc @@ -615,7 +615,7 @@ static bool attr_w gen_upcall(struct codegen_context *ctx, unsigned offset, unsi gen_one(R_UPCALL); gen_eight(offset); #endif - g(gen_upcall_end(ctx, n_args)); + g(gen_upcall_end(ctx, offset, n_args)); return true; } diff --git a/cg-alu.inc b/cg-alu.inc index ea45da4..54388ac 100644 --- a/cg-alu.inc +++ b/cg-alu.inc @@ -24,26 +24,31 @@ static bool attr_w gen_alu_upcall(struct codegen_context *ctx, size_t upcall, un g(spill(ctx, slot_1)); if (slot_2 != NO_FRAME_T && slot_is_register(ctx, slot_2)) g(spill(ctx, slot_2)); - g(gen_upcall_start(ctx, frame_t_is_const(slot_2) ? 4 : slot_2 != NO_FRAME_T ? 3 : 2)); - g(gen_frame_address(ctx, slot_1, 0, R_ARG0)); - g(gen_upcall_argument(ctx, 0)); if (frame_t_is_const(slot_2)) { - size_t x_offs; + size_t x_offs = offsetof(struct cg_upcall_vector_s, INT_binary_const_int8_t) + op_size * sizeof(void (*)(void)); + g(gen_upcall_start(ctx, x_offs, 4)); + g(gen_frame_address(ctx, slot_1, 0, R_ARG0)); + g(gen_upcall_argument(ctx, 0)); g(gen_load_constant(ctx, R_ARG1, frame_t_get_const(slot_2))); g(gen_upcall_argument(ctx, 1)); g(gen_frame_address(ctx, slot_r, 0, R_ARG2)); g(gen_upcall_argument(ctx, 2)); g(gen_get_upcall_pointer(ctx, upcall, R_ARG3)); g(gen_upcall_argument(ctx, 3)); - x_offs = offsetof(struct cg_upcall_vector_s, INT_binary_const_int8_t) + op_size * sizeof(void (*)(void)); g(gen_upcall(ctx, x_offs, 4)); } else if (slot_2 != NO_FRAME_T) { + g(gen_upcall_start(ctx, upcall, 3)); + g(gen_frame_address(ctx, slot_1, 0, R_ARG0)); + g(gen_upcall_argument(ctx, 0)); g(gen_frame_address(ctx, slot_2, 0, R_ARG1)); g(gen_upcall_argument(ctx, 1)); g(gen_frame_address(ctx, slot_r, 0, R_ARG2)); g(gen_upcall_argument(ctx, 2)); g(gen_upcall(ctx, upcall, 3)); } else { + g(gen_upcall_start(ctx, upcall, 2)); + g(gen_frame_address(ctx, slot_1, 0, R_ARG0)); + g(gen_upcall_argument(ctx, 0)); g(gen_frame_address(ctx, slot_r, 0, R_ARG1)); g(gen_upcall_argument(ctx, 1)); g(gen_upcall(ctx, upcall, 2)); @@ -3637,7 +3642,7 @@ static bool attr_w gen_system_property(struct codegen_context *ctx, frame_t slot g(gen_test_1_cached(ctx, slot_1, escape_label)); - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_system_property), 1)); g(gen_frame_load(ctx, OP_SIZE_INT, garbage, slot_1, 0, false, R_ARG0)); g(gen_upcall_argument(ctx, 0)); diff --git a/cg-frame.inc b/cg-frame.inc index 3e329ec..83c62f4 100644 --- a/cg-frame.inc +++ b/cg-frame.inc @@ -830,29 +830,39 @@ static bool attr_w gen_frame_load_cmp_imm_set_cond(struct codegen_context *ctx, return true; } -static bool attr_w gen_upcall_start(struct codegen_context *ctx, unsigned args) +static bool attr_w gen_upcall_start(struct codegen_context *ctx, unsigned offset, unsigned args) { size_t i; - size_t attr_unused n_pushes; ajla_assert_lo(ctx->upcall_args == -1, (file_line, "gen_upcall_start: gen_upcall_end not called")); + ajla_assert_lo(ctx->upcall_offset == -1, (file_line, "gen_upcall_start: gen_upcall_end not called")); ctx->upcall_args = (int)args; + ctx->upcall_offset = (int)offset; + ctx->upcall_hacked_abi = false; #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI) + if (asm_generated_upcalls && ( + offset == offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference) || + offset == offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned))) { + ctx->upcall_hacked_abi = true; + } + for (i = 0; i < ctx->need_spill_l; i++) { unsigned reg = ctx->registers[ctx->need_spill[i]]; if (reg_is_fp(reg)) g(spill(ctx, ctx->need_spill[i])); } - n_pushes = 0; + ctx->n_pushes = 0; for (i = 0; i < ctx->need_spill_l; i++) { unsigned reg = ctx->registers[ctx->need_spill[i]]; if (!reg_is_fp(reg)) { + if (ctx->upcall_hacked_abi && reg != R_DI && !reg_is_saved(reg)) + continue; gen_insn(INSN_PUSH, OP_SIZE_8, 0, 0); gen_one(reg); - n_pushes++; + ctx->n_pushes++; } } - if (n_pushes & 1) { + if (ctx->n_pushes & 1) { gen_insn(INSN_PUSH, OP_SIZE_8, 0, 0); gen_one(R_AX); } @@ -863,21 +873,14 @@ static bool attr_w gen_upcall_start(struct codegen_context *ctx, unsigned args) return true; } -static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args) +static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned offset, unsigned args) { size_t i; - size_t attr_unused n_pushes; - ajla_assert_lo(ctx->upcall_args == (int)args, (file_line, "gen_upcall_end: gen_upcall_start mismatch: %d", ctx->upcall_args)); - ctx->upcall_args = -1; + ajla_assert_lo(ctx->upcall_args == (int)args, (file_line, "gen_upcall_end: gen_upcall_start args mismatch: %d != %d", ctx->upcall_args, (int)args)); + ajla_assert_lo(ctx->upcall_offset == (int)offset, (file_line, "gen_upcall_end: gen_upcall_start offset mismatch: %d != %d", ctx->upcall_offset, (int)offset)); #if (defined(ARCH_X86_64) || defined(ARCH_X86_X32)) && !defined(ARCH_X86_WIN_ABI) - n_pushes = 0; - for (i = 0; i < ctx->need_spill_l; i++) { - unsigned reg = ctx->registers[ctx->need_spill[i]]; - if (!reg_is_fp(reg)) - n_pushes++; - } - if (n_pushes & 1) { + if (ctx->n_pushes & 1) { gen_insn(INSN_POP, OP_SIZE_8, 0, 0); gen_one(R_CX); } @@ -886,6 +889,8 @@ static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args) i--; reg = ctx->registers[ctx->need_spill[i]]; if (!reg_is_fp(reg)) { + if (ctx->upcall_hacked_abi && reg != R_DI && !reg_is_saved(reg)) + continue; gen_insn(INSN_POP, OP_SIZE_8, 0, 0); gen_one(reg); } @@ -899,6 +904,9 @@ static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args) for (i = 0; i < ctx->need_spill_l; i++) g(unspill(ctx, ctx->need_spill[i])); #endif + + ctx->upcall_args = -1; + ctx->upcall_offset = -1; return true; } @@ -989,7 +997,7 @@ do_explicit_copy: } call_memcpy: - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, mem_copy), 3)); if (unlikely(R_ARG0 == src_base)) { if (unlikely(R_ARG1 == dest_base)) internal(file_line, "gen_memcpy_raw: swapped registers: %u, %u", src_base, dest_base); @@ -1017,7 +1025,7 @@ call_memcpy: gen_one(R_SI); gen_eight(0); gen_one(R_CX); - g(gen_upcall_end(ctx, 3)); + g(gen_upcall_end(ctx, offsetof(struct cg_upcall_vector_s, mem_copy), 3)); return true; } #endif @@ -1205,7 +1213,7 @@ next_loop: return true; } #endif - g(gen_upcall_start(ctx, 2)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, mem_clear), 2)); g(gen_3address_alu_imm(ctx, i_size(OP_SIZE_ADDRESS), ALU_ADD, R_ARG0, dest_base, dest_offset, 0)); g(gen_upcall_argument(ctx, 0)); diff --git a/cg-ops.inc b/cg-ops.inc index 280c365..dd5997a 100644 --- a/cg-ops.inc +++ b/cg-ops.inc @@ -58,7 +58,7 @@ static bool attr_w gen_ref_move_copy(struct codegen_context *ctx, code_t code, f g(gen_set_1(ctx, R_FRAME, slot_r, 0, true)); flag_set(ctx, slot_r, true); if (code == OPCODE_REF_COPY) { - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, reg)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); @@ -77,7 +77,7 @@ static bool attr_w gen_ref_move_copy(struct codegen_context *ctx, code_t code, f goto do_reference; g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_CLEAR)); do_reference: - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, reg)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); @@ -104,7 +104,7 @@ static bool attr_w gen_box_move_copy(struct codegen_context *ctx, code_t code, f if (slot_is_register(ctx, slot_1)) g(spill(ctx, slot_1)); - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_copy_variable_to_pointer), 3)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -223,7 +223,7 @@ static bool attr_w gen_load_fn_or_curry(struct codegen_context *ctx, frame_t fn_ if (unlikely(!escape_label)) return false; - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_function_reference_mayfail), 1)); g(gen_load_constant(ctx, R_ARG0, ctx->args_l)); g(gen_upcall_argument(ctx, 0)); @@ -326,7 +326,7 @@ copied: if (slot_is_register(ctx, arg_slot)) g(spill(ctx, arg_slot)); - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -445,7 +445,7 @@ static bool attr_w gen_call(struct codegen_context *ctx, code_t code, frame_t fn if (dest_arg->may_be_flat) { g(gen_memcpy_from_slot(ctx, R_FRAME, new_fp_offset + (size_t)dest_arg->slot * slot_size, src_arg->slot)); } else { - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -507,7 +507,7 @@ static bool attr_w gen_call(struct codegen_context *ctx, code_t code, frame_t fn do_reference: gen_label(incr_ref_label); - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, reg)); g(gen_upcall_argument(ctx, 0)); @@ -690,7 +690,7 @@ static bool attr_w gen_return(struct codegen_context *ctx) if (slot_is_register(ctx, src_arg->slot)) g(spill(ctx, src_arg->slot)); - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -713,13 +713,13 @@ static bool attr_w gen_return(struct codegen_context *ctx) gen_label(copy_ptr_label); if (unlikely(!(src_arg->flags & OPCODE_FLAG_FREE_ARGUMENT))) { - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); g(gen_frame_load_slot(ctx, src_arg->slot, R_ARG0)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); } else if (da(ctx->fn,function)->local_variables_flags[src_arg->slot].may_be_borrowed) { g(gen_test_1_cached(ctx, src_arg->slot, load_write_ptr_label)); - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); g(gen_frame_load_slot(ctx, src_arg->slot, R_ARG0)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); @@ -1035,7 +1035,7 @@ struct_zero: g(gen_jmp_on_zero(ctx, OP_SIZE_SLOT, R_SCRATCH_1, COND_E, skip_deref_label)); - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_SCRATCH_1)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1)); @@ -1091,7 +1091,7 @@ static bool attr_w gen_record_create(struct codegen_context *ctx, frame_t slot_r def = type_def(t,record); - g(gen_upcall_start(ctx, 2)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_record_mayfail), 2)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -1140,7 +1140,7 @@ static bool attr_w gen_record_create(struct codegen_context *ctx, frame_t slot_r if (slot_is_register(ctx, var_slot)) g(spill(ctx, var_slot)); - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -1302,7 +1302,7 @@ static bool attr_w gen_option_create(struct codegen_context *ctx, ajla_option_t type = get_type_of_local(ctx, slot_1); - g(gen_upcall_start(ctx, 0)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_option_mayfail), 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_option_mayfail), 0)); g(gen_sanitize_returned_pointer(ctx, R_RET0)); g(gen_jmp_on_zero(ctx, OP_SIZE_ADDRESS, R_RET0, COND_E, escape_label)); @@ -1321,7 +1321,7 @@ static bool attr_w gen_option_create(struct codegen_context *ctx, ajla_option_t if (slot_is_register(ctx, slot_1)) g(spill(ctx, slot_1)); - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -1566,7 +1566,7 @@ static bool attr_w gen_array_create(struct codegen_context *ctx, frame_t slot_r) flag_set(ctx, ctx->args[i].slot, false); } - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_flat_slot_mayfail), 3)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -1590,7 +1590,7 @@ static bool attr_w gen_array_create(struct codegen_context *ctx, frame_t slot_r) } } else { int64_t offset; - g(gen_upcall_start(ctx, 2)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_pointers_mayfail), 2)); g(gen_load_constant(ctx, R_ARG0, ctx->args_l)); g(gen_upcall_argument(ctx, 0)); @@ -1632,7 +1632,7 @@ static bool attr_w gen_array_create_empty_flat(struct codegen_context *ctx, fram if (unlikely(!escape_label)) return false; - g(gen_upcall_start(ctx, 4)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_flat_types_ptr_mayfail), 4)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -1664,7 +1664,7 @@ static bool attr_w gen_array_create_empty(struct codegen_context *ctx, frame_t s if (unlikely(!escape_label)) return false; - g(gen_upcall_start(ctx, 2)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_pointers_mayfail), 2)); g(gen_load_constant(ctx, R_ARG0, ARRAY_PREALLOC_SIZE)); g(gen_upcall_argument(ctx, 0)); @@ -1734,7 +1734,7 @@ static bool attr_w gen_array_fill(struct codegen_context *ctx, frame_t slot_1, f if (TYPE_IS_FLAT(content_type)) { g(gen_test_1_cached(ctx, slot_1, get_ptr_label)); - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_flat_to_data), 3)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); @@ -1762,7 +1762,7 @@ static bool attr_w gen_array_fill(struct codegen_context *ctx, frame_t slot_1, f g(gen_frame_get(ctx, OP_SIZE_INT, sign_x, slot_2, R_SCRATCH_1, ®1)); g(gen_jmp_if_negative(ctx, reg1, escape_label)); - g(gen_upcall_start(ctx, 2)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_create_sparse), 2)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, reg1)); g(gen_upcall_argument(ctx, 0)); @@ -1777,7 +1777,7 @@ static bool attr_w gen_array_fill(struct codegen_context *ctx, frame_t slot_1, f g(gen_frame_get(ctx, OP_SIZE_INT, sign_x, slot_2, R_SCRATCH_4, ®4)); g(gen_jmp_if_negative(ctx, reg4, escape_label)); - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_create_flat), 3)); g(gen_mov(ctx, i_size(OP_SIZE_INT), R_ARG1, reg4)); g(gen_upcall_argument(ctx, 1)); @@ -1794,7 +1794,7 @@ static bool attr_w gen_array_fill(struct codegen_context *ctx, frame_t slot_1, f g(gen_frame_get_pointer(ctx, slot_1, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_SCRATCH_1)); - g(gen_upcall_start(ctx, 4)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_create_pointers), 4)); g(gen_mov(ctx, i_size(OP_SIZE_SLOT), R_ARG3, R_SCRATCH_1)); g(gen_upcall_argument(ctx, 3)); @@ -1825,7 +1825,7 @@ static bool attr_w gen_array_string(struct codegen_context *ctx, type_tag_t tag, if (unlikely(!escape_label)) return false; - g(gen_upcall_start(ctx, 2)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_data_alloc_array_flat_tag_mayfail), 2)); g(gen_load_constant(ctx, R_ARG0, tag)); g(gen_upcall_argument(ctx, 0)); @@ -2403,7 +2403,7 @@ static bool attr_w gen_array_sub(struct codegen_context *ctx, frame_t slot_array if (slot_is_register(ctx, slot_to)) g(spill(ctx, slot_to)); - g(gen_upcall_start(ctx, 4)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_sub), 4)); g(gen_frame_load_raw(ctx, OP_SIZE_SLOT, garbage, slot_array, 0, false, R_ARG0)); g(gen_upcall_argument(ctx, 0)); @@ -2465,7 +2465,7 @@ static bool attr_w gen_array_skip(struct codegen_context *ctx, frame_t slot_arra if (slot_is_register(ctx, slot_from)) g(spill(ctx, slot_from)); - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_skip), 3)); g(gen_frame_load_raw(ctx, OP_SIZE_SLOT, garbage, slot_array, 0, false, R_ARG0)); g(gen_upcall_argument(ctx, 0)); @@ -2523,7 +2523,7 @@ static bool attr_w gen_array_append(struct codegen_context *ctx, frame_t slot_1, g(gen_frame_get_pointer(ctx, slot_2, (flags & OPCODE_FLAG_FREE_ARGUMENT_2) != 0, R_SAVED_1)); g(gen_frame_get_pointer(ctx, slot_1, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0, R_SCRATCH_1)); - g(gen_upcall_start(ctx, 2)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_array_join), 2)); g(gen_mov(ctx, i_size(OP_SIZE_SLOT), R_ARG0, R_SCRATCH_1)); g(gen_upcall_argument(ctx, 0)); g(gen_mov(ctx, i_size(OP_SIZE_SLOT), R_ARG1, R_SAVED_1)); @@ -2684,7 +2684,7 @@ static bool attr_w gen_io(struct codegen_context *ctx, frame_t code, frame_t slo /*gen_insn(INSN_JMP, 0, 0, 0); gen_four(alloc_escape_label(ctx));*/ - g(gen_upcall_start(ctx, 3)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_ipret_io), 3)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, R_FRAME)); g(gen_upcall_argument(ctx, 0)); diff --git a/cg-ptr.inc b/cg-ptr.inc index 3032c7d..091a52a 100644 --- a/cg-ptr.inc +++ b/cg-ptr.inc @@ -300,7 +300,7 @@ static bool attr_w gen_compress_pointer(struct codegen_context attr_unused *ctx, static bool attr_w gen_frame_get_pointer(struct codegen_context *ctx, frame_t slot, bool deref, unsigned dest) { if (!deref) { - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot, 0, false, R_ARG0)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); @@ -322,7 +322,7 @@ static bool attr_w gen_frame_get_pointer(struct codegen_context *ctx, frame_t sl goto do_reference; g(gen_test_1(ctx, R_FRAME, slot, 0, skip_label, false, TEST_CLEAR)); do_reference: - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot, 0, false, R_ARG0)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); @@ -347,7 +347,7 @@ static bool attr_w gen_frame_set_pointer(struct codegen_context *ctx, frame_t sl if (reference) { if (escape_on_thunk) g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_SAVED_1, src)); - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); g(gen_mov(ctx, i_size(OP_SIZE_ADDRESS), R_ARG0, src)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); diff --git a/codegen.c b/codegen.c index 5b0447d..de4e7db 100644 --- a/codegen.c +++ b/codegen.c @@ -498,7 +498,10 @@ struct codegen_context { struct data *codegen; - int upcall_args; + int upcall_offset; + int8_t upcall_args; + uint8_t n_pushes; + bool upcall_hacked_abi; frame_t *var_aux; ajla_error_t err; @@ -530,6 +533,7 @@ static void init_ctx(struct codegen_context *ctx) ctx->need_spill = NULL; ctx->codegen = NULL; ctx->upcall_args = -1; + ctx->upcall_offset = -1; ctx->var_aux = NULL; } @@ -969,7 +973,7 @@ static unsigned alu_trap_purpose(unsigned alu) static bool attr_w gen_imm(struct codegen_context *ctx, int64_t imm, unsigned purpose, unsigned size); -static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned args); +static bool attr_w gen_upcall_end(struct codegen_context *ctx, unsigned offset, unsigned args); #define gen_address_offset() \ do { \ @@ -1636,7 +1640,7 @@ unconditional_escape: } g(gen_test_1(ctx, R_FRAME, slot_1, 0, label_id, false, TEST_SET)); do_take_borrowed: - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, false, R_ARG0)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_reference_owned), 1)); @@ -1662,7 +1666,7 @@ take_borrowed_done: g(gen_set_1(ctx, R_FRAME, slot_1, 0, false)); label_id = 0; /* avoid warning */ } - g(gen_upcall_start(ctx, 1)); + g(gen_upcall_start(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1)); g(gen_frame_load(ctx, OP_SIZE_SLOT, garbage, slot_1, 0, false, R_ARG0)); g(gen_upcall_argument(ctx, 0)); g(gen_upcall(ctx, offsetof(struct cg_upcall_vector_s, cg_upcall_pointer_dereference), 1)); -- 2.11.4.GIT