| blob | f21d244173107d0f15912def45d38bdf2265b4f8 |
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
19 #if ARG_MODE == 0
20 #define get_param(ip, n) (!((n) & 1) ? get_lo((ip) + (n) / 2) : get_hi((ip) + (n) / 2))
21 #define param_size(n) (((n) + 1) / 2)
22 #elif ARG_MODE == 1
23 /* A cast to uintptr_t because of gcc optimizer inefficiency */
24 #define get_param(ip, n) ((uintptr_t)(ip)[n])
25 #define param_size(n) (n)
26 #elif ARG_MODE == 2
27 #define get_param(ip, n) get_unaligned_32(&(ip)[(n) * 2])
28 #define param_size(n) ((n) * 2)
29 #endif
31 #define get_i_param(n) get_param(ip + 1, n)
32 #define ADVANCE_I_PARAM(n) ADVANCE_IP(1 + param_size(n))
35 #ifdef EMIT_FUNCTIONS
36 static const code_t attr_cold *cat(free_parameters_,ARG_MODE)(frame_s *fp, const code_t *ip, frame_t n_entries)
37 {
38 while (n_entries--) {
39 if (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT)
40 frame_free_and_clear(fp, get_param(ip, 0));
41 ADVANCE_IP(param_size(2));
42 }
43 return ip;
44 }
45 #endif
48 #define move_scalar(utype) \
49 do { \
50 frame_t slot_1 = get_i_param(0); \
51 frame_t slot_r = get_i_param(1); \
52 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "scalar move: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
53 barrier_aliasing(); \
54 if (unlikely(frame_test_and_clear_flag(fp, slot_1))) { \
55 frame_set_pointer(fp, slot_r, *frame_pointer(fp, slot_1));\
56 pointer_poison(frame_pointer(fp, slot_1)); \
57 } else { \
58 *frame_slot(fp, slot_r, utype) = \
59 *frame_slot(fp, slot_1, utype); \
60 } \
61 barrier_aliasing(); \
62 ADVANCE_I_PARAM(2); \
63 } while (0)
65 #define copy_scalar(utype) \
66 do { \
67 frame_t slot_1 = get_i_param(0); \
68 frame_t slot_r = get_i_param(1); \
69 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "scalar copy: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
70 barrier_aliasing(); \
71 if (unlikely(frame_test_flag(fp, slot_1))) { \
72 pointer_t ptr = *frame_pointer(fp, slot_1); \
73 frame_set_pointer(fp, slot_r, ptr); \
74 pointer_reference_owned(ptr); \
75 } else { \
76 *frame_slot(fp, slot_r, utype) = \
77 *frame_slot(fp, slot_1, utype); \
78 } \
79 barrier_aliasing(); \
80 ADVANCE_I_PARAM(2); \
81 } while (0)
84 #define DEFINE_FIXED_REAL_BINARY_OPCODE(tc, type, op) \
85 DEFINE_OPCODE( \
86 cat3(OPCODE_,tc,OP) + \
87 cat4(OPCODE_,tc,OP_,op) * cat3(OPCODE_,tc,OP_MULT) + \
88 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
89 cat7(tc,binary_,op,_,type,_,ARG_MODE), \
90 frame_t slot_1 = get_i_param(0); \
91 frame_t slot_2 = get_i_param(1); \
92 frame_t slot_r = get_i_param(2); \
93 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "fixed binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
94 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
95 goto cat(escape_fixed_binary_thunk_,ARG_MODE); \
96 } else { \
97 const void *val1; \
98 const void *val2; \
99 void *result; \
100 val1 = frame_var(fp, slot_1); \
101 val2 = frame_var(fp, slot_2); \
102 result = frame_var(fp, slot_r); \
103 barrier_aliasing(); \
104 if (unlikely(!cat5(tc,binary_,op,_,type)(val1, val2, result))) {\
105 barrier_aliasing(); \
106 goto cat(escape_fixed_binary_thunk_,ARG_MODE); \
107 } \
108 barrier_aliasing(); \
109 ADVANCE_I_PARAM(4); \
110 } \
111 )
113 #define DEFINE_FIXED_REAL_UNARY_OPCODE(tc, type, op) \
114 DEFINE_OPCODE( \
115 cat3(OPCODE_,tc,OP) + \
116 cat4(OPCODE_,tc,OP_,op) * cat3(OPCODE_,tc,OP_MULT) + \
117 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
118 cat7(tc,unary_,op,_,type,_,ARG_MODE), \
119 frame_t slot_1 = get_i_param(0); \
120 frame_t slot_r = get_i_param(1); \
121 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "fixed unary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
122 if (unlikely(frame_test_flag(fp, slot_1))) { \
123 goto cat(escape_fixed_unary_thunk_,ARG_MODE); \
124 } else { \
125 const void *val1; \
126 void *result; \
127 val1 = frame_var(fp, slot_1); \
128 result = frame_var(fp, slot_r); \
129 barrier_aliasing(); \
130 cat5(tc,unary_,op,_,type)(val1, result); \
131 barrier_aliasing(); \
132 ADVANCE_I_PARAM(3); \
133 } \
134 )
136 #define DEFINE_FIXED_TO_INT(type, utype) \
137 DEFINE_OPCODE( \
138 OPCODE_FIXED_OP + \
139 OPCODE_FIXED_OP_to_int * OPCODE_FIXED_OP_MULT + \
140 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
141 cat4(FIXED_unary_to_int_,type,_,ARG_MODE), \
142 frame_t slot_1 = get_i_param(0); \
143 frame_t slot_r = get_i_param(1); \
144 if (unlikely(frame_test_flag(fp, slot_1))) { \
145 goto cat(escape_convert_thunk_,ARG_MODE); \
146 } else { \
147 type val1; \
148 int_default_t r; \
149 barrier_aliasing(); \
150 val1 = *frame_slot(fp, slot_1, type); \
151 r = (int_default_t)val1; \
152 if (unlikely(r != val1)) \
153 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, false));\
154 else \
155 *frame_slot(fp, slot_r, int_default_t) = r; \
156 barrier_aliasing(); \
157 ADVANCE_I_PARAM(3); \
158 } \
159 )
161 #define DEFINE_FIXED_UTO_INT(type, utype) \
162 DEFINE_OPCODE( \
163 OPCODE_FIXED_OP + \
164 OPCODE_FIXED_OP_uto_int * OPCODE_FIXED_OP_MULT + \
165 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
166 cat4(FIXED_unary_uto_int_,type,_,ARG_MODE), \
167 frame_t slot_1 = get_i_param(0); \
168 frame_t slot_r = get_i_param(1); \
169 if (unlikely(frame_test_flag(fp, slot_1))) { \
170 goto cat(escape_convert_thunk_,ARG_MODE); \
171 } else { \
172 utype val1; \
173 int_default_t r; \
174 barrier_aliasing(); \
175 val1 = *frame_slot(fp, slot_1, utype); \
176 r = (int_default_t)val1; \
177 if (unlikely((utype)r != val1) || unlikely(r < 0)) \
178 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, true));\
179 else \
180 *frame_slot(fp, slot_r, int_default_t) = r; \
181 barrier_aliasing(); \
182 ADVANCE_I_PARAM(3); \
183 } \
184 )
186 #define DEFINE_FIXED_FROM_INT(type, utype) \
187 DEFINE_OPCODE( \
188 OPCODE_FIXED_OP + \
189 OPCODE_FIXED_OP_from_int * OPCODE_FIXED_OP_MULT + \
190 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
191 cat4(FIXED_unary_from_int_,type,_,ARG_MODE), \
192 frame_t slot_1 = get_i_param(0); \
193 frame_t slot_r = get_i_param(1); \
194 if (unlikely(frame_test_flag(fp, slot_1))) { \
195 goto cat(escape_convert_thunk_,ARG_MODE); \
196 } else { \
197 int_default_t val1; \
198 type r; \
199 barrier_aliasing(); \
200 val1 = *frame_slot(fp, slot_1, int_default_t); \
201 r = (type)val1; \
202 if (unlikely(r != val1)) \
203 frame_set_pointer(fp, slot_r, pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_DOESNT_FIT), fp, ip pass_file_line));\
204 else \
205 *frame_slot(fp, slot_r, type) = r; \
206 barrier_aliasing(); \
207 ADVANCE_I_PARAM(3); \
208 } \
209 )
211 #define DEFINE_FIXED_UFROM_INT(type, utype) \
212 DEFINE_OPCODE( \
213 OPCODE_FIXED_OP + \
214 OPCODE_FIXED_OP_ufrom_int * OPCODE_FIXED_OP_MULT + \
215 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
216 cat4(FIXED_unary_ufrom_int_,type,_,ARG_MODE), \
217 frame_t slot_1 = get_i_param(0); \
218 frame_t slot_r = get_i_param(1); \
219 if (unlikely(frame_test_flag(fp, slot_1))) { \
220 goto cat(escape_convert_thunk_,ARG_MODE); \
221 } else { \
222 int_default_t val1; \
223 utype r; \
224 barrier_aliasing(); \
225 val1 = *frame_slot(fp, slot_1, int_default_t); \
226 r = (utype)val1; \
227 if (unlikely((int_default_t)r != val1) || unlikely(val1 < 0))\
228 frame_set_pointer(fp, slot_r, pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_DOESNT_FIT), fp, ip pass_file_line));\
229 else \
230 *frame_slot(fp, slot_r, type) = r; \
231 barrier_aliasing(); \
232 ADVANCE_I_PARAM(3); \
233 } \
234 )
236 #define DEFINE_SCALAR_MOVE_OPCODE(tc, type) \
237 DEFINE_OPCODE( \
238 cat3(OPCODE_,tc,OP) + \
239 cat3(OPCODE_,tc,OP_move) * cat3(OPCODE_,tc,OP_MULT) + \
240 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
241 cat5(tc,move_,type,_,ARG_MODE), \
242 move_scalar(type); \
243 )
245 #define DEFINE_SCALAR_COPY_OPCODE(tc, type) \
246 DEFINE_OPCODE( \
247 cat3(OPCODE_,tc,OP) + \
248 cat3(OPCODE_,tc,OP_copy) * cat3(OPCODE_,tc,OP_MULT) + \
249 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
250 cat5(tc,copy_,type,_,ARG_MODE), \
251 copy_scalar(type); \
252 )
254 #define DEFINE_FIXED_LDC_OPCODE(cls, type, utype, opc, shrt) \
255 DEFINE_OPCODE( \
256 OPCODE_##cls##_OP + \
257 OPCODE_##cls##_OP_##opc * OPCODE_##cls##_OP_MULT + \
258 cat(OPCODE_##cls##_TYPE_,type) * OPCODE_##cls##_TYPE_MULT,\
259 cat6(fixed_,opc,_,type,_,ARG_MODE), \
260 frame_t slot; \
261 utype *val; \
262 ip_t sz; \
263 slot = get_i_param(0); \
264 ajla_assert(!frame_test_flag(fp, slot), (file_line, "ldc fixed: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot));\
265 val = frame_slot(fp, slot, utype); \
266 barrier_aliasing(); \
267 sz = cat(fixed_ldc_,type)(val, ip + 1 + param_size(1), shrt); \
268 barrier_aliasing(); \
269 ADVANCE_I_PARAM(1); \
270 ADVANCE_IP(sz); \
271 )
273 #define DEFINE_FIXED_OPCODES(n, type, utype, sz, bits) \
274 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, add) \
275 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, subtract) \
276 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, multiply) \
277 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, divide) \
278 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, udivide) \
279 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, modulo) \
280 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, umodulo) \
281 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, power) \
282 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, and) \
283 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, or) \
284 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, xor) \
285 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, shl) \
286 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, shr) \
287 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, ushr) \
288 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, rol) \
289 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, ror) \
290 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, bts) \
291 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, btr) \
292 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, btc) \
293 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, equal) \
294 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, not_equal)\
295 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, less) \
296 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, less_equal)\
297 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, uless) \
298 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, uless_equal)\
299 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, bt) \
300 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, not) \
301 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, neg) \
302 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, inc) \
303 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, dec) \
304 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, bswap) \
305 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, brev) \
306 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, bsf) \
307 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, bsr) \
308 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, popcnt) \
309 DEFINE_FIXED_TO_INT(type, utype) \
310 DEFINE_FIXED_UTO_INT(type, utype) \
311 DEFINE_FIXED_FROM_INT(type, utype) \
312 DEFINE_FIXED_UFROM_INT(type, utype) \
313 DEFINE_SCALAR_MOVE_OPCODE(FIXED_, type) \
314 DEFINE_SCALAR_COPY_OPCODE(FIXED_, type) \
315 DEFINE_FIXED_LDC_OPCODE(FIXED, type, utype, ldc, false)
317 for_all_fixed(DEFINE_FIXED_OPCODES)
319 #if TYPE_FIXED_N >= 3
320 DEFINE_FIXED_LDC_OPCODE(FIXED, int32_t, uint32_t, ldc16, true)
321 #if TYPE_FIXED_N >= 4
322 DEFINE_FIXED_LDC_OPCODE(FIXED, int64_t, uint64_t, ldc16, true)
323 #if TYPE_FIXED_N >= 5
324 DEFINE_FIXED_LDC_OPCODE(FIXED, int128_t, uint128_t, ldc16, true)
325 #endif
326 #endif
327 #endif
330 #define DEFINE_REAL_TO_INT(type, op) \
331 DEFINE_OPCODE( \
332 OPCODE_REAL_OP + \
333 cat(OPCODE_REAL_OP_,op) * OPCODE_REAL_OP_MULT + \
334 cat(OPCODE_REAL_TYPE_,type) * OPCODE_REAL_TYPE_MULT, \
335 cat6(REAL_unary_,op,_,type,_,ARG_MODE), \
336 frame_t slot_1 = get_i_param(0); \
337 frame_t slot_r = get_i_param(1); \
338 if (unlikely(frame_test_flag(fp, slot_1))) { \
339 goto cat(escape_convert_thunk_,ARG_MODE); \
340 } else { \
341 bool ret; \
342 barrier_aliasing(); \
343 ret = cat4(REAL_unary_,op,_,type)(frame_slot(fp, slot_1, type), frame_slot(fp, slot_r, int_default_t));\
344 barrier_aliasing(); \
345 if (unlikely(!ret)) \
346 frame_set_pointer(fp, slot_r, convert_real_to_mpint(fp, slot_1, frame_get_type_of_local(fp, slot_1)));\
347 ADVANCE_I_PARAM(3); \
348 } \
349 )
351 #define DEFINE_REAL_FROM_INT(type, op) \
352 DEFINE_OPCODE( \
353 OPCODE_REAL_OP + \
354 cat(OPCODE_REAL_OP_,op) * OPCODE_REAL_OP_MULT + \
355 cat(OPCODE_REAL_TYPE_,type) * OPCODE_REAL_TYPE_MULT, \
356 cat6(REAL_unary_,op,_,type,_,ARG_MODE), \
357 frame_t slot_1 = get_i_param(0); \
358 frame_t slot_r = get_i_param(1); \
359 if (unlikely(frame_test_flag(fp, slot_1))) { \
360 goto cat(escape_convert_thunk_,ARG_MODE); \
361 } else { \
362 barrier_aliasing(); \
363 cat4(REAL_unary_,op,_,type)(frame_slot(fp, slot_1, int_default_t), frame_slot(fp, slot_r, type));\
364 barrier_aliasing(); \
365 ADVANCE_I_PARAM(3); \
366 } \
367 )
370 #define DEFINE_REAL_OPCODES(n, type, ntype, pack, unpack)\
371 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, add) \
372 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, subtract) \
373 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, multiply) \
374 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, divide) \
375 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, modulo) \
376 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, power) \
377 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, ldexp) \
378 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, atan2) \
379 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, equal) \
380 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, not_equal) \
381 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, less) \
382 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, less_equal)\
383 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, neg) \
384 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, sqrt) \
385 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, cbrt) \
386 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, sin) \
387 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, cos) \
388 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, tan) \
389 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, asin) \
390 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, acos) \
391 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, atan) \
392 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, sinh) \
393 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, cosh) \
394 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, tanh) \
395 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, asinh) \
396 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, acosh) \
397 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, atanh) \
398 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exp2) \
399 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exp) \
400 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exp10) \
401 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, log2) \
402 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, log) \
403 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, log10) \
404 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, round) \
405 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, ceil) \
406 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, floor) \
407 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, trunc) \
408 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, fract) \
409 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, mantissa) \
410 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exponent) \
411 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, next_number)\
412 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, prev_number)\
413 DEFINE_REAL_TO_INT(type, to_int) \
414 DEFINE_REAL_FROM_INT(type, from_int) \
415 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, is_exception)\
416 DEFINE_SCALAR_MOVE_OPCODE(REAL_, type) \
417 DEFINE_SCALAR_COPY_OPCODE(REAL_, type) \
418 DEFINE_FIXED_LDC_OPCODE(REAL, type, type, ldc, false)
420 for_all_real(DEFINE_REAL_OPCODES, for_all_empty)
423 DEFINE_LABEL(cat(escape_fixed_binary_thunk_,ARG_MODE),
424 void *ex;
425 ASM_PREVENT_CSE;
426 ex = thunk_fixed_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3));
427 if (ex == POINTER_FOLLOW_THUNK_GO)
428 ADVANCE_I_PARAM(4);
429 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
430 RELOAD_EX_POSITION(ex);
431 )
434 DEFINE_LABEL(cat(escape_fixed_unary_thunk_,ARG_MODE),
435 void *ex;
436 ASM_PREVENT_CSE;
437 ex = thunk_fixed_operator(fp, ip, get_i_param(0), NO_FRAME_T, get_i_param(1), get_i_param(2));
438 if (ex == POINTER_FOLLOW_THUNK_GO)
439 ADVANCE_I_PARAM(3);
440 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
441 RELOAD_EX_POSITION(ex);
442 )
445 DEFINE_LABEL(cat(escape_convert_thunk_,ARG_MODE),
446 void *ex;
447 ASM_PREVENT_CSE;
448 ex = thunk_convert(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2));
449 if (ex == POINTER_FOLLOW_THUNK_GO)
450 ADVANCE_I_PARAM(3);
451 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
452 RELOAD_EX_POSITION(ex);
453 )
456 START_BLOCK(
457 bool (attr_fastcall *function_int_binary)(const mpint_t *s1, const mpint_t *s2, mpint_t *r, ajla_error_t *err);
458 bool (attr_fastcall *function_int_binary_logical)(const mpint_t *s1, const mpint_t *s2, ajla_flat_option_t *r, ajla_error_t *err);
459 bool (attr_fastcall *function_int_unary)(const mpint_t *s1, mpint_t *r, ajla_error_t *err);
460 )
462 #define DEFINE_INT_BINARY_OPCODE(type, op) \
463 DEFINE_OPCODE( \
464 OPCODE_INT_OP + \
465 cat(OPCODE_INT_OP_,op) * OPCODE_INT_OP_MULT + \
466 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
467 cat6(int_binary_,op,_,type,_,ARG_MODE), \
468 frame_t slot_1 = get_i_param(0); \
469 frame_t slot_2 = get_i_param(1); \
470 frame_t slot_r = get_i_param(2); \
471 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "int binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
472 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
473 cat6(escape_int_binary_,op,_,type,_,ARG_MODE): \
474 function_int_binary = cat(mpint_,op); \
475 goto cat(escape_int_binary_thunk_,ARG_MODE); \
476 } else { \
477 const void *val1; \
478 const void *val2; \
479 void *result; \
480 val1 = frame_var(fp, slot_1); \
481 val2 = frame_var(fp, slot_2); \
482 result = frame_var(fp, slot_r); \
483 barrier_aliasing(); \
484 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, val2, result))) {\
485 barrier_aliasing(); \
486 goto cat6(escape_int_binary_,op,_,type,_,ARG_MODE);\
487 } \
488 barrier_aliasing(); \
489 ADVANCE_I_PARAM(4); \
490 } \
491 )
493 #define DEFINE_INT_BINARY_LOGICAL_OPCODE(type, op) \
494 DEFINE_OPCODE( \
495 OPCODE_INT_OP + \
496 cat(OPCODE_INT_OP_,op) * OPCODE_INT_OP_MULT + \
497 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
498 cat6(int_binary_,op,_,type,_,ARG_MODE), \
499 frame_t slot_1 = get_i_param(0); \
500 frame_t slot_2 = get_i_param(1); \
501 frame_t slot_r = get_i_param(2); \
502 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "int logical: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
503 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
504 cat6(escape_int_binary_logical_,op,_,type,_,ARG_MODE): \
505 function_int_binary_logical = cat(mpint_,op); \
506 goto cat(escape_int_binary_logical_thunk_,ARG_MODE); \
507 } else { \
508 const void *val1; \
509 const void *val2; \
510 void *result; \
511 val1 = frame_var(fp, slot_1); \
512 val2 = frame_var(fp, slot_2); \
513 result = frame_var(fp, slot_r); \
514 barrier_aliasing(); \
515 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, val2, result))) {\
516 barrier_aliasing(); \
517 goto cat6(escape_int_binary_logical_,op,_,type,_,ARG_MODE);\
518 } \
519 barrier_aliasing(); \
520 ADVANCE_I_PARAM(4); \
521 } \
522 )
524 #define DEFINE_INT_BINARY_CONST_OPCODE(type, op) \
525 DEFINE_OPCODE( \
526 OPCODE_INT_OP + \
527 cat(OPCODE_INT_OP_C_,op) * OPCODE_INT_OP_MULT + \
528 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
529 cat6(int_binary_const_,op,_,type,_,ARG_MODE), \
530 frame_t slot_1 = get_i_param(0); \
531 frame_t slot_2 = get_i_param(1); \
532 frame_t slot_r = get_i_param(2); \
533 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "int binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
534 if (unlikely(frame_test_flag(fp, slot_1))) { \
535 cat6(escape_int_binary_const_,op,_,type,_,ARG_MODE): \
536 function_int_binary = cat(mpint_,op); \
537 goto cat(escape_int_binary_const_thunk_,ARG_MODE); \
538 } else { \
539 const void *val1; \
540 type val2; \
541 void *result; \
542 val1 = frame_var(fp, slot_1); \
543 val2 = (int32_t)slot_2; \
544 result = frame_var(fp, slot_r); \
545 barrier_aliasing(); \
546 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, &val2, result))) {\
547 barrier_aliasing(); \
548 goto cat6(escape_int_binary_const_,op,_,type,_,ARG_MODE);\
549 } \
550 barrier_aliasing(); \
551 ADVANCE_I_PARAM(4); \
552 } \
553 )
555 #define DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(type, op) \
556 DEFINE_OPCODE( \
557 OPCODE_INT_OP + \
558 cat(OPCODE_INT_OP_C_,op) * OPCODE_INT_OP_MULT + \
559 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
560 cat6(int_binary_const_,op,_,type,_,ARG_MODE), \
561 frame_t slot_1 = get_i_param(0); \
562 frame_t slot_2 = get_i_param(1); \
563 frame_t slot_r = get_i_param(2); \
564 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "int logical: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
565 if (unlikely(frame_test_flag(fp, slot_1))) { \
566 cat6(escape_int_binary_const_logical_,op,_,type,_,ARG_MODE): \
567 function_int_binary_logical = cat(mpint_,op); \
568 goto cat(escape_int_binary_const_logical_thunk_,ARG_MODE);\
569 } else { \
570 const void *val1; \
571 type val2; \
572 void *result; \
573 val1 = frame_var(fp, slot_1); \
574 val2 = (int32_t)slot_2; \
575 result = frame_var(fp, slot_r); \
576 barrier_aliasing(); \
577 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, &val2, result))) {\
578 barrier_aliasing(); \
579 goto cat6(escape_int_binary_const_logical_,op,_,type,_,ARG_MODE);\
580 } \
581 barrier_aliasing(); \
582 ADVANCE_I_PARAM(4); \
583 } \
584 )
586 #define DEFINE_INT_UNARY_OPCODE(type, op) \
587 DEFINE_OPCODE( \
588 OPCODE_INT_OP + \
589 cat(OPCODE_INT_OP_,op) * OPCODE_INT_OP_MULT + \
590 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
591 cat6(int_binary_,op,_,type,_,ARG_MODE), \
592 frame_t slot_1 = get_i_param(0); \
593 frame_t slot_r = get_i_param(1); \
594 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "int unary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
595 if (unlikely(frame_test_flag(fp, slot_1))) { \
596 cat6(escape_int_unary_,op,_,type,_,ARG_MODE): \
597 function_int_unary = cat(mpint_,op); \
598 goto cat(escape_int_unary_thunk_,ARG_MODE); \
599 } else { \
600 const void *val1; \
601 void *result; \
602 val1 = frame_var(fp, slot_1); \
603 result = frame_var(fp, slot_r); \
604 barrier_aliasing(); \
605 if (unlikely(!cat4(INT_unary_,op,_,type)(val1, result))) {\
606 barrier_aliasing(); \
607 goto cat6(escape_int_unary_,op,_,type,_,ARG_MODE);\
608 } \
609 barrier_aliasing(); \
610 ADVANCE_I_PARAM(3); \
611 } \
612 )
614 #define DEFINE_INT_TO_INT(type) \
615 DEFINE_OPCODE( \
616 OPCODE_INT_OP + \
617 OPCODE_INT_OP_to_int * OPCODE_INT_OP_MULT + \
618 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
619 cat4(INT_unary_to_int_,type,_,ARG_MODE), \
620 frame_t slot_1 = get_i_param(0); \
621 frame_t slot_r = get_i_param(1); \
622 if (unlikely(frame_test_flag(fp, slot_1))) { \
623 goto cat(escape_convert_thunk_,ARG_MODE); \
624 } else { \
625 type val1; \
626 int_default_t r; \
627 barrier_aliasing(); \
628 val1 = *frame_slot(fp, slot_1, type); \
629 r = (int_default_t)val1; \
630 if (unlikely(r != val1)) \
631 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, false));\
632 else \
633 *frame_slot(fp, slot_r, int_default_t) = r; \
634 barrier_aliasing(); \
635 ADVANCE_I_PARAM(3); \
636 } \
637 )
639 #define DEFINE_INT_FROM_INT(typeid, type) \
640 DEFINE_OPCODE( \
641 OPCODE_INT_OP + \
642 OPCODE_INT_OP_from_int * OPCODE_INT_OP_MULT + \
643 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
644 cat4(INT_unary_from_int_,type,_,ARG_MODE), \
645 frame_t slot_1 = get_i_param(0); \
646 frame_t slot_r = get_i_param(1); \
647 if (unlikely(frame_test_flag(fp, slot_1))) { \
648 goto cat(escape_convert_thunk_,ARG_MODE); \
649 } else { \
650 int_default_t val1; \
651 type r; \
652 barrier_aliasing(); \
653 val1 = *frame_slot(fp, slot_1, int_default_t); \
654 r = (type)val1; \
655 if (unlikely(r != val1)) \
656 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, false));\
657 else \
658 *frame_slot(fp, slot_r, type) = r; \
659 barrier_aliasing(); \
660 ADVANCE_I_PARAM(3); \
661 } \
662 )
664 #define DEFINE_INT_LDC_OPCODE(type, opc, shrt) \
665 DEFINE_OPCODE( \
666 OPCODE_INT_OP + \
667 OPCODE_INT_OP_##opc * OPCODE_INT_OP_MULT + \
668 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
669 cat6(int_,opc,_,type,_,ARG_MODE), \
670 frame_t slot; \
671 type *val; \
672 ip_t sz; \
673 slot = get_i_param(0); \
674 ajla_assert(!frame_test_flag(fp, slot), (file_line, "ldc int: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot));\
675 val = frame_slot(fp, slot, type); \
676 barrier_aliasing(); \
677 sz = cat(int_ldc_,type)(val, ip + 1 + param_size(1), shrt); \
678 barrier_aliasing(); \
679 ADVANCE_I_PARAM(1); \
680 ADVANCE_IP(sz); \
681 )
683 #define DEFINE_INT_OPCODES(typeid, s, u, sz, bits) \
684 DEFINE_INT_BINARY_OPCODE(s, add) \
685 DEFINE_INT_BINARY_OPCODE(s, subtract) \
686 DEFINE_INT_BINARY_OPCODE(s, multiply) \
687 DEFINE_INT_BINARY_OPCODE(s, divide) \
688 DEFINE_INT_BINARY_OPCODE(s, modulo) \
689 DEFINE_INT_BINARY_OPCODE(s, power) \
690 DEFINE_INT_BINARY_OPCODE(s, and) \
691 DEFINE_INT_BINARY_OPCODE(s, or) \
692 DEFINE_INT_BINARY_OPCODE(s, xor) \
693 DEFINE_INT_BINARY_OPCODE(s, shl) \
694 DEFINE_INT_BINARY_OPCODE(s, shr) \
695 DEFINE_INT_BINARY_OPCODE(s, bts) \
696 DEFINE_INT_BINARY_OPCODE(s, btr) \
697 DEFINE_INT_BINARY_OPCODE(s, btc) \
698 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, equal) \
699 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, not_equal) \
700 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, less) \
701 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, less_equal) \
702 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, bt) \
703 DEFINE_INT_BINARY_CONST_OPCODE(s, add) \
704 DEFINE_INT_BINARY_CONST_OPCODE(s, subtract) \
705 DEFINE_INT_BINARY_CONST_OPCODE(s, multiply) \
706 DEFINE_INT_BINARY_CONST_OPCODE(s, divide) \
707 DEFINE_INT_BINARY_CONST_OPCODE(s, modulo) \
708 DEFINE_INT_BINARY_CONST_OPCODE(s, power) \
709 DEFINE_INT_BINARY_CONST_OPCODE(s, and) \
710 DEFINE_INT_BINARY_CONST_OPCODE(s, or) \
711 DEFINE_INT_BINARY_CONST_OPCODE(s, xor) \
712 DEFINE_INT_BINARY_CONST_OPCODE(s, shl) \
713 DEFINE_INT_BINARY_CONST_OPCODE(s, shr) \
714 DEFINE_INT_BINARY_CONST_OPCODE(s, bts) \
715 DEFINE_INT_BINARY_CONST_OPCODE(s, btr) \
716 DEFINE_INT_BINARY_CONST_OPCODE(s, btc) \
717 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, equal) \
718 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, not_equal) \
719 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, less) \
720 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, less_equal) \
721 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, bt) \
722 DEFINE_INT_UNARY_OPCODE(s, not) \
723 DEFINE_INT_UNARY_OPCODE(s, neg) \
724 DEFINE_INT_UNARY_OPCODE(s, inc) \
725 DEFINE_INT_UNARY_OPCODE(s, dec) \
726 DEFINE_INT_UNARY_OPCODE(s, bsf) \
727 DEFINE_INT_UNARY_OPCODE(s, bsr) \
728 DEFINE_INT_UNARY_OPCODE(s, popcnt) \
729 DEFINE_INT_TO_INT(s) \
730 DEFINE_INT_FROM_INT(typeid, s) \
731 DEFINE_SCALAR_MOVE_OPCODE(INT_, s) \
732 DEFINE_SCALAR_COPY_OPCODE(INT_, s) \
733 DEFINE_INT_LDC_OPCODE(s, ldc, false) \
734 DEFINE_INT_LDC_OPCODE(s, ldc16, true)
736 for_all_int(DEFINE_INT_OPCODES, for_all_empty)
738 DEFINE_LABEL(cat(escape_int_binary_thunk_,ARG_MODE),
739 void *ex;
740 ASM_PREVENT_CSE;
741 ex = thunk_int_binary_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3), function_int_binary);
742 if (ex == POINTER_FOLLOW_THUNK_GO)
743 ADVANCE_I_PARAM(4);
744 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
745 RELOAD_EX_POSITION(ex);
746 )
748 DEFINE_LABEL(cat(escape_int_binary_logical_thunk_,ARG_MODE),
749 void *ex;
750 ASM_PREVENT_CSE;
751 ex = thunk_int_binary_logical_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3), function_int_binary_logical);
752 if (ex == POINTER_FOLLOW_THUNK_GO)
753 ADVANCE_I_PARAM(4);
754 else if (ex != POINTER_FOLLOW_THUNK_RETRY)
755 RELOAD_EX_POSITION(ex);
756 )
758 DEFINE_LABEL(cat(escape_int_binary_const_thunk_,ARG_MODE),
759 void *ex;
760 ASM_PREVENT_CSE;
761 ex = thunk_int_binary_operator(fp, ip, get_i_param(0), frame_t_from_const((int32_t)get_i_param(1)), get_i_param(2), get_i_param(3), function_int_binary);
762 if (ex == POINTER_FOLLOW_THUNK_GO)
763 ADVANCE_I_PARAM(4);
764 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
765 RELOAD_EX_POSITION(ex);
766 )
768 DEFINE_LABEL(cat(escape_int_binary_const_logical_thunk_,ARG_MODE),
769 void *ex;
770 ASM_PREVENT_CSE;
771 ex = thunk_int_binary_logical_operator(fp, ip, get_i_param(0), frame_t_from_const((int32_t)get_i_param(1)), get_i_param(2), get_i_param(3), function_int_binary_logical);
772 if (ex == POINTER_FOLLOW_THUNK_GO)
773 ADVANCE_I_PARAM(4);
774 else if (ex != POINTER_FOLLOW_THUNK_RETRY)
775 RELOAD_EX_POSITION(ex);
776 )
778 DEFINE_LABEL(cat(escape_int_unary_thunk_,ARG_MODE),
779 void *ex;
780 ASM_PREVENT_CSE;
781 ex = thunk_int_unary_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), function_int_unary);
782 if (ex == POINTER_FOLLOW_THUNK_GO)
783 ADVANCE_I_PARAM(3);
784 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
785 RELOAD_EX_POSITION(ex);
786 )
788 #define file_inc "ipret-a1.inc"
789 #include "for-fix.inc"
791 #define file_inc "ipret-a2.inc"
792 #include "for-int.inc"
794 #define file_inc "ipret-a3.inc"
795 #include "for-real.inc"
797 END_BLOCK()
800 #define DEFINE_BOOL_BINARY_OPCODE(op, operator) \
801 DEFINE_OPCODE( \
802 OPCODE_BOOL_OP + \
803 cat(OPCODE_BOOL_OP_,op) * OPCODE_BOOL_OP_MULT, \
804 cat4(bool_binary_,op,_,ARG_MODE), \
805 frame_t slot_1 = get_i_param(0); \
806 frame_t slot_2 = get_i_param(1); \
807 frame_t slot_r = get_i_param(2); \
808 \
809 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
810 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "bool binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
811 goto cat(escape_bool_binary_thunk_,ARG_MODE); \
812 } else { \
813 ajla_flat_option_t val1; \
814 ajla_flat_option_t val2; \
815 ajla_flat_option_t result; \
816 barrier_aliasing(); \
817 val1 = *frame_slot(fp, slot_1, ajla_flat_option_t); \
818 val2 = *frame_slot(fp, slot_2, ajla_flat_option_t); \
819 result = val1 operator val2; \
820 *frame_slot(fp, slot_r, ajla_flat_option_t) = result; \
821 barrier_aliasing(); \
822 ADVANCE_I_PARAM(4); \
823 } \
824 )
826 #define DEFINE_BOOL_UNARY_OPCODE(op, operator) \
827 DEFINE_OPCODE( \
828 OPCODE_BOOL_OP + \
829 cat(OPCODE_BOOL_OP_,op) * OPCODE_BOOL_OP_MULT, \
830 cat4(bool_unary_,op,_,ARG_MODE), \
831 frame_t slot_1 = get_i_param(0); \
832 frame_t slot_r = get_i_param(1); \
833 \
834 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "bool unary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
835 if (unlikely(frame_test_flag(fp, slot_1))) { \
836 goto cat(escape_bool_unary_thunk_,ARG_MODE); \
837 } else { \
838 ajla_flat_option_t val1; \
839 ajla_flat_option_t result; \
840 barrier_aliasing(); \
841 val1 = *frame_slot(fp, slot_1, ajla_flat_option_t); \
842 result = val1 operator; \
843 *frame_slot(fp, slot_r, ajla_flat_option_t) = result; \
844 barrier_aliasing(); \
845 ADVANCE_I_PARAM(3); \
846 } \
847 )
849 DEFINE_BOOL_BINARY_OPCODE(and, &)
850 DEFINE_BOOL_BINARY_OPCODE(or, |)
851 DEFINE_BOOL_BINARY_OPCODE(equal, ==)
852 DEFINE_BOOL_BINARY_OPCODE(not_equal, ^)
853 DEFINE_BOOL_BINARY_OPCODE(less, <)
854 DEFINE_BOOL_BINARY_OPCODE(less_equal, <=)
855 DEFINE_BOOL_UNARY_OPCODE(not, ^ 1)
857 DEFINE_LABEL(cat(escape_bool_binary_thunk_,ARG_MODE),
858 void *ex;
859 ASM_PREVENT_CSE;
860 ex = thunk_bool_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3));
861 if (ex == POINTER_FOLLOW_THUNK_GO)
862 ADVANCE_I_PARAM(4);
863 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
864 RELOAD_EX_POSITION(ex);
865 )
867 DEFINE_LABEL(cat(escape_bool_unary_thunk_,ARG_MODE),
868 void *ex;
869 ASM_PREVENT_CSE;
870 ex = thunk_bool_operator(fp, ip, get_i_param(0), NO_FRAME_T, get_i_param(1), get_i_param(2));
871 if (ex == POINTER_FOLLOW_THUNK_GO)
872 ADVANCE_I_PARAM(3);
873 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
874 RELOAD_EX_POSITION(ex);
875 )
878 DEFINE_OPCODE_START_LBL(OPCODE_BOOL_OP + OPCODE_BOOL_OP_move, cat(bool_move_,ARG_MODE))
879 #ifdef EMIT_CODE
880 move_scalar(ajla_flat_option_t);
881 #endif
882 DEFINE_OPCODE_END(OPCODE_BOOL_OP + OPCODE_BOOL_OP_move)
884 DEFINE_OPCODE_START_LBL(OPCODE_BOOL_OP + OPCODE_BOOL_OP_copy, cat(bool_copy_,ARG_MODE))
885 #ifdef EMIT_CODE
886 copy_scalar(ajla_flat_option_t);
887 #endif
888 DEFINE_OPCODE_END(OPCODE_BOOL_OP + OPCODE_BOOL_OP_copy)
891 DEFINE_OPCODE_START(OPCODE_INT_LDC_LONG)
892 #ifdef EMIT_CODE
893 frame_t slot;
894 ip_t sz;
895 slot = get_i_param(0);
896 sz = ipret_int_ldc_long(fp, slot, ip + 1 + param_size(1));
897 ADVANCE_I_PARAM(1);
898 ADVANCE_IP(sz);
899 #endif
900 DEFINE_OPCODE_END(OPCODE_INT_LDC_LONG)
903 DEFINE_OPCODE_START(OPCODE_IS_EXCEPTION)
904 #ifdef EMIT_CODE
905 frame_t slot_1 = get_i_param(0);
906 frame_t slot_r = get_i_param(1);
907 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "is exception: %s: flag already set for destination slot %"PRIuMAX"", da(get_frame(fp)->function,function)->function_name, (uintmax_t)slot_r));
908 if (!frame_variable_is_flat(fp, slot_1)) {
909 void *ex = is_thunk_operator(fp, ip, slot_1, slot_r, get_i_param(2));
910 if (ex == POINTER_FOLLOW_THUNK_GO)
911 ADVANCE_I_PARAM(3);
912 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
913 RELOAD_EX_POSITION(ex);
914 } else {
915 barrier_aliasing();
916 *frame_slot(fp, slot_r, ajla_flat_option_t) = 0;
917 barrier_aliasing();
918 ADVANCE_I_PARAM(3);
919 }
920 #endif
921 DEFINE_OPCODE_END(OPCODE_IS_EXCEPTION)
923 DEFINE_OPCODE_START(OPCODE_EXCEPTION_CLASS)
924 #ifdef EMIT_CODE
925 void *ex;
926 ex = thunk_get_param(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), 0);
927 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
928 ADVANCE_I_PARAM(3);
929 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
930 RELOAD_EX_POSITION(ex);
931 #endif
932 DEFINE_OPCODE_END(OPCODE_EXCEPTION_CLASS)
934 DEFINE_OPCODE_START(OPCODE_EXCEPTION_TYPE)
935 #ifdef EMIT_CODE
936 void *ex;
937 ex = thunk_get_param(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), 1);
938 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
939 ADVANCE_I_PARAM(3);
940 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
941 RELOAD_EX_POSITION(ex);
942 #endif
943 DEFINE_OPCODE_END(OPCODE_EXCEPTION_TYPE)
945 DEFINE_OPCODE_START(OPCODE_EXCEPTION_AUX)
946 #ifdef EMIT_CODE
947 void *ex;
948 ex = thunk_get_param(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), 2);
949 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
950 ADVANCE_I_PARAM(3);
951 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
952 RELOAD_EX_POSITION(ex);
953 #endif
954 DEFINE_OPCODE_END(OPCODE_EXCEPTION_AUX)
956 DEFINE_OPCODE_START(OPCODE_SYSTEM_PROPERTY)
957 #ifdef EMIT_CODE
958 void *ex;
959 ex = ipret_get_system_property(fp, ip, get_i_param(0), get_i_param(1));
960 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
961 ADVANCE_I_PARAM(3);
962 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
963 RELOAD_EX_POSITION(ex);
964 #endif
965 DEFINE_OPCODE_END(OPCODE_SYSTEM_PROPERTY)
968 DEFINE_OPCODE_START(OPCODE_FLAT_MOVE)
969 #ifdef EMIT_CODE
970 frame_t slot_1 = get_i_param(0);
971 frame_t slot_r = get_i_param(1);
972 if (likely(frame_variable_is_flat(fp, slot_1))) {
973 memcpy(frame_var(fp, slot_r), frame_var(fp, slot_1), frame_get_type_of_local(fp, slot_1)->size);
974 } else {
975 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, true);
976 frame_set_pointer(fp, slot_r, ptr);
977 }
978 ADVANCE_I_PARAM(2);
979 #endif
980 DEFINE_OPCODE_END(OPCODE_FLAT_MOVE)
982 DEFINE_OPCODE_START(OPCODE_FLAT_COPY)
983 #ifdef EMIT_CODE
984 frame_t slot_1 = get_i_param(0);
985 frame_t slot_r = get_i_param(1);
986 if (likely(frame_variable_is_flat(fp, slot_1))) {
987 memcpy(frame_var(fp, slot_r), frame_var(fp, slot_1), frame_get_type_of_local(fp, slot_1)->size);
988 } else {
989 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, false);
990 frame_set_pointer(fp, slot_r, ptr);
991 }
992 ADVANCE_I_PARAM(2);
993 #endif
994 DEFINE_OPCODE_END(OPCODE_FLAT_COPY)
996 DEFINE_OPCODE_START(OPCODE_REF_MOVE)
997 #ifdef EMIT_CODE
998 frame_t slot_1 = get_i_param(0);
999 frame_t slot_r = get_i_param(1);
1000 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, true);
1001 pointer_poison(frame_pointer(fp, slot_1));
1002 frame_set_pointer(fp, slot_r, ptr);
1003 ADVANCE_I_PARAM(2);
1004 #endif
1005 DEFINE_OPCODE_END(OPCODE_REF_MOVE)
1007 DEFINE_OPCODE_START(OPCODE_REF_MOVE_CLEAR)
1008 #ifdef EMIT_CODE
1009 frame_t slot_1 = get_i_param(0);
1010 frame_t slot_r = get_i_param(1);
1011 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, true);
1012 frame_set_pointer(fp, slot_r, ptr);
1013 ADVANCE_I_PARAM(2);
1014 #endif
1015 DEFINE_OPCODE_END(OPCODE_REF_MOVE_CLEAR)
1017 DEFINE_OPCODE_START(OPCODE_REF_COPY)
1018 #ifdef EMIT_CODE
1019 frame_t slot_1 = get_i_param(0);
1020 frame_t slot_r = get_i_param(1);
1021 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, false);
1022 frame_set_pointer(fp, slot_r, ptr);
1023 ADVANCE_I_PARAM(2);
1024 #endif
1025 DEFINE_OPCODE_END(OPCODE_REF_COPY)
1027 DEFINE_OPCODE_START(OPCODE_BOX_MOVE_CLEAR)
1028 #ifdef EMIT_CODE
1029 frame_t slot_1 = get_i_param(0);
1030 frame_t slot_r = get_i_param(1);
1031 pointer_t ptr = ipret_copy_variable_to_pointer(fp, slot_1, true);
1032 *frame_pointer(fp, slot_1) = pointer_empty();
1033 frame_set_pointer(fp, slot_r, ptr);
1034 ADVANCE_I_PARAM(2);
1035 #endif
1036 DEFINE_OPCODE_END(OPCODE_BOX_MOVE_CLEAR)
1038 DEFINE_OPCODE_START(OPCODE_BOX_COPY)
1039 #ifdef EMIT_CODE
1040 frame_t slot_1 = get_i_param(0);
1041 frame_t slot_r = get_i_param(1);
1042 pointer_t ptr = ipret_copy_variable_to_pointer(fp, slot_1, false);
1043 frame_set_pointer(fp, slot_r, ptr);
1044 ADVANCE_I_PARAM(2);
1045 #endif
1046 DEFINE_OPCODE_END(OPCODE_BOX_COPY)
1048 DEFINE_OPCODE_START(OPCODE_TAKE_BORROWED)
1049 #ifdef EMIT_CODE
1050 frame_t slot = get_i_param(0);
1051 if (!frame_test_and_set_flag(fp, slot)) {
1052 pointer_t ptr = *frame_pointer(fp, slot);
1053 pointer_reference_owned(ptr);
1054 }
1055 ADVANCE_I_PARAM(1);
1056 #endif
1057 DEFINE_OPCODE_END(OPCODE_TAKE_BORROWED)
1060 DEFINE_OPCODE_START(OPCODE_DEREFERENCE)
1061 #ifdef EMIT_CODE
1062 frame_t slot = get_i_param(0);
1063 #if 0
1064 {
1065 struct data *fn = get_frame(fp)->function;
1066 const struct type *type = frame_get_type_of_local(fp, slot);
1067 bool may_be_borrowed = da(fn,function)->local_variables[slot].may_be_borrowed;
1068 if (!frame_test_flag(fp, slot) && !may_be_borrowed && !TYPE_IS_FLAT(type))
1069 internal(file_line, "dereference: %s: the value is unexpectedly borrowed", da(fn,function)->function_name);
1070 }
1071 #endif
1072 frame_free(fp, slot);
1073 pointer_poison(frame_pointer(fp, slot));
1074 ADVANCE_I_PARAM(1);
1075 #endif
1076 DEFINE_OPCODE_END(OPCODE_DEREFERENCE)
1078 DEFINE_OPCODE_START(OPCODE_DEREFERENCE_CLEAR)
1079 #ifdef EMIT_CODE
1080 frame_t slot = get_i_param(0);
1081 #if 0
1082 {
1083 struct data *fn = get_frame(fp)->function;
1084 const struct type *type = frame_get_type_of_local(fp, slot);
1085 bool may_be_borrowed = da(fn,function)->local_variables[slot].may_be_borrowed;
1086 if (!frame_test_flag(fp, slot) && !may_be_borrowed && !TYPE_IS_FLAT(type))
1087 internal(file_line, "dereference clear: %s: the value is unexpectedly borrowed", da(fn,function)->function_name);
1088 }
1089 #endif
1090 frame_free_and_clear(fp, slot);
1091 ADVANCE_I_PARAM(1);
1092 #endif
1093 DEFINE_OPCODE_END(OPCODE_DEREFERENCE_CLEAR)
1095 DEFINE_OPCODE_START(OPCODE_EVAL)
1096 #ifdef EMIT_CODE
1097 frame_t slot = get_i_param(0);
1098 if (frame_test_flag(fp, slot)) {
1099 pointer_t *ptr = frame_pointer(fp, slot);
1100 struct data attr_unused *result;
1101 /*cat(eval_again_,ARG_MODE):*/
1102 pointer_follow(ptr, true, result, PF_WAIT, fp, ip,
1103 RELOAD_EX_POSITION(ex_),
1104 goto cat(eval_skip_,ARG_MODE));
1105 /*
1106 if (unlikely(da_tag(result) == DATA_TAG_array_incomplete)) {
1107 ptr = &da(result,array_incomplete)->next;
1108 goto cat(eval_again_,ARG_MODE);
1109 }
1110 */
1111 cat(eval_skip_,ARG_MODE):;
1112 }
1113 ADVANCE_I_PARAM(1);
1114 #endif
1115 DEFINE_OPCODE_END(OPCODE_EVAL)
1118 DEFINE_OPCODE_START(OPCODE_ESCAPE_NONFLAT)
1119 #ifdef EMIT_CODE
1120 frame_t n = get_i_param(0);
1121 ADVANCE_I_PARAM(1);
1122 ADVANCE_IP(n * ARG_MODE);
1123 #endif
1124 DEFINE_OPCODE_END(OPCODE_ESCAPE_NONFLAT)
1126 DEFINE_OPCODE_START(OPCODE_CHECKPOINT)
1127 #ifdef EMIT_CODE
1128 const code_t attr_unused *orig_ip = ip;
1129 frame_t attr_unused id;
1130 ip_t len;
1131 if (SIZEOF_IP_T == 2) {
1132 id = ip[1];
1133 ADVANCE_IP(2);
1134 } else if (SIZEOF_IP_T == 4) {
1135 id = get_unaligned_32(&ip[1]);
1136 ADVANCE_IP(3);
1137 } else {
1138 not_reached();
1139 }
1140 len = get_param(ip, 0);
1141 ADVANCE_IP((len + 1) * ARG_MODE);
1143 if (unlikely(tick_elapsed(&ts)) && likely(frame_execution_control(fp) != NULL)) {
1144 ipret_checkpoint_forced;
1145 } else {
1146 #ifdef HAVE_CODEGEN
1147 struct data *fn = get_frame(fp)->function;
1148 code_return_t r;
1149 void *new_fp;
1150 ip_t new_ip;
1151 struct data *codegen;
1152 if (unlikely(load_relaxed(&da(fn,function)->codegen_failed)))
1153 goto cat(checkpoint_exit_,ARG_MODE);
1154 #if 1
1155 pointer_follow(&da(fn,function)->codegen, false, codegen, PF_WAIT, fp, orig_ip,
1156 RELOAD_EX_POSITION(ex_),
1157 store_relaxed(&da(fn,function)->codegen_failed, 1);
1158 goto cat(checkpoint_exit_,ARG_MODE);
1159 );
1160 #else
1161 pointer_follow(&da(fn,function)->codegen, false, codegen, PF_SPARK, NULL, 0,
1162 /*debug("sparked: %s %p", da(fn,function)->function_name, ex_);*/
1163 SUBMIT_EX(ex_);
1164 goto cat(checkpoint_exit_,ARG_MODE),
1165 store_relaxed(&da(fn,function)->codegen_failed, 1);
1166 goto cat(checkpoint_exit_,ARG_MODE);
1167 );
1168 #endif
1170 /*for (frame_t l = MIN_USEABLE_SLOT; l < function_n_variables(fn); l++) {
1171 if (da(fn,function)->local_variables_flags[l].must_be_flat) {
1172 if (unlikely(frame_test_flag(fp, l)))
1173 goto cat(checkpoint_exit_,ARG_MODE);
1174 }
1175 }*/
1176 /*for (frame_t l = MIN_USEABLE_SLOT; l < function_n_variables(fn); l++) {
1177 if (da(fn,function)->local_variables_flags[l].must_be_data) {
1178 if (unlikely(pointer_is_thunk(*frame_pointer(fp, l))))
1179 goto cat(checkpoint_exit_,ARG_MODE);
1180 }
1181 }*/
1183 /*debug("running optimized code: %s, %p, %x, %p, %p, %p, %u, %p, %lx", da(fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, id, da(codegen,codegen)->unoptimized_code[id], fp, &cg_upcall_vector, ts, da(codegen,codegen)->unoptimized_code[id], (unsigned long)((char *)da(codegen,codegen)->unoptimized_code[id] - (char *)da(codegen,codegen)->unoptimized_code_base));*/
1184 /*debug("fp[0] = %lx", *(long *)((char *)fp + 0));
1185 debug("fp[8] = %lx", *(long *)((char *)fp + 8));
1186 debug("fp[16] = %lx", *(long *)((char *)fp + 16));
1187 debug("fp[24] = %lx", *(long *)((char *)fp + 24));
1188 debug("fp[32] = %lx", *(long *)((char *)fp + 32));
1189 debug("fp[40] = %lx", *(long *)((char *)fp + 40));*/
1190 /*{
1191 uintptr_t *stub = (void *)da(codegen,codegen)->unoptimized_code[id];
1192 debug("entry: %p, %lx %lx %lx %lx", stub, stub[0], stub[1], stub[2], stub[3]);
1193 }*/
1194 /*__asm__ volatile("nopr %r6");*/
1195 /*__asm__ volatile("xnop");*/
1196 /*__asm__ volatile("cover");*/
1197 /*debug("calling: %p, %p, %lx, %lx", da(codegen,codegen)->unoptimized_code[id], codegen_entry, ((long *)codegen_entry)[0], ((long *)codegen_entry)[1]);*/
1198 /*debug("calling: %p, %p, %lx, %lx", da(codegen,codegen)->unoptimized_code[id], codegen_entry, ((long *)((char *)codegen_entry - 2))[0], ((long *)((char *)codegen_entry - 2))[1]);*/
1199 r = codegen_entry(fp, &cg_upcall_vector, ts, da(codegen,codegen)->unoptimized_code[id]);
1200 #if defined(ARCH_X86_32) || defined(ARCH_ARM32) || defined(ARCH_MIPS32) || defined(ARCH_POWER32) || defined(ARCH_SPARC32)
1201 #if defined(C_LITTLE_ENDIAN) || defined(ARCH_MIPS_N32)
1202 new_fp = num_to_ptr(r & 0xffffffffU);
1203 new_ip = r >> 32;
1204 #else
1205 new_fp = num_to_ptr(r >> 32);
1206 new_ip = r & 0xffffffffU;
1207 #endif
1208 #else
1209 new_fp = r.fp;
1210 new_ip = r.ip;
1211 #endif
1212 /*debug("ran optimized code: %s, %p, %lx -> %p, %x", da(fn,function)->function_name, fp, (unsigned long)(ip - da(fn,function)->code), new_fp, new_ip);*/
1213 if (unlikely(new_ip == (ip_t)-1)) {
1214 /*debug("reload on optimized code: %p", new_fp);*/
1215 RELOAD_EX_POSITION(new_fp);
1216 } else {
1217 /*if (fp != new_fp) debug("switching function: %s -> %s", da(fn,function)->function_name, da(get_frame(new_fp)->function,function)->function_name);*/
1218 fp = new_fp;
1219 ip = da(get_frame(fp)->function,function)->code + new_ip;
1220 if (unlikely(profiling_escapes)) {
1221 profile_counter_t profiling_counter;
1222 fn = get_frame(fp)->function;
1223 profiling_counter = load_relaxed(&da(fn,function)->escape_data[new_ip].counter);
1224 profiling_counter++;
1225 store_relaxed(&da(fn,function)->escape_data[new_ip].counter, profiling_counter);
1226 }
1227 }
1228 #endif
1229 goto cat(checkpoint_exit_,ARG_MODE);
1230 cat(checkpoint_exit_,ARG_MODE):;
1231 }
1232 #endif
1233 DEFINE_OPCODE_END(OPCODE_CHECKPOINT)
1236 #if ARG_MODE == 0
1237 DEFINE_OPCODE_START(OPCODE_JMP)
1238 #ifdef EMIT_CODE
1239 if (SIZEOF_IP_T == 2) {
1240 int16_t offset = ip[1];
1241 ADVANCE_IP(2);
1242 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1243 } else if (SIZEOF_IP_T == 4) {
1244 int32_t offset = get_unaligned_32(&ip[1]);
1245 ADVANCE_IP(3);
1246 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1247 } else {
1248 not_reached();
1249 }
1250 #endif
1251 DEFINE_OPCODE_END(OPCODE_JMP)
1252 #endif
1254 #if ARG_MODE == 0 && SIZEOF_IP_T > 2
1255 DEFINE_OPCODE_START(OPCODE_JMP_BACK_16)
1256 #ifdef EMIT_CODE
1257 uint16_t offset;
1259 offset = ip[1];
1260 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) - offset);
1261 #endif
1262 DEFINE_OPCODE_END(OPCODE_JMP_BACK_16)
1263 #endif
1265 DEFINE_OPCODE_START(OPCODE_JMP_FALSE)
1266 #ifdef EMIT_CODE
1267 int32_t offset;
1268 frame_t slot;
1269 ajla_flat_option_t val1;
1271 slot = get_i_param(0);
1272 if (unlikely(frame_test_flag(fp, slot))) {
1273 void *ex = thunk_bool_jump(fp, ip, slot);
1274 if (ex != POINTER_FOLLOW_THUNK_RETRY) {
1275 if (likely(ex != POINTER_FOLLOW_THUNK_EXCEPTION)) {
1276 RELOAD_EX_POSITION(ex);
1277 } else {
1278 if (SIZEOF_IP_T == 2) {
1279 offset = ip[1 + param_size(1) + 1];
1280 ADVANCE_I_PARAM(1);
1281 ADVANCE_IP(2);
1282 } else if (SIZEOF_IP_T == 4) {
1283 offset = get_unaligned_32(&ip[1 + param_size(1) + 2]);
1284 ADVANCE_I_PARAM(1);
1285 ADVANCE_IP(4);
1286 } else {
1287 not_reached();
1288 }
1289 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1290 goto cat(jmp_false_exception_,ARG_MODE);
1291 }
1292 }
1293 }
1294 barrier_aliasing();
1295 val1 = *frame_slot(fp, slot, ajla_flat_option_t);
1296 barrier_aliasing();
1297 if (SIZEOF_IP_T == 2) {
1298 offset = ip[1 + param_size(1)];
1299 ADVANCE_I_PARAM(1);
1300 ADVANCE_IP(2);
1301 } else if (SIZEOF_IP_T == 4) {
1302 offset = get_unaligned_32(&ip[1 + param_size(1)]);
1303 ADVANCE_I_PARAM(1);
1304 ADVANCE_IP(4);
1305 } else {
1306 not_reached();
1307 }
1308 if (!val1)
1309 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1311 cat(jmp_false_exception_,ARG_MODE):;
1312 #endif
1313 DEFINE_OPCODE_END(OPCODE_JMP_FALSE)
1315 #if ARG_MODE == 0
1316 DEFINE_OPCODE_START(OPCODE_LABEL)
1317 #ifdef EMIT_CODE
1318 ADVANCE_IP(1);
1319 #endif
1320 DEFINE_OPCODE_END(OPCODE_LABEL)
1321 #endif
1324 START_BLOCK(
1325 pointer_t *direct_data;
1326 pointer_t indirect_data;
1327 arg_t n_arguments;
1328 frame_t result_slot;
1329 struct thunk *function_error;
1330 )
1332 DEFINE_OPCODE_START(OPCODE_LOAD_FN)
1333 #ifdef EMIT_CODE
1334 struct data *new_reference;
1335 pointer_t result_ptr;
1337 n_arguments = get_i_param(0);
1338 result_slot = get_i_param(1);
1339 direct_data = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1340 indirect_data = pointer_empty(); /* avoid warning */
1342 ADVANCE_I_PARAM(3);
1344 cat(fn_copy_arguments_,ARG_MODE):
1346 new_reference = data_alloc_function_reference_mayfail(n_arguments, &ajla_error pass_file_line);
1347 if (unlikely(!new_reference)) {
1348 function_error = thunk_alloc_exception_error(ajla_error, NULL, fp, ip pass_file_line);
1349 if (!direct_data)
1350 pointer_dereference(indirect_data);
1351 /*cat(fn_set_error_,ARG_MODE):*/
1352 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_arguments);
1353 result_ptr = pointer_thunk(function_error);
1354 } else {
1355 arg_t i;
1356 if (direct_data) {
1357 da(new_reference,function_reference)->is_indirect = false;
1358 da(new_reference,function_reference)->u.direct = direct_data;
1359 } else {
1360 da(new_reference,function_reference)->is_indirect = true;
1361 da(new_reference,function_reference)->u.indirect = indirect_data;
1362 }
1363 for (i = 0; i < n_arguments; i++) {
1364 ipret_fill_function_reference_from_slot(new_reference, i, fp, get_param(ip, 0), (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1365 ADVANCE_IP(param_size(2));
1366 }
1367 result_ptr = pointer_data(new_reference);
1368 }
1370 frame_set_pointer(fp, result_slot, result_ptr);
1371 #endif
1372 DEFINE_OPCODE_END(OPCODE_LOAD_FN)
1374 DEFINE_OPCODE_START(OPCODE_CURRY)
1375 #ifdef EMIT_CODE
1376 frame_t fn_ref_slot;
1378 n_arguments = get_i_param(0);
1379 result_slot = get_i_param(1);
1381 fn_ref_slot = get_i_param(2);
1382 direct_data = NULL;
1383 indirect_data = frame_get_pointer_reference(fp, fn_ref_slot, (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1385 ADVANCE_I_PARAM(4);
1387 goto cat(fn_copy_arguments_,ARG_MODE);
1388 #endif
1389 DEFINE_OPCODE_END(OPCODE_CURRY)
1391 END_BLOCK()
1393 START_BLOCK(
1394 arg_t n_arguments;
1395 arg_t n_return_values;
1396 struct thunk *function_error;
1397 struct data *function;
1398 const code_t *orig_ip;
1399 frame_s *new_fp;
1400 arg_t i;
1401 pointer_t *direct_function;
1402 pointer_t indirect_function;
1403 unsigned char mode;
1404 )
1405 DEFINE_OPCODE_START(OPCODE_CALL)
1406 #ifdef EMIT_CODE
1407 pointer_t *function_ptr;
1409 mode = CALL_MODE_NORMAL;
1410 cat(call_from_mode_,ARG_MODE):
1412 n_arguments = get_i_param(0);
1413 n_return_values = get_i_param(1);
1414 function_ptr = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1416 pointer_follow(function_ptr, false, function, PF_WAIT, fp, ip,
1417 RELOAD_EX_POSITION(ex_),
1418 function_error = thunk_;
1419 thunk_reference(function_error);
1420 ADVANCE_I_PARAM(3);
1421 goto cat(call_set_error_,ARG_MODE);
1422 );
1424 ajla_assert(da(function,function)->n_arguments == n_arguments && da(function,function)->n_return_values == n_return_values,
1425 (file_line, "call %s->%s: the number of arguments does not match: %lu != %lu || %lu != %lu",
1426 da(get_frame(fp)->function,function)->function_name, da(function,function)->function_name,
1427 (unsigned long)da(function,function)->n_arguments, (unsigned long)n_arguments,
1428 (unsigned long)da(function,function)->n_return_values, (unsigned long)n_return_values));
1430 ADVANCE_I_PARAM(3);
1432 new_fp = frame_build(fp, function, &ajla_error);
1433 if (unlikely(!new_fp)) {
1434 cat(call_allocation_error_,ARG_MODE):
1435 function_error = thunk_alloc_exception_error(ajla_error, NULL, fp, ip pass_file_line);
1436 cat(call_set_error_,ARG_MODE):
1437 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_arguments);
1438 pointer_reference_owned_multiple(pointer_thunk(function_error), n_return_values - 1);
1439 i = n_return_values;
1440 while (1) {
1441 frame_t slot_r = get_max_param(ip, 0);
1442 ADVANCE_IP(max_param_size(1) + 1);
1443 frame_set_pointer(fp, slot_r, pointer_thunk(function_error));
1444 if (!--i)
1445 break;
1446 }
1447 } else {
1448 fp = frame_up(new_fp);
1449 frame_init(new_fp, function, get_frame(fp)->timestamp, mode);
1451 i = 0;
1452 cat(call_copy_arguments_,ARG_MODE):
1453 for (; i < da(function,function)->n_arguments; i++) {
1454 frame_t src_slot = get_param(ip, 0);
1455 frame_t dst_slot = da(function,function)->args[i].slot;
1456 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1457 bool may_be_borrowed = da(function,function)->args[i].may_be_borrowed;
1458 if (may_be_borrowed && src_flag & OPCODE_CALL_MAY_LEND && !pointer_is_thunk(*frame_pointer(fp, src_slot))) {
1459 *frame_pointer(new_fp, dst_slot) = *frame_pointer(fp, src_slot);
1460 } else if (may_be_borrowed && src_flag & OPCODE_CALL_MAY_GIVE && !frame_test_flag(fp, src_slot) && !pointer_is_thunk(*frame_pointer(fp, src_slot))) {
1461 *frame_pointer(new_fp, dst_slot) = *frame_pointer(fp, src_slot);
1462 *frame_pointer(fp, src_slot) = pointer_empty();
1463 } else {
1464 ipret_copy_variable(fp, src_slot, new_fp, dst_slot, (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1465 }
1466 ADVANCE_IP(param_size(2));
1467 }
1468 get_frame(new_fp)->previous_ip = frame_ip(fp, ip);
1470 fp = new_fp;
1471 ip = &da(function,function)->code[0];
1473 if (unlikely(mode == CALL_MODE_SPARK)) {
1474 ipret_checkpoint_forced;
1475 }
1476 }
1477 #endif
1478 DEFINE_OPCODE_END(OPCODE_CALL)
1480 DEFINE_OPCODE_START(OPCODE_CALL_STRICT)
1481 #ifdef EMIT_CODE
1482 mode = CALL_MODE_STRICT;
1483 goto cat(call_from_mode_,ARG_MODE);
1484 #endif
1485 DEFINE_OPCODE_END(OPCODE_CALL_STRICT)
1487 DEFINE_OPCODE_START(OPCODE_CALL_SPARK)
1488 #ifdef EMIT_CODE
1489 mode = CALL_MODE_SPARK;
1490 goto cat(call_from_mode_,ARG_MODE);
1491 #endif
1492 DEFINE_OPCODE_END(OPCODE_CALL_SPARK)
1494 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT)
1495 #ifdef EMIT_CODE
1496 frame_t fn_ref_slot;
1497 bool deref;
1498 pointer_t *ptr;
1500 mode = CALL_MODE_NORMAL;
1501 cat(call_indirect_from_mode_,ARG_MODE):
1503 orig_ip = ip;
1504 n_arguments = get_i_param(0);
1505 n_return_values = get_i_param(1);
1506 fn_ref_slot = get_i_param(2);
1507 deref = (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1508 ADVANCE_I_PARAM(4);
1510 ptr = frame_pointer(fp, fn_ref_slot);
1511 while (1) {
1512 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1513 RELOAD_EX_POSITION(ex_),
1514 function_error = thunk_;
1515 thunk_reference(function_error);
1516 if (deref)
1517 frame_free_and_clear(fp, fn_ref_slot);
1518 goto cat(call_set_error_,ARG_MODE)
1519 );
1520 if (!da(function,function_reference)->is_indirect)
1521 break;
1522 ptr = &da(function,function_reference)->u.indirect;
1523 }
1524 ptr = da(function,function_reference)->u.direct;
1525 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1526 RELOAD_EX_POSITION(ex_),
1527 function_error = thunk_;
1528 thunk_reference(function_error);
1529 if (deref)
1530 frame_free_and_clear(fp, fn_ref_slot);
1531 goto cat(call_set_error_,ARG_MODE)
1532 );
1534 new_fp = frame_build(fp, function, &ajla_error);
1535 if (unlikely(!new_fp)) {
1536 if (deref)
1537 frame_free_and_clear(fp, fn_ref_slot);
1538 goto cat(call_allocation_error_,ARG_MODE);
1539 } else {
1540 fp = frame_up(new_fp);
1541 frame_init(new_fp, function, get_frame(fp)->timestamp, mode);
1543 i = da(function,function)->n_arguments - n_arguments;
1545 copy_from_function_reference_to_frame(new_fp, pointer_get_data(*frame_pointer(fp, fn_ref_slot)), i, deref && frame_test_flag(fp, fn_ref_slot));
1546 if (deref)
1547 frame_free_and_clear(fp, fn_ref_slot);
1549 goto cat(call_copy_arguments_,ARG_MODE);
1550 }
1551 #endif
1552 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT)
1554 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_STRICT)
1555 #ifdef EMIT_CODE
1556 mode = CALL_MODE_STRICT;
1557 goto cat(call_indirect_from_mode_,ARG_MODE);
1558 #endif
1559 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_STRICT)
1561 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_SPARK)
1562 #ifdef EMIT_CODE
1563 mode = CALL_MODE_SPARK;
1564 goto cat(call_indirect_from_mode_,ARG_MODE);
1565 #endif
1566 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_SPARK)
1568 DEFINE_OPCODE_START(OPCODE_CALL_LAZY)
1569 #ifdef EMIT_CODE
1570 struct data *function_reference;
1571 struct thunk **results;
1573 n_arguments = get_i_param(0);
1574 n_return_values = get_i_param(1);
1575 direct_function = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1576 indirect_function = pointer_empty(); /* avoid warning */
1578 ADVANCE_I_PARAM(3);
1580 cat(call_lazy_copy_arguments_,ARG_MODE):
1581 results = mem_alloc_array_mayfail(mem_alloc_mayfail, struct thunk **, 0, 0, n_return_values, sizeof(struct thunk *), &ajla_error);
1582 if (unlikely(!results)) {
1583 if (!direct_function)
1584 pointer_dereference(indirect_function);
1585 goto cat(call_allocation_error_,ARG_MODE);
1586 }
1588 if (!(function_reference = data_alloc_function_reference_mayfail(n_arguments, &ajla_error pass_file_line))) {
1589 if (!direct_function)
1590 pointer_dereference(indirect_function);
1591 mem_free(results);
1592 goto cat(call_allocation_error_,ARG_MODE);
1593 }
1594 if (direct_function) {
1595 da(function_reference,function_reference)->is_indirect = false;
1596 da(function_reference,function_reference)->u.direct = direct_function;
1597 } else {
1598 da(function_reference,function_reference)->is_indirect = true;
1599 da(function_reference,function_reference)->u.indirect = indirect_function;
1600 }
1602 if (!thunk_alloc_function_call(pointer_data(function_reference), n_return_values, results, &ajla_error)) {
1603 if (!direct_function)
1604 pointer_dereference(indirect_function);
1605 mem_free(results);
1606 data_free_r1(function_reference);
1607 goto cat(call_allocation_error_,ARG_MODE);
1608 }
1610 for (i = 0; i < n_arguments; i++) {
1611 frame_t src_slot = get_param(ip, 0);
1612 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1613 ipret_fill_function_reference_from_slot(function_reference, i, fp, src_slot, (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1614 ADVANCE_IP(param_size(2));
1615 }
1617 for (i = 0; i < n_return_values; i++) {
1618 frame_t slot_r = get_max_param(ip, 0);
1619 frame_set_pointer(fp, slot_r, pointer_thunk(results[i]));
1620 ADVANCE_IP(max_param_size(1) + 1);
1621 }
1623 mem_free(results);
1624 #endif
1625 DEFINE_OPCODE_END(OPCODE_CALL_LAZY)
1627 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_LAZY)
1628 #ifdef EMIT_CODE
1629 frame_t fn_ref_slot;
1630 bool deref;
1632 n_arguments = get_i_param(0);
1633 n_return_values = get_i_param(1);
1634 fn_ref_slot = get_i_param(2);
1635 deref = (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1636 ADVANCE_I_PARAM(4);
1638 indirect_function = frame_get_pointer_reference(fp, fn_ref_slot, deref);
1639 direct_function = NULL;
1641 goto cat(call_lazy_copy_arguments_,ARG_MODE);
1642 #endif
1643 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_LAZY)
1645 DEFINE_OPCODE_START(OPCODE_CALL_CACHE)
1646 #ifdef EMIT_CODE
1647 struct ipret_call_cache_arg *arguments;
1648 frame_t *return_values;
1649 void *ex;
1651 cat(call_cache_,ARG_MODE):
1652 orig_ip = ip;
1653 n_arguments = get_i_param(0);
1654 n_return_values = get_i_param(1);
1655 direct_function = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1656 ADVANCE_I_PARAM(3);
1658 pointer_follow(direct_function, false, function, PF_WAIT, fp, orig_ip,
1659 RELOAD_EX_POSITION(ex_),
1660 function_error = thunk_;
1661 thunk_reference(function_error);
1662 goto cat(call_set_error_,ARG_MODE);
1663 );
1664 ajla_assert(da(function,function)->n_arguments == n_arguments && da(function,function)->n_return_values == n_return_values,
1665 (file_line, "call_cache: the number of arguments does not match: %lu != %lu || %lu != %lu",
1666 (unsigned long)da(function,function)->n_arguments, (unsigned long)n_arguments,
1667 (unsigned long)da(function,function)->n_return_values, (unsigned long)n_return_values));
1669 arguments = mem_alloc_array_mayfail(mem_alloc_mayfail, struct ipret_call_cache_arg *, 0, 0, n_arguments, sizeof(struct ipret_call_cache_arg), &ajla_error);
1670 if (!arguments) {
1671 goto cat(call_allocation_error_,ARG_MODE);
1672 }
1673 return_values = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n_return_values, sizeof(frame_t), &ajla_error);
1674 if (!return_values) {
1675 mem_free(arguments);
1676 goto cat(call_allocation_error_,ARG_MODE);
1677 }
1679 for (i = 0; i < n_arguments; i++) {
1680 frame_t src_slot = get_param(ip, 0);
1681 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1682 arguments[i].f_arg = NULL;
1683 arguments[i].slot = src_slot;
1684 arguments[i].deref = (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1685 ADVANCE_IP(param_size(2));
1686 }
1688 for (i = 0; i < n_return_values; i++) {
1689 frame_t slot_r = get_max_param(ip, 0);
1690 return_values[i] = slot_r;
1691 ADVANCE_IP(max_param_size(1) + 1);
1692 }
1694 ex = ipret_call_cache(fp, orig_ip, direct_function, arguments, return_values, NO_FRAME_T);
1695 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
1696 RELOAD_EX_POSITION(ex);
1697 }
1698 #endif
1699 DEFINE_OPCODE_END(OPCODE_CALL_CACHE)
1701 DEFINE_OPCODE_START(OPCODE_CALL_SAVE)
1702 #ifdef EMIT_CODE
1703 goto cat(call_cache_,ARG_MODE);
1704 #endif
1705 DEFINE_OPCODE_END(OPCODE_CALL_SAVE)
1707 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_CACHE)
1708 #ifdef EMIT_CODE
1709 struct ipret_call_cache_arg *arguments;
1710 frame_t *return_values;
1711 void *ex;
1713 frame_t fn_ref_slot;
1714 bool deref;
1715 pointer_t *ptr;
1716 arg_t n_curried_arguments;
1718 cat(call_indirect_cache_,ARG_MODE):
1719 orig_ip = ip;
1720 n_arguments = get_i_param(0);
1721 n_return_values = get_i_param(1);
1722 fn_ref_slot = get_i_param(2);
1723 deref = (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1724 ADVANCE_I_PARAM(4);
1726 n_curried_arguments = 0;
1727 ptr = frame_pointer(fp, fn_ref_slot);
1728 while (1) {
1729 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1730 RELOAD_EX_POSITION(ex_),
1731 function_error = thunk_;
1732 thunk_reference(function_error);
1733 if (deref)
1734 frame_free_and_clear(fp, fn_ref_slot);
1735 goto cat(call_set_error_,ARG_MODE)
1736 );
1737 n_curried_arguments += da(function,function_reference)->n_curried_arguments;
1738 if (!da(function,function_reference)->is_indirect)
1739 break;
1740 ptr = &da(function,function_reference)->u.indirect;
1741 }
1742 ptr = da(function,function_reference)->u.direct;
1743 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1744 RELOAD_EX_POSITION(ex_),
1745 function_error = thunk_;
1746 thunk_reference(function_error);
1747 if (deref)
1748 frame_free_and_clear(fp, fn_ref_slot);
1749 goto cat(call_set_error_,ARG_MODE)
1750 );
1752 arguments = mem_alloc_array_mayfail(mem_alloc_mayfail, struct ipret_call_cache_arg *, 0, 0, n_curried_arguments + n_arguments, sizeof(struct ipret_call_cache_arg), &ajla_error);
1753 if (!arguments) {
1754 if (deref)
1755 frame_free_and_clear(fp, fn_ref_slot);
1756 goto cat(call_allocation_error_,ARG_MODE);
1757 }
1758 return_values = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n_return_values, sizeof(frame_t), &ajla_error);
1759 if (!return_values) {
1760 mem_free(arguments);
1761 if (deref)
1762 frame_free_and_clear(fp, fn_ref_slot);
1763 goto cat(call_allocation_error_,ARG_MODE);
1764 }
1766 i = n_curried_arguments;
1767 function = pointer_get_data(*frame_pointer(fp, fn_ref_slot));
1768 while (1) {
1769 arg_t j;
1770 j = da(function,function_reference)->n_curried_arguments;
1771 while (j--) {
1772 i--;
1773 arguments[i].f_arg = &da(function,function_reference)->arguments[j];
1774 arguments[i].deref = false;
1775 }
1776 if (!da(function,function_reference)->is_indirect)
1777 break;
1778 function = pointer_get_data(da(function,function_reference)->u.indirect);
1779 }
1781 for (i = n_curried_arguments; i < n_curried_arguments + n_arguments; i++) {
1782 frame_t src_slot = get_param(ip, 0);
1783 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1784 arguments[i].f_arg = NULL;
1785 arguments[i].slot = src_slot;
1786 arguments[i].deref = (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1787 ADVANCE_IP(param_size(2));
1788 }
1790 for (i = 0; i < n_return_values; i++) {
1791 frame_t slot_r = get_max_param(ip, 0);
1792 return_values[i] = slot_r;
1793 ADVANCE_IP(max_param_size(1) + 1);
1794 }
1796 ex = ipret_call_cache(fp, orig_ip, ptr, arguments, return_values, deref ? fn_ref_slot : NO_FRAME_T);
1797 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
1798 RELOAD_EX_POSITION(ex);
1799 }
1800 #endif
1801 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_CACHE)
1803 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_SAVE)
1804 #ifdef EMIT_CODE
1805 goto cat(call_indirect_cache_,ARG_MODE);
1806 #endif
1807 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_SAVE)
1809 END_BLOCK()
1811 DEFINE_OPCODE_START(OPCODE_RETURN)
1812 #ifdef EMIT_CODE
1813 struct data *function;
1814 frame_s *previous_fp;
1815 const code_t *previous_ip;
1816 arg_t i, n;
1818 /*ADVANCE_I_PARAM(0);*/
1820 function = get_frame(fp)->function;
1821 previous_fp = frame_up(fp);
1822 if (unlikely(frame_is_top(previous_fp))) {
1823 struct execution_control *ex;
1824 void *exx;
1825 struct thunk *t;
1826 struct stack_bottom *sb = frame_stack_bottom(previous_fp);
1827 ex = sb->ex;
1828 if (!ex) {
1829 frame_t src_slot = get_i_param(0);
1830 sb->ret = ipret_copy_variable_to_pointer(fp, src_slot, true);
1831 goto exit_ipret;
1832 }
1833 t = ex->thunk;
1834 n = da(function,function)->n_return_values;
1835 i = 0;
1836 do {
1837 frame_t src_slot = get_i_param(i * 2);
1838 frame_t flags = get_i_param(i * 2 + 1);
1839 t->u.function_call.results[i].ptr = ipret_copy_variable_to_pointer(fp, src_slot, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1840 } while (++i < n);
1841 exx = thunk_terminate(t, n);
1842 #if 0
1843 if (exx != POINTER_FOLLOW_THUNK_EXIT)
1844 task_submit(exx, true);
1845 goto exit_ipret;
1846 #else
1847 RELOAD_EX_POSITION(exx);
1848 #endif
1849 }
1851 get_frame(previous_fp)->timestamp = get_frame(fp)->timestamp;
1852 previous_ip = &da(get_frame(previous_fp)->function,function)->code[get_frame(fp)->previous_ip];
1854 n = da(function,function)->n_return_values;
1855 i = 0;
1856 do {
1857 frame_t src_slot, dst_slot, flags;
1858 src_slot = get_i_param(i * 2);
1859 flags = get_i_param(i * 2 + 1);
1860 dst_slot = (frame_t)get_max_param(previous_ip, 0);
1861 previous_ip += max_param_size(1) + 1;
1862 ipret_copy_variable(fp, src_slot, previous_fp, dst_slot, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1863 } while (++i < n);
1865 fp = previous_fp;
1866 ip = previous_ip;
1867 #endif
1868 DEFINE_OPCODE_END(OPCODE_RETURN)
1871 DEFINE_OPCODE_START(OPCODE_STRUCTURED)
1872 #ifdef EMIT_CODE
1873 const code_t *orig_ip = ip;
1875 frame_t struc, elem;
1876 pointer_t *struc_ptr;
1877 unsigned char *struc_flat;
1878 const struct type *t;
1879 bool optimize_elem_is_flat;
1881 unsigned char type;
1882 frame_t arg;
1884 pointer_t thunk_ptr;
1886 elem = get_i_param(1);
1887 optimize_elem_is_flat = frame_variable_is_flat(fp, elem);
1889 cat(structured_retry_,ARG_MODE):
1890 struc = get_i_param(0);
1891 ADVANCE_I_PARAM(2);
1893 t = frame_get_type_of_local(fp, struc);
1894 if (frame_variable_is_flat(fp, struc)) {
1895 if (!optimize_elem_is_flat) {
1896 frame_set_pointer(fp, struc, flat_to_data(frame_get_type_of_local(fp, struc), frame_var(fp, struc)));
1897 struc_flat = NULL;
1898 struc_ptr = frame_pointer(fp, struc);
1899 } else {
1900 struc_flat = frame_var(fp, struc);
1901 struc_ptr = NULL; /* avoid warning */
1902 }
1903 } else {
1904 struc_flat = NULL;
1905 struc_ptr = frame_pointer(fp, struc);
1906 }
1908 do {
1909 array_index_t array_index = index_invalid(); /* avoid warning */
1910 struct data *data = NULL; /* avoid warning */
1912 type = (unsigned char)get_param(ip, 0);
1913 arg = get_param(ip, 1);
1914 ADVANCE_IP(param_size(3));
1916 if ((type & OPCODE_STRUCTURED_MASK) == OPCODE_STRUCTURED_ARRAY) {
1917 void *ex = ipret_get_index(fp, orig_ip, fp, arg, NULL, &array_index, &thunk_ptr pass_file_line);
1918 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
1919 if (ex == POINTER_FOLLOW_THUNK_EXCEPTION)
1920 goto cat(structured_error_,ARG_MODE);
1921 RELOAD_EX_POSITION(ex);
1922 }
1923 }
1925 if (struc_flat) {
1926 switch (type & OPCODE_STRUCTURED_MASK) {
1927 case OPCODE_STRUCTURED_RECORD: {
1928 const struct flat_record_definition_entry *e;
1929 ajla_assert(arg < flat_record_n_slots(type_def(t,flat_record)), (file_line, "structured flat record: invalid index: %"PRIuMAX" >= %"PRIuMAX"", (uintmax_t)arg, (uintmax_t)flat_record_n_slots(type_def(t,flat_record))));
1930 e = &type_def(t,flat_record)->entries[arg];
1931 t = e->subtype;
1932 struc_flat += e->flat_offset;
1933 break;
1934 }
1935 case OPCODE_STRUCTURED_OPTION: {
1936 thunk_ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_OPTION_DOESNT_MATCH), fp, ip pass_file_line);
1937 goto cat(structured_error_,ARG_MODE);
1938 }
1939 case OPCODE_STRUCTURED_ARRAY: {
1940 if (unlikely(index_ge_int(array_index, type_def(t,flat_array)->n_elements))) {
1941 index_free(&array_index);
1942 thunk_ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_INDEX_OUT_OF_RANGE), fp, ip pass_file_line);
1943 goto cat(structured_error_,ARG_MODE);
1944 }
1945 t = type_def(t,flat_array)->base;
1946 struc_flat += t->size * index_to_int(array_index);
1947 index_free(&array_index);
1948 break;
1949 }
1950 default:
1951 internal(file_line, "structured flat: invalid structure type %02x", type);
1952 }
1953 } else {
1954 cat(structured_restart_pointer_follow_,ARG_MODE):
1955 pointer_follow(struc_ptr, false, data, PF_WAIT, fp, orig_ip,
1956 if ((type & OPCODE_STRUCTURED_MASK) == OPCODE_STRUCTURED_ARRAY)
1957 index_free(&array_index);
1958 RELOAD_EX_POSITION(ex_),
1959 if ((type & OPCODE_STRUCTURED_MASK) == OPCODE_STRUCTURED_ARRAY)
1960 index_free(&array_index);
1961 goto cat(structured_read_to_the_end_,ARG_MODE);
1962 );
1963 if (unlikely(!data_is_writable(data))) {
1964 struct_clone(struc_ptr);
1965 goto cat(structured_restart_pointer_follow_,ARG_MODE);
1966 }
1967 switch (type & OPCODE_STRUCTURED_MASK) {
1968 case OPCODE_STRUCTURED_RECORD: {
1969 const struct record_definition *def = type_def(da(data,record)->definition,record);
1970 ajla_assert(arg < def->n_slots, (file_line, "structured record: invalid index: %"PRIuMAX" >= %"PRIuMAX"", (uintmax_t)arg, (uintmax_t)def->n_slots));
1971 t = def->types[arg];
1972 struc_ptr = frame_pointer(da_record_frame(data), arg);
1974 if (type & OPCODE_STRUCTURED_FLAG_END) {
1975 if (frame_variable_is_flat(fp, elem)) {
1976 if (TYPE_IS_FLAT(t)) {
1977 frame_free(da_record_frame(data), arg);
1978 struc_flat = frame_var(da_record_frame(data), arg);
1979 goto cat(structured_write_flat_,ARG_MODE);
1980 }
1981 } else {
1982 if (!frame_test_and_set_flag(da_record_frame(data), arg))
1983 goto cat(structured_write_ptr_,ARG_MODE);
1984 }
1985 }
1987 if (!frame_test_flag(da_record_frame(data), arg)) {
1988 struc_flat = frame_var(da_record_frame(data), arg);
1989 if (unlikely(!optimize_elem_is_flat)) {
1990 frame_set_pointer(da_record_frame(data), arg, flat_to_data(t, struc_flat));
1991 struc_flat = NULL;
1992 }
1993 }
1994 break;
1995 }
1996 case OPCODE_STRUCTURED_OPTION: {
1997 if (unlikely(da(data,option)->option != arg)) {
1998 if (likely((type & OPCODE_STRUCTURED_FLAG_END) != 0)) {
1999 da(data,option)->option = arg;
2000 } else {
2001 thunk_ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_OPTION_DOESNT_MATCH), fp, ip pass_file_line);
2002 cat(structured_error_,ARG_MODE):
2003 if (struc_flat) {
2004 pointer_dereference(thunk_ptr);
2005 optimize_elem_is_flat = false;
2006 ip = orig_ip;
2007 goto cat(structured_retry_,ARG_MODE);
2008 } else {
2009 pointer_dereference(*struc_ptr);
2010 *struc_ptr = thunk_ptr;
2011 goto cat(structured_read_to_the_end_,ARG_MODE);
2012 }
2013 }
2014 }
2015 struc_ptr = &da(data,option)->pointer;
2016 break;
2017 }
2018 case OPCODE_STRUCTURED_ARRAY: {
2019 unsigned array_flags;
2020 if (unlikely(da_tag(data) == DATA_TAG_array_incomplete)) {
2021 array_index_t len_first;
2023 if (array_incomplete_collapse(struc_ptr))
2024 goto cat(structured_restart_pointer_follow_,ARG_MODE);
2026 len_first = array_len(pointer_get_data(da(data,array_incomplete)->first));
2027 if (!index_ge_index(array_index, len_first)) {
2028 index_free(&len_first);
2029 struc_ptr = &da(data,array_incomplete)->first;
2030 goto cat(structured_restart_pointer_follow_,ARG_MODE);
2031 } else {
2032 index_sub(&array_index, len_first);
2033 index_free(&len_first);
2034 struc_ptr = &da(data,array_incomplete)->next;
2035 goto cat(structured_restart_pointer_follow_,ARG_MODE);
2036 }
2037 }
2038 if (!optimize_elem_is_flat) {
2039 array_flags = ARRAY_MODIFY_NEED_PTR;
2040 } else if (type & OPCODE_STRUCTURED_FLAG_END) {
2041 array_flags = ARRAY_MODIFY_NEED_FLAT;
2042 t = frame_get_type_of_local(fp, elem);
2043 } else {
2044 array_flags = 0;
2045 }
2046 if (unlikely(!array_modify(struc_ptr, array_index, array_flags, &struc_ptr, &struc_flat, &t, fp, ip))) {
2047 goto cat(structured_read_to_the_end_,ARG_MODE);
2048 }
2049 break;
2050 }
2051 default:
2052 internal(file_line, "structured: invalid structure type %02x", type);
2053 }
2054 }
2055 } while (!(type & OPCODE_STRUCTURED_FLAG_END));
2056 if (struc_flat) {
2057 cat(structured_write_flat_,ARG_MODE):
2058 memcpy_fast(struc_flat, frame_var(fp, elem), frame_get_type_of_local(fp, elem)->size);
2059 } else {
2060 /* The pointer may be empty if we moved from an option without data to an option with data */
2061 if (likely(!pointer_is_empty(*struc_ptr)))
2062 pointer_dereference(*struc_ptr);
2063 if (frame_variable_is_flat(fp, elem)) {
2064 *struc_ptr = flat_to_data(frame_get_type_of_local(fp, elem), frame_var(fp, elem));
2065 } else {
2066 cat(structured_write_ptr_,ARG_MODE):
2067 *struc_ptr = frame_get_pointer_reference(fp, elem, (type & OPCODE_STRUCTURED_FREE_VARIABLE) != 0);
2068 }
2069 }
2070 if (false) {
2071 do {
2072 type = (unsigned char)get_param(ip, 0);
2073 arg = get_param(ip, 1);
2074 ADVANCE_IP(param_size(3));
2075 cat(structured_read_to_the_end_,ARG_MODE):;
2076 } while (!(type & OPCODE_STRUCTURED_FLAG_END));
2077 if (type & OPCODE_STRUCTURED_FREE_VARIABLE)
2078 frame_free(fp, elem);
2079 }
2080 #endif
2081 DEFINE_OPCODE_END(OPCODE_STRUCTURED)
2083 DEFINE_OPCODE_START(OPCODE_RECORD_CREATE)
2084 #ifdef EMIT_CODE
2085 frame_t result_slot;
2086 arg_t n_entries, i, ii;
2087 const struct type *t;
2088 const struct record_definition *def;
2090 result_slot = get_i_param(0);
2091 n_entries = get_i_param(1);
2092 ADVANCE_I_PARAM(2);
2094 t = frame_get_type_of_local(fp, result_slot);
2095 if (t->tag == TYPE_TAG_flat_record) {
2096 const code_t *backup_ip;
2097 def = type_def(type_def(t,flat_record)->base,record);
2098 backup_ip = ip;
2099 for (i = 0, ii = 0; i < n_entries; i++, ii++) {
2100 frame_t var_slot, record_slot;
2101 flat_size_t flat_offset;
2102 const struct type *flat_type;
2104 while (unlikely(record_definition_is_elided(def, ii)))
2105 ii++;
2107 var_slot = get_param(ip, 0);
2109 if (unlikely(!frame_variable_is_flat(fp, var_slot))) {
2110 ip = backup_ip;
2111 goto cat(create_record_no_flat_,ARG_MODE);
2112 }
2113 record_slot = record_definition_slot(def, ii);
2114 flat_offset = type_def(t,flat_record)->entries[record_slot].flat_offset;
2115 flat_type = type_def(t,flat_record)->entries[record_slot].subtype;
2116 ajla_assert(type_is_equal(frame_get_type_of_local(fp, var_slot), flat_type), (file_line, "record create (flat): copying between different types (%u,%u,%u) -> (%u,%u,%u)", frame_get_type_of_local(fp, var_slot)->tag, frame_get_type_of_local(fp, var_slot)->size, frame_get_type_of_local(fp, var_slot)->align, flat_type->tag, flat_type->size, flat_type->align));
2117 memcpy_fast(frame_var(fp, result_slot) + flat_offset, frame_var(fp, var_slot), flat_type->size);
2118 ADVANCE_IP(param_size(2));
2119 }
2120 ajla_assert(!frame_test_flag(fp, result_slot), (file_line, "record create (flat): flag already set for destination slot %"PRIuMAX"", (uintmax_t)result_slot));
2121 } else {
2122 struct data *result;
2123 def = type_def(t,record);
2124 cat(create_record_no_flat_,ARG_MODE):
2125 result = data_alloc_record_mayfail(def, &ajla_error pass_file_line);
2126 if (unlikely(!result)) {
2127 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_entries);
2128 frame_set_pointer(fp, result_slot, pointer_error(ajla_error, fp, ip pass_file_line));
2129 break;
2130 }
2131 memset(da_record_frame(result), 0, bitmap_slots(def->n_slots) * slot_size);
2132 for (i = 0, ii = 0; i < n_entries; i++, ii++) {
2133 frame_t var_slot, record_slot;
2134 const struct type *rec_type, *var_type;
2136 while (unlikely(record_definition_is_elided(def, ii)))
2137 ii++;
2139 var_slot = get_param(ip, 0);
2140 record_slot = record_definition_slot(def, ii);
2141 rec_type = def->types[record_slot];
2142 var_type = frame_get_type_of_local(fp, var_slot);
2144 if (!frame_variable_is_flat(fp, var_slot)) {
2145 pointer_t ptr = frame_get_pointer_reference(fp, var_slot, (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2146 frame_set_pointer(da_record_frame(result), record_slot, ptr);
2147 } else if (TYPE_IS_FLAT(rec_type)) {
2148 ajla_assert(type_is_equal(var_type, rec_type), (file_line, "record create: copying between different types (%u,%u,%u) -> (%u,%u,%u)", var_type->tag, var_type->size, var_type->align, rec_type->tag, rec_type->size, rec_type->align));
2149 memcpy_fast(frame_var(da_record_frame(result), record_slot), frame_var(fp, var_slot), rec_type->size);
2150 } else {
2151 pointer_t ptr = flat_to_data(var_type, frame_var(fp, var_slot));
2152 frame_set_pointer(da_record_frame(result), record_slot, ptr);
2153 }
2155 ADVANCE_IP(param_size(2));
2156 }
2157 frame_set_pointer(fp, result_slot, pointer_data(result));
2158 }
2159 #endif
2160 DEFINE_OPCODE_END(OPCODE_RECORD_CREATE)
2162 DEFINE_OPCODE_START(OPCODE_RECORD_LOAD)
2163 #ifdef EMIT_CODE
2164 frame_t record, record_slot, result;
2165 unsigned flags;
2166 const struct type *t;
2167 record = get_i_param(0);
2168 record_slot = get_i_param(1);
2169 result = get_i_param(2);
2170 flags = get_i_param(3);
2172 t = frame_get_type_of_local(fp, record);
2173 if (t->tag == TYPE_TAG_flat_record && !frame_test_flag(fp, record)) {
2174 const struct flat_record_definition_entry *ft = &type_def(t,flat_record)->entries[record_slot];
2175 if (likely(TYPE_IS_FLAT(frame_get_type_of_local(fp, result)))) {
2176 memcpy_fast(frame_var(fp, result), frame_var(fp, record) + ft->flat_offset, ft->subtype->size);
2177 } else {
2178 pointer_t ptr = flat_to_data(ft->subtype, frame_var(fp, record) + ft->flat_offset);
2179 frame_set_pointer(fp, result, ptr);
2180 }
2181 } else {
2182 const struct type *rtype, *etype;
2183 struct data *data;
2184 frame_s *fr;
2185 pointer_follow(frame_pointer(fp, record), true, data, unlikely(flags & OPCODE_OP_FLAG_STRICT) ? PF_WAIT : PF_NOEVAL, fp, ip,
2186 if (!(flags & OPCODE_OP_FLAG_STRICT)) {
2187 ex_ = ipret_record_load_create_thunk(fp, ip, record, record_slot, result);
2188 if (ex_ == POINTER_FOLLOW_THUNK_GO)
2189 goto cat(record_load_end_,ARG_MODE);
2190 }
2191 RELOAD_EX_POSITION(ex_),
2192 thunk_reference(thunk_);
2193 frame_set_pointer(fp, result, pointer_thunk(thunk_));
2194 goto cat(record_load_end_,ARG_MODE);
2195 );
2196 rtype = da(data,record)->definition;
2197 etype = type_def(rtype,record)->types[record_slot];
2198 fr = da_record_frame(data);
2199 if (!frame_test_flag(fr, record_slot)) {
2200 if (likely(TYPE_IS_FLAT(frame_get_type_of_local(fp, result)))) {
2201 memcpy_fast(frame_var(fp, result), frame_var(fr, record_slot), etype->size);
2202 } else {
2203 pointer_t ptr = flat_to_data(etype, frame_var(fr, record_slot));
2204 frame_set_pointer(fp, result, ptr);
2205 }
2206 } else {
2207 pointer_reference_maybe(fp, result, frame_pointer(fr, record_slot), flags);
2208 }
2209 }
2211 cat(record_load_end_,ARG_MODE):
2212 ADVANCE_I_PARAM(4);
2213 #endif
2214 DEFINE_OPCODE_END(OPCODE_RECORD_CREATE)
2216 DEFINE_OPCODE_START(OPCODE_OPTION_CREATE_EMPTY_FLAT)
2217 #ifdef EMIT_CODE
2218 frame_t slot_r = get_i_param(0);
2219 ajla_flat_option_t opt = (ajla_flat_option_t)get_i_param(1);
2220 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "option create empty flat: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
2221 barrier_aliasing();
2222 *frame_slot(fp, slot_r, ajla_flat_option_t) = opt;
2223 barrier_aliasing();
2224 ADVANCE_I_PARAM(2);
2225 #endif
2226 DEFINE_OPCODE_END(OPCODE_OPTION_CREATE_EMPTY_FLAT)
2228 DEFINE_OPCODE_START(OPCODE_OPTION_CREATE_EMPTY)
2229 #ifdef EMIT_CODE
2230 struct data *d;
2231 frame_t slot_r;
2233 d = data_alloc(option, &ajla_error);
2234 slot_r = get_i_param(0);
2235 if (likely(d != NULL)) {
2236 da(d,option)->option = get_i_param(1);
2237 da(d,option)->pointer = pointer_empty();
2238 frame_set_pointer(fp, slot_r, pointer_data(d));
2239 } else {
2240 frame_set_pointer(fp, slot_r, pointer_error(ajla_error, fp, ip pass_file_line));
2241 }
2242 ADVANCE_I_PARAM(2);
2243 #endif
2244 DEFINE_OPCODE_END(OPCODE_OPTION_CREATE_EMPTY)
2246 DEFINE_OPCODE_START(OPCODE_OPTION_CREATE)
2247 #ifdef EMIT_CODE
2248 struct data *d;
2249 frame_t slot_1, slot_r;
2250 pointer_t ptr;
2252 slot_1 = get_i_param(2);
2253 if (!frame_variable_is_flat(fp, slot_1)) {
2254 ptr = frame_get_pointer_reference(fp, slot_1, (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2255 } else {
2256 ptr = flat_to_data(frame_get_type_of_local(fp, slot_1), frame_var(fp, slot_1));
2257 }
2259 d = data_alloc(option, &ajla_error);
2260 slot_r = get_i_param(0);
2261 if (likely(d != NULL)) {
2262 da(d,option)->option = get_i_param(1);
2263 da(d,option)->pointer = ptr;
2264 frame_set_pointer(fp, slot_r, pointer_data(d));
2265 } else {
2266 pointer_dereference(ptr);
2267 frame_set_pointer(fp, slot_r, pointer_error(ajla_error, fp, ip pass_file_line));
2268 }
2269 ADVANCE_I_PARAM(4);
2270 #endif
2271 DEFINE_OPCODE_END(OPCODE_OPTION_CREATE)
2273 DEFINE_OPCODE_START(OPCODE_OPTION_LOAD)
2274 #ifdef EMIT_CODE
2275 frame_t option, option_idx, result;
2276 unsigned flags;
2277 struct data *data;
2278 pointer_t ptr;
2280 option = get_i_param(0);
2281 option_idx = get_i_param(1);
2282 result = get_i_param(2);
2283 flags = get_i_param(3);
2285 if (unlikely(frame_variable_is_flat(fp, option)))
2286 goto cat(option_load_mismatch,ARG_MODE);
2288 pointer_follow(frame_pointer(fp, option), true, data, unlikely(flags & OPCODE_OP_FLAG_STRICT) ? PF_WAIT : PF_NOEVAL, fp, ip,
2289 if (!(flags & OPCODE_OP_FLAG_STRICT)) {
2290 ex_ = ipret_option_load_create_thunk(fp, ip, option, option_idx, result);
2291 if (ex_ == POINTER_FOLLOW_THUNK_GO)
2292 goto cat(option_load_end_,ARG_MODE);
2293 }
2294 RELOAD_EX_POSITION(ex_),
2295 thunk_reference(thunk_);
2296 frame_set_pointer(fp, result, pointer_thunk(thunk_));
2297 goto cat(option_load_end_,ARG_MODE);
2298 );
2300 if (unlikely(da(data,option)->option != option_idx)) {
2301 cat(option_load_mismatch,ARG_MODE):
2302 ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_OPTION_DOESNT_MATCH), fp, ip pass_file_line);
2303 frame_set_pointer(fp, result, ptr);
2304 } else {
2305 pointer_reference_maybe(fp, result, &da(data,option)->pointer, flags);
2306 }
2308 cat(option_load_end_,ARG_MODE):
2309 ADVANCE_I_PARAM(4);
2310 #endif
2311 DEFINE_OPCODE_END(OPCODE_OPTION_LOAD)
2314 START_BLOCK(
2315 frame_t slot_1;
2316 ajla_option_t option;
2317 )
2319 DEFINE_OPCODE_START(OPCODE_OPTION_TEST_FLAT)
2320 #ifdef EMIT_CODE
2321 slot_1 = get_i_param(0);
2323 if (!frame_test_flag(fp, slot_1)) {
2324 barrier_aliasing();
2325 option = *frame_slot(fp, slot_1, ajla_flat_option_t);
2326 barrier_aliasing();
2327 goto cat(option_test_store_result_,ARG_MODE);
2328 }
2329 goto cat(option_test_,ARG_MODE);
2330 #endif
2331 DEFINE_OPCODE_END(OPCODE_OPTION_TEST_FLAT)
2333 DEFINE_OPCODE_START(OPCODE_OPTION_TEST)
2334 #ifdef EMIT_CODE
2335 frame_t slot_r;
2336 pointer_t ptr;
2337 struct data *data;
2339 slot_1 = get_i_param(0);
2340 cat(option_test_,ARG_MODE):
2341 ptr = *frame_pointer(fp, slot_1);
2342 if (unlikely(pointer_is_thunk(ptr))) {
2343 void *ex;
2344 ajla_assert(!frame_test_flag(fp, get_i_param(2)), (file_line, "option test: flag already set for destination slot %"PRIuMAX"", (uintmax_t)get_i_param(2)));
2345 ex = thunk_option_test(fp, ip, slot_1, get_i_param(1), get_i_param(2));
2346 if (ex == POINTER_FOLLOW_THUNK_GO) {
2347 ADVANCE_I_PARAM(3);
2348 break;
2349 } else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY)) {
2350 RELOAD_EX_POSITION(ex);
2351 }
2352 break;
2353 }
2354 data = pointer_get_data(ptr);
2355 option = da(data,option)->option;
2356 cat(option_test_store_result_,ARG_MODE):
2357 slot_r = get_i_param(2);
2358 barrier_aliasing();
2359 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "option test: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
2360 *frame_slot(fp, slot_r, ajla_flat_option_t) = option == (ajla_option_t)get_i_param(1);
2361 barrier_aliasing();
2362 ADVANCE_I_PARAM(3);
2363 #endif
2364 DEFINE_OPCODE_END(OPCODE_OPTION_TEST)
2366 DEFINE_OPCODE_START(OPCODE_OPTION_ORD_FLAT)
2367 #ifdef EMIT_CODE
2368 slot_1 = get_i_param(0);
2370 if (!frame_test_flag(fp, slot_1)) {
2371 barrier_aliasing();
2372 option = *frame_slot(fp, slot_1, ajla_flat_option_t);
2373 barrier_aliasing();
2374 goto cat(option_ord_store_result_,ARG_MODE);
2375 }
2376 goto cat(option_ord_,ARG_MODE);
2377 #endif
2378 DEFINE_OPCODE_END(OPCODE_OPTION_ORD_FLAT)
2380 DEFINE_OPCODE_START(OPCODE_OPTION_ORD)
2381 #ifdef EMIT_CODE
2382 frame_t slot_r;
2383 pointer_t ptr;
2384 struct data *data;
2386 slot_1 = get_i_param(0);
2387 cat(option_ord_,ARG_MODE):
2388 ptr = *frame_pointer(fp, slot_1);
2389 if (unlikely(pointer_is_thunk(ptr))) {
2390 void *ex;
2391 ajla_assert(!frame_test_flag(fp, get_i_param(1)), (file_line, "option ord: flag already set for destination slot %"PRIuMAX"", (uintmax_t)get_i_param(2)));
2392 ex = thunk_option_ord(fp, ip, slot_1, get_i_param(1));
2393 if (ex == POINTER_FOLLOW_THUNK_GO) {
2394 ADVANCE_I_PARAM(2);
2395 break;
2396 } else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY)) {
2397 RELOAD_EX_POSITION(ex);
2398 }
2399 break;
2400 }
2401 data = pointer_get_data(ptr);
2402 option = da(data,option)->option;
2403 cat(option_ord_store_result_,ARG_MODE):
2404 slot_r = get_i_param(1);
2405 barrier_aliasing();
2406 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "option ord: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
2407 *frame_slot(fp, slot_r, int_default_t) = option;
2408 barrier_aliasing();
2409 ADVANCE_I_PARAM(2);
2410 #endif
2411 DEFINE_OPCODE_END(OPCODE_OPTION_ORD)
2413 END_BLOCK()
2416 DEFINE_OPCODE_START(OPCODE_ARRAY_CREATE)
2417 #ifdef EMIT_CODE
2418 frame_t result_slot;
2419 size_t n_entries;
2420 struct data *total;
2422 result_slot = get_i_param(0);
2423 n_entries = get_i_param(1);
2424 ADVANCE_I_PARAM(2);
2426 ajla_assert(n_entries != 0, (file_line, "array create: zero entries"));
2428 total = NULL;
2429 while (1) {
2430 size_t n, i;
2431 struct data *a;
2432 const struct type *type = frame_get_type_of_local(fp, get_param(ip, 0));
2433 bool flat = frame_variable_is_flat(fp, get_param(ip, 0));
2434 for (n = 1; n < n_entries; n++) {
2435 if (frame_variable_is_flat(fp, get_param(ip, n * 2)) != flat)
2436 break;
2437 if (unlikely((int_default_t)(n + 1) < zero))
2438 break;
2439 }
2440 do {
2441 if (flat)
2442 a = data_alloc_array_flat_mayfail(type, n, n, false, &ajla_error pass_file_line);
2443 else
2444 a = data_alloc_array_pointers_mayfail(n, n, &ajla_error pass_file_line);
2445 } while (unlikely(!a) && (n >>= 1));
2446 if (unlikely(!a)) {
2447 if (total)
2448 data_dereference(total);
2449 goto cat(array_create_error_,ARG_MODE);
2450 }
2451 n_entries -= n;
2452 if (flat) {
2453 unsigned char *flat_ptr = da_array_flat(a);
2454 for (i = 0; i < n; i++) {
2455 frame_t var_slot = get_param(ip, 0);
2456 memcpy_fast(flat_ptr, frame_var(fp, var_slot), type->size);
2457 flat_ptr += type->size;
2458 ADVANCE_IP(param_size(2));
2459 }
2460 } else {
2461 for (i = 0; i < n; i++) {
2462 frame_t var_slot = get_param(ip, 0);
2463 pointer_t ptr = frame_get_pointer_reference(fp, var_slot, (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2464 da(a,array_pointers)->pointer[i] = ptr;
2465 ADVANCE_IP(param_size(2));
2466 }
2467 }
2468 if (likely(!total)) {
2469 total = a;
2470 } else {
2471 total = array_join(total, a, &ajla_error);
2472 if (unlikely(!total)) {
2473 cat(array_create_error_,ARG_MODE):
2474 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_entries);
2475 frame_set_pointer(fp, result_slot, pointer_error(ajla_error, fp, ip pass_file_line));
2476 break;
2477 }
2478 }
2479 if (likely(!n_entries)) {
2480 frame_set_pointer(fp, result_slot, pointer_data(total));
2481 break;
2482 }
2483 }
2484 #endif
2485 DEFINE_OPCODE_END(OPCODE_ARRAY_CREATE)
2487 DEFINE_OPCODE_START(OPCODE_ARRAY_CREATE_EMPTY_FLAT)
2488 #ifdef EMIT_CODE
2489 frame_t result_slot;
2490 struct data *a;
2491 const struct type *type = da_type(get_frame(fp)->function, get_i_param(1));
2492 TYPE_TAG_VALIDATE(type->tag);
2494 a = data_alloc_array_flat_mayfail(type, 0, 0, false, &ajla_error pass_file_line);
2495 result_slot = get_i_param(0);
2496 frame_set_pointer(fp, result_slot, likely(a != NULL) ? pointer_data(a) : pointer_error(ajla_error, fp, ip pass_file_line));
2497 ADVANCE_I_PARAM(2);
2498 #endif
2499 DEFINE_OPCODE_END(OPCODE_ARRAY_CREATE_EMPTY_FLAT)
2501 DEFINE_OPCODE_START(OPCODE_ARRAY_CREATE_EMPTY)
2502 #ifdef EMIT_CODE
2503 frame_t result_slot;
2504 struct data *a;
2506 a = data_alloc_array_pointers_mayfail(0, 0, &ajla_error pass_file_line);
2507 result_slot = get_i_param(0);
2508 frame_set_pointer(fp, result_slot, likely(a != NULL) ? pointer_data(a) : pointer_error(ajla_error, fp, ip pass_file_line));
2509 ADVANCE_I_PARAM(1);
2510 #endif
2511 DEFINE_OPCODE_END(OPCODE_ARRAY_CREATE_EMPTY)
2513 DEFINE_OPCODE_START(OPCODE_ARRAY_FILL)
2514 #ifdef EMIT_CODE
2515 frame_t content_slot, length_slot, result_slot;
2516 unsigned char content_flag;
2517 array_index_t length;
2518 pointer_t result_ptr;
2519 void *ex;
2521 content_slot = get_i_param(0);
2522 content_flag = get_i_param(1);
2523 length_slot = get_i_param(2);
2524 result_slot = get_i_param(3);
2526 ex = ipret_get_index(fp, ip, fp, length_slot, NULL, &length, &result_ptr pass_file_line);
2527 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
2528 if (ex == POINTER_FOLLOW_THUNK_EXCEPTION) {
2529 if (content_flag & OPCODE_FLAG_FREE_ARGUMENT)
2530 frame_free_and_clear(fp, content_slot);
2531 goto cat(array_fill_error_,ARG_MODE);
2532 }
2533 RELOAD_EX_POSITION(ex);
2534 }
2536 if (!frame_variable_is_flat(fp, content_slot)) {
2537 pointer_t ptr = frame_get_pointer_reference(fp, content_slot, (content_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2538 if (unlikely((content_flag & OPCODE_ARRAY_FILL_FLAG_SPARSE) != 0)) {
2539 result_ptr = array_create_sparse(length, ptr);
2540 } else {
2541 result_ptr = array_create(length, NULL, NULL, ptr);
2542 }
2543 } else {
2544 const struct type *type = frame_get_type_of_local(fp, content_slot);
2545 barrier_aliasing();
2546 if (unlikely((content_flag & OPCODE_ARRAY_FILL_FLAG_SPARSE) != 0) && likely(index_ge_int(length, 1))) {
2547 pointer_t ptr = flat_to_data(type, frame_var(fp, content_slot));
2548 result_ptr = array_create_sparse(length, ptr);
2549 } else {
2550 if (TYPE_IS_FLAT(frame_get_type_of_local(fp, result_slot))) {
2551 unsigned char *result = frame_var(fp, result_slot);
2552 int_default_t l = index_to_int(length);
2553 index_free(&length);
2554 while (l--) {
2555 result = mempcpy(result, frame_var(fp, content_slot), type->size);
2556 }
2557 goto cat(array_exit_,ARG_MODE);
2558 } else {
2559 result_ptr = array_create(length, type, frame_var(fp, content_slot), pointer_empty());
2560 }
2561 }
2562 barrier_aliasing();
2563 }
2565 cat(array_fill_error_,ARG_MODE):
2566 frame_set_pointer(fp, result_slot, result_ptr);
2568 cat(array_exit_,ARG_MODE):
2569 ADVANCE_I_PARAM(4);
2570 #endif
2571 DEFINE_OPCODE_END(OPCODE_ARRAY_FILL)
2573 DEFINE_OPCODE_START(OPCODE_ARRAY_STRING)
2574 #ifdef EMIT_CODE
2575 frame_t result_slot, length;
2576 pointer_t result_ptr;
2578 result_slot = get_i_param(0);
2579 length = get_i_param(1);
2580 ADVANCE_I_PARAM(2);
2582 if (unlikely((int_default_t)length < (int_default_t)zero) ||
2583 unlikely((frame_t)(int_default_t)length != length)) {
2584 result_ptr = pointer_error(error_ajla(EC_ASYNC, AJLA_ERROR_SIZE_OVERFLOW), fp, ip pass_file_line);
2585 } else {
2586 result_ptr = array_string(length, type_get_fixed(0, true), cast_ptr(unsigned char *, ip));
2587 }
2588 frame_set_pointer(fp, result_slot, result_ptr);
2590 ADVANCE_IP((length + 1) >> 1);
2591 #endif
2592 DEFINE_OPCODE_END(OPCODE_ARRAY_STRING)
2594 DEFINE_OPCODE_START(OPCODE_ARRAY_UNICODE)
2595 #ifdef EMIT_CODE
2596 frame_t result_slot, length;
2597 pointer_t result_ptr;
2599 result_slot = get_i_param(0);
2600 length = get_i_param(1);
2601 ADVANCE_I_PARAM(2);
2603 if (unlikely((int_default_t)length < (int_default_t)zero) ||
2604 unlikely((frame_t)(int_default_t)length != length)) {
2605 result_ptr = pointer_error(error_ajla(EC_ASYNC, AJLA_ERROR_SIZE_OVERFLOW), fp, ip pass_file_line);
2606 } else {
2607 result_ptr = array_string(length, type_get_int(2), cast_ptr(unsigned char *, ip));
2608 }
2609 frame_set_pointer(fp, result_slot, result_ptr);
2611 ADVANCE_IP(length * 2);
2612 #endif
2613 DEFINE_OPCODE_END(OPCODE_ARRAY_STRING)
2615 DEFINE_OPCODE_START(OPCODE_ARRAY_LOAD)
2616 #ifdef EMIT_CODE
2617 frame_t array, idx, result;
2618 unsigned flags;
2619 const struct type *t, *t_elem;
2620 void *ex;
2621 array_index_t array_index;
2622 pointer_t ptr;
2623 unsigned char *flat;
2625 array = get_i_param(0);
2626 idx = get_i_param(1);
2627 result = get_i_param(2);
2628 flags = get_i_param(3);
2630 ex = ipret_get_index(fp, ip, fp, idx, NULL, &array_index, &ptr pass_file_line);
2631 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
2632 if (ex == POINTER_FOLLOW_THUNK_EXCEPTION)
2633 goto cat(array_load_set_ptr_,ARG_MODE);
2634 RELOAD_EX_POSITION(ex);
2635 }
2637 t = frame_get_type_of_local(fp, array);
2638 if (t->tag == TYPE_TAG_flat_array && !frame_test_flag(fp, array)) {
2639 const struct flat_array_definition *def = type_def(t,flat_array);
2640 if (unlikely(index_ge_int(array_index, def->n_elements))) {
2641 if (unlikely((flags & OPCODE_ARRAY_INDEX_IN_RANGE) != 0))
2642 internal(file_line, "array_load: flat array index out of range");
2643 index_free(&array_index);
2644 ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_INDEX_OUT_OF_RANGE), fp, ip pass_file_line);
2645 goto cat(array_load_set_ptr_,ARG_MODE);
2646 }
2647 t_elem = def->base;
2648 flat = frame_var(fp, array) + t_elem->size * index_to_int(array_index);
2649 index_free(&array_index);
2650 goto cat(array_load_set_flat_,ARG_MODE);
2651 } else {
2652 pointer_t *array_ptr = frame_pointer(fp, array);
2653 pointer_t *pptr;
2654 struct data *data;
2656 cat(array_restart_pointer_follow_,ARG_MODE):
2657 pointer_follow(array_ptr, false, data, unlikely(flags & OPCODE_OP_FLAG_STRICT) ? PF_WAIT : PF_NOEVAL, fp, ip,
2658 index_free(&array_index);
2659 if (!(flags & OPCODE_OP_FLAG_STRICT)) {
2660 ex_ = ipret_array_load_create_thunk(fp, ip, array, idx, result);
2661 if (ex_ == POINTER_FOLLOW_THUNK_GO)
2662 goto cat(array_load_end_,ARG_MODE);
2663 }
2664 RELOAD_EX_POSITION(ex_),
2665 index_free(&array_index);
2666 thunk_reference(thunk_);
2667 ptr = pointer_thunk(thunk_);
2668 goto cat(array_load_set_ptr_,ARG_MODE);
2669 );
2671 if (unlikely(da_tag(data) == DATA_TAG_array_incomplete)) {
2672 array_index_t len_first;
2674 if (array_ptr == frame_pointer(fp, array)) {
2675 if (!frame_test_and_set_flag(fp, array))
2676 data_reference(data);
2677 if (array_incomplete_collapse(array_ptr))
2678 goto cat(array_restart_pointer_follow_,ARG_MODE);
2679 }
2681 len_first = array_len(pointer_get_data(da(data,array_incomplete)->first));
2682 if (!index_ge_index(array_index, len_first)) {
2683 index_free(&len_first);
2684 data = pointer_get_data(da(data,array_incomplete)->first);
2685 } else {
2686 index_sub(&array_index, len_first);
2687 index_free(&len_first);
2688 array_ptr = &da(data,array_incomplete)->next;
2689 goto cat(array_restart_pointer_follow_,ARG_MODE);
2690 }
2691 }
2693 if (unlikely(!array_read(data, array_index, &pptr, &flat, &t_elem, NULL))) {
2694 if (unlikely((flags & OPCODE_ARRAY_INDEX_IN_RANGE) != 0))
2695 internal(file_line, "array_load: %s: array index out of range", da(get_frame(fp)->function,function)->function_name);
2696 ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_INDEX_OUT_OF_RANGE), fp, ip pass_file_line);
2697 goto cat(array_load_set_ptr_,ARG_MODE);
2698 }
2699 if (pptr) {
2700 pointer_reference_maybe(fp, result, pptr, flags);
2701 goto cat(array_load_end_,ARG_MODE);
2702 } else {
2703 cat(array_load_set_flat_,ARG_MODE):
2704 if (likely(TYPE_IS_FLAT(frame_get_type_of_local(fp, result)))) {
2705 memcpy_fast(frame_var(fp, result), flat, t_elem->size);
2706 goto cat(array_load_end_,ARG_MODE);
2707 } else {
2708 ptr = flat_to_data(t_elem, flat);
2709 }
2710 }
2711 }
2713 cat(array_load_set_ptr_,ARG_MODE):
2714 frame_set_pointer(fp, result, ptr);
2716 cat(array_load_end_,ARG_MODE):
2717 ADVANCE_I_PARAM(4);
2718 #endif
2719 DEFINE_OPCODE_END(OPCODE_ARRAY_LOAD)
2721 DEFINE_OPCODE_START(OPCODE_ARRAY_LEN)
2722 #ifdef EMIT_CODE
2723 frame_t slot_a, slot_r;
2724 unsigned flags;
2725 void *ex;
2727 slot_a = get_i_param(0);
2728 slot_r = get_i_param(1);
2729 flags = get_i_param(2);
2731 ex = ipret_array_len(fp, ip, slot_r, slot_a, flags);
2732 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2733 RELOAD_EX_POSITION(ex);
2735 ADVANCE_I_PARAM(3);
2736 #endif
2737 DEFINE_OPCODE_END(OPCODE_ARRAY_LEN)
2739 DEFINE_OPCODE_START(OPCODE_ARRAY_LEN_GREATER_THAN)
2740 #ifdef EMIT_CODE
2741 frame_t slot_a, slot_l, slot_r;
2742 unsigned flags;
2743 void *ex;
2745 slot_a = get_i_param(0);
2746 slot_l = get_i_param(1);
2747 slot_r = get_i_param(2);
2748 flags = get_i_param(3);
2750 ex = ipret_array_len_greater_than(fp, ip, slot_r, slot_a, slot_l, flags);
2751 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2752 RELOAD_EX_POSITION(ex);
2754 ADVANCE_I_PARAM(4);
2755 #endif
2756 DEFINE_OPCODE_END(OPCODE_ARRAY_LEN_GREATER_THAN)
2758 DEFINE_OPCODE_START(OPCODE_ARRAY_SUB)
2759 #ifdef EMIT_CODE
2760 frame_t slot_a, slot_start, slot_end, slot_r;
2761 unsigned flags;
2762 void *ex;
2764 slot_a = get_i_param(0);
2765 slot_start = get_i_param(1);
2766 slot_end = get_i_param(2);
2767 slot_r = get_i_param(3);
2768 flags = get_i_param(4);
2770 ex = ipret_array_sub(fp, ip, slot_r, slot_a, slot_start, slot_end, flags);
2771 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2772 RELOAD_EX_POSITION(ex);
2774 ADVANCE_I_PARAM(5);
2775 #endif
2776 DEFINE_OPCODE_END(OPCODE_ARRAY_SUB)
2778 DEFINE_OPCODE_START(OPCODE_ARRAY_SKIP)
2779 #ifdef EMIT_CODE
2780 frame_t slot_a, slot_start, slot_r;
2781 unsigned flags;
2782 void *ex;
2784 slot_a = get_i_param(0);
2785 slot_start = get_i_param(1);
2786 slot_r = get_i_param(2);
2787 flags = get_i_param(3);
2789 ex = ipret_array_skip(fp, ip, slot_r, slot_a, slot_start, flags);
2790 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2791 RELOAD_EX_POSITION(ex);
2793 ADVANCE_I_PARAM(4);
2794 #endif
2795 DEFINE_OPCODE_END(OPCODE_ARRAY_SKIP)
2797 DEFINE_OPCODE_START(OPCODE_ARRAY_APPEND)
2798 #ifdef EMIT_CODE
2799 frame_t slot_r, slot_1, slot_2;
2800 unsigned flags;
2801 void *ex;
2803 slot_r = get_i_param(0);
2804 flags = get_i_param(1);
2805 slot_1 = get_i_param(2);
2806 slot_2 = get_i_param(3);
2808 ex = ipret_array_append(fp, ip, slot_r, slot_1, slot_2, flags);
2809 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2810 RELOAD_EX_POSITION(ex);
2812 ADVANCE_I_PARAM(4);
2813 #endif
2814 DEFINE_OPCODE_END(OPCODE_ARRAY_APPEND)
2816 DEFINE_OPCODE_START(OPCODE_ARRAY_APPEND_ONE_FLAT)
2817 #ifdef EMIT_CODE
2818 frame_t slot_r, slot_1, slot_2;
2819 unsigned flags;
2820 void *ex;
2822 slot_r = get_i_param(0);
2823 flags = get_i_param(1);
2824 slot_1 = get_i_param(2);
2825 slot_2 = get_i_param(3);
2827 ex = ipret_array_append_one_flat(fp, ip, slot_r, slot_1, slot_2, flags);
2828 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2829 RELOAD_EX_POSITION(ex);
2831 ADVANCE_I_PARAM(4);
2832 #endif
2833 DEFINE_OPCODE_END(OPCODE_ARRAY_APPEND_ONE_FLAT)
2835 DEFINE_OPCODE_START(OPCODE_ARRAY_APPEND_ONE)
2836 #ifdef EMIT_CODE
2837 frame_t slot_r, slot_1, slot_2;
2838 unsigned flags;
2839 void *ex;
2841 slot_r = get_i_param(0);
2842 flags = get_i_param(1);
2843 slot_1 = get_i_param(2);
2844 slot_2 = get_i_param(3);
2846 ex = ipret_array_append_one(fp, ip, slot_r, slot_1, slot_2, flags);
2847 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2848 RELOAD_EX_POSITION(ex);
2850 ADVANCE_I_PARAM(4);
2851 #endif
2852 DEFINE_OPCODE_END(OPCODE_ARRAY_APPEND_ONE)
2854 DEFINE_OPCODE_START(OPCODE_ARRAY_FLATTEN)
2855 #ifdef EMIT_CODE
2856 frame_t slot_r, slot_1;
2857 unsigned flags;
2858 void *ex;
2860 slot_r = get_i_param(0);
2861 flags = get_i_param(1);
2862 slot_1 = get_i_param(2);
2864 ex = ipret_array_flatten(fp, ip, slot_r, slot_1, flags);
2865 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2866 RELOAD_EX_POSITION(ex);
2868 ADVANCE_I_PARAM(3);
2869 #endif
2870 DEFINE_OPCODE_END(OPCODE_ARRAY_FLATTEN)
2873 #if ARG_MODE == 0
2874 DEFINE_OPCODE_START(OPCODE_IO)
2875 #ifdef EMIT_CODE
2876 unsigned char io_code, n_outputs, n_inputs, n_params;
2877 void *ex;
2879 io_code = get_i_param(0);
2880 n_outputs = get_i_param(1);
2881 n_inputs = get_i_param(2);
2882 n_params = get_i_param(3);
2884 ex = ipret_io(fp, ip, io_code, n_outputs, n_inputs, n_params);
2885 if (ex != POINTER_FOLLOW_THUNK_GO) {
2886 RELOAD_EX_POSITION(ex);
2887 } else {
2888 ADVANCE_IP(3 + 2 * (n_outputs + n_inputs + n_params));
2889 }
2890 #endif
2891 DEFINE_OPCODE_END(OPCODE_IO)
2892 #endif
2894 #if ARG_MODE == 0
2895 DEFINE_OPCODE_START(OPCODE_INTERNAL_FUNCTION)
2896 #ifdef EMIT_CODE
2897 void *ex = function_call_internal(fp, ip);
2898 if (ex != POINTER_FOLLOW_THUNK_RETRY) {
2899 RELOAD_EX_POSITION(ex);
2900 }
2901 #endif
2902 DEFINE_OPCODE_END(OPCODE_INTERNAL_FUNCTION)
2903 #endif
2905 #if ARG_MODE == 0
2906 DEFINE_OPCODE_START(OPCODE_UNREACHABLE)
2907 #ifdef EMIT_CODE
2908 internal(file_line, "unreachable code");
2909 #endif
2910 DEFINE_OPCODE_END(OPCODE_UNREACHABLE)
2911 #endif
2913 #if ARG_MODE == 0
2914 DEFINE_OPCODE_START(OPCODE_EXIT_THREAD)
2915 #ifdef EMIT_CODE
2916 struct execution_control *ex = frame_execution_control(fp);
2917 pointer_t *var_ptr = frame_pointer(fp, get_i_param(0));
2918 struct data attr_unused *data;
2920 pointer_follow(var_ptr, true, data, PF_WAIT, fp, ip,
2921 RELOAD_EX_POSITION(ex_),
2922 ajla_assert_lo(thunk_tag(thunk_) == THUNK_TAG_EXCEPTION, (file_line, "exit_thread: invalid thunk tag %u", thunk_tag(thunk_)));
2923 goto exit_ipret_thunk;
2924 );
2926 exit_ipret_thunk:
2927 ajla_assert_lo(ex->thunk == NULL, (file_line, "exit_thread: non-NULL thunk %p", ex->thunk));
2928 ex->current_frame = fp;
2929 ex->current_ip = frame_ip(fp, ip);
2930 execution_control_terminate(ex, *var_ptr);
2931 goto exit_ipret;
2932 #endif
2933 DEFINE_OPCODE_END(OPCODE_EXIT_THREAD)
2934 #endif
2937 #undef move_scalar
2938 #undef copy_scalar
2940 #undef get_param
2941 #undef param_size
2942 #undef get_i_param
2943 #undef ADVANCE_I_PARAM
2945 #undef ARG_MODE
2946 #undef DEFINE_OPCODE_START