| blob | 5de55390c206139a45650458e9ea9c6aa1c54cfc |
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
19 #if ARG_MODE == 0
20 #define get_param(ip, n) (!((n) & 1) ? get_lo((ip) + (n) / 2) : get_hi((ip) + (n) / 2))
21 #define param_size(n) (((n) + 1) / 2)
22 #elif ARG_MODE == 1
23 /* A cast to uintptr_t because of gcc optimizer inefficiency */
24 #define get_param(ip, n) ((uintptr_t)(ip)[n])
25 #define param_size(n) (n)
26 #elif ARG_MODE == 2
27 #define get_param(ip, n) get_unaligned_32(&(ip)[(n) * 2])
28 #define param_size(n) ((n) * 2)
29 #endif
31 #define get_i_param(n) get_param(ip + 1, n)
32 #define ADVANCE_I_PARAM(n) ADVANCE_IP(1 + param_size(n))
35 #ifdef EMIT_FUNCTIONS
36 static const code_t attr_cold *cat(free_parameters_,ARG_MODE)(frame_s *fp, const code_t *ip, frame_t n_entries)
37 {
38 while (n_entries--) {
39 if (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT)
40 frame_free_and_clear(fp, get_param(ip, 0));
41 ADVANCE_IP(param_size(2));
42 }
43 return ip;
44 }
45 #endif
48 #define move_scalar(utype) \
49 do { \
50 frame_t slot_1 = get_i_param(0); \
51 frame_t slot_r = get_i_param(1); \
52 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "scalar move: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
53 barrier_aliasing(); \
54 if (unlikely(frame_test_and_clear_flag(fp, slot_1))) { \
55 frame_set_pointer(fp, slot_r, *frame_pointer(fp, slot_1));\
56 pointer_poison(frame_pointer(fp, slot_1)); \
57 } else { \
58 *frame_slot(fp, slot_r, utype) = \
59 *frame_slot(fp, slot_1, utype); \
60 } \
61 barrier_aliasing(); \
62 ADVANCE_I_PARAM(2); \
63 } while (0)
65 #define copy_scalar(utype) \
66 do { \
67 frame_t slot_1 = get_i_param(0); \
68 frame_t slot_r = get_i_param(1); \
69 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "scalar copy: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
70 barrier_aliasing(); \
71 if (unlikely(frame_test_flag(fp, slot_1))) { \
72 pointer_t ptr = *frame_pointer(fp, slot_1); \
73 frame_set_pointer(fp, slot_r, ptr); \
74 pointer_reference_owned(ptr); \
75 } else { \
76 *frame_slot(fp, slot_r, utype) = \
77 *frame_slot(fp, slot_1, utype); \
78 } \
79 barrier_aliasing(); \
80 ADVANCE_I_PARAM(2); \
81 } while (0)
84 #define DEFINE_FIXED_REAL_BINARY_OPCODE(tc, type, op) \
85 DEFINE_OPCODE( \
86 cat3(OPCODE_,tc,OP) + \
87 cat4(OPCODE_,tc,OP_,op) * cat3(OPCODE_,tc,OP_MULT) + \
88 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
89 cat7(tc,binary_,op,_,type,_,ARG_MODE), \
90 frame_t slot_1 = get_i_param(0); \
91 frame_t slot_2 = get_i_param(1); \
92 frame_t slot_r = get_i_param(2); \
93 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "fixed binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
94 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
95 goto cat(escape_fixed_binary_thunk_,ARG_MODE); \
96 } else { \
97 const void *val1; \
98 const void *val2; \
99 void *result; \
100 val1 = frame_var(fp, slot_1); \
101 val2 = frame_var(fp, slot_2); \
102 result = frame_var(fp, slot_r); \
103 barrier_aliasing(); \
104 if (unlikely(!cat5(tc,binary_,op,_,type)(val1, val2, result))) {\
105 barrier_aliasing(); \
106 goto cat(escape_fixed_binary_thunk_,ARG_MODE); \
107 } \
108 barrier_aliasing(); \
109 ADVANCE_I_PARAM(4); \
110 } \
111 )
113 #define DEFINE_FIXED_REAL_UNARY_OPCODE(tc, type, op) \
114 DEFINE_OPCODE( \
115 cat3(OPCODE_,tc,OP) + \
116 cat4(OPCODE_,tc,OP_,op) * cat3(OPCODE_,tc,OP_MULT) + \
117 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
118 cat7(tc,unary_,op,_,type,_,ARG_MODE), \
119 frame_t slot_1 = get_i_param(0); \
120 frame_t slot_r = get_i_param(1); \
121 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "fixed unary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
122 if (unlikely(frame_test_flag(fp, slot_1))) { \
123 goto cat(escape_fixed_unary_thunk_,ARG_MODE); \
124 } else { \
125 const void *val1; \
126 void *result; \
127 val1 = frame_var(fp, slot_1); \
128 result = frame_var(fp, slot_r); \
129 barrier_aliasing(); \
130 cat5(tc,unary_,op,_,type)(val1, result); \
131 barrier_aliasing(); \
132 ADVANCE_I_PARAM(3); \
133 } \
134 )
136 #define DEFINE_FIXED_TO_INT(type, utype) \
137 DEFINE_OPCODE( \
138 OPCODE_FIXED_OP + \
139 OPCODE_FIXED_OP_to_int * OPCODE_FIXED_OP_MULT + \
140 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
141 cat4(FIXED_unary_to_int_,type,_,ARG_MODE), \
142 frame_t slot_1 = get_i_param(0); \
143 frame_t slot_r = get_i_param(1); \
144 if (unlikely(frame_test_flag(fp, slot_1))) { \
145 goto cat(escape_convert_thunk_,ARG_MODE); \
146 } else { \
147 type val1; \
148 int_default_t r; \
149 barrier_aliasing(); \
150 val1 = *frame_slot(fp, slot_1, type); \
151 r = (int_default_t)val1; \
152 if (unlikely(r != val1)) \
153 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, false));\
154 else \
155 *frame_slot(fp, slot_r, int_default_t) = r; \
156 barrier_aliasing(); \
157 ADVANCE_I_PARAM(3); \
158 } \
159 )
161 #define DEFINE_FIXED_UTO_INT(type, utype) \
162 DEFINE_OPCODE( \
163 OPCODE_FIXED_OP + \
164 OPCODE_FIXED_OP_uto_int * OPCODE_FIXED_OP_MULT + \
165 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
166 cat4(FIXED_unary_uto_int_,type,_,ARG_MODE), \
167 frame_t slot_1 = get_i_param(0); \
168 frame_t slot_r = get_i_param(1); \
169 if (unlikely(frame_test_flag(fp, slot_1))) { \
170 goto cat(escape_convert_thunk_,ARG_MODE); \
171 } else { \
172 utype val1; \
173 int_default_t r; \
174 barrier_aliasing(); \
175 val1 = *frame_slot(fp, slot_1, utype); \
176 r = (int_default_t)val1; \
177 if (unlikely((utype)r != val1) || unlikely(r < 0)) \
178 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, true));\
179 else \
180 *frame_slot(fp, slot_r, int_default_t) = r; \
181 barrier_aliasing(); \
182 ADVANCE_I_PARAM(3); \
183 } \
184 )
186 #define DEFINE_FIXED_FROM_INT(type, utype) \
187 DEFINE_OPCODE( \
188 OPCODE_FIXED_OP + \
189 OPCODE_FIXED_OP_from_int * OPCODE_FIXED_OP_MULT + \
190 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
191 cat4(FIXED_unary_from_int_,type,_,ARG_MODE), \
192 frame_t slot_1 = get_i_param(0); \
193 frame_t slot_r = get_i_param(1); \
194 if (unlikely(frame_test_flag(fp, slot_1))) { \
195 goto cat(escape_convert_thunk_,ARG_MODE); \
196 } else { \
197 int_default_t val1; \
198 type r; \
199 barrier_aliasing(); \
200 val1 = *frame_slot(fp, slot_1, int_default_t); \
201 r = (type)val1; \
202 if (unlikely(r != val1)) \
203 frame_set_pointer(fp, slot_r, pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_DOESNT_FIT), fp, ip pass_file_line));\
204 else \
205 *frame_slot(fp, slot_r, type) = r; \
206 barrier_aliasing(); \
207 ADVANCE_I_PARAM(3); \
208 } \
209 )
211 #define DEFINE_FIXED_UFROM_INT(type, utype) \
212 DEFINE_OPCODE( \
213 OPCODE_FIXED_OP + \
214 OPCODE_FIXED_OP_ufrom_int * OPCODE_FIXED_OP_MULT + \
215 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
216 cat4(FIXED_unary_ufrom_int_,type,_,ARG_MODE), \
217 frame_t slot_1 = get_i_param(0); \
218 frame_t slot_r = get_i_param(1); \
219 if (unlikely(frame_test_flag(fp, slot_1))) { \
220 goto cat(escape_convert_thunk_,ARG_MODE); \
221 } else { \
222 int_default_t val1; \
223 utype r; \
224 barrier_aliasing(); \
225 val1 = *frame_slot(fp, slot_1, int_default_t); \
226 r = (utype)val1; \
227 if (unlikely((int_default_t)r != val1) || unlikely(val1 < 0))\
228 frame_set_pointer(fp, slot_r, pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_DOESNT_FIT), fp, ip pass_file_line));\
229 else \
230 *frame_slot(fp, slot_r, type) = r; \
231 barrier_aliasing(); \
232 ADVANCE_I_PARAM(3); \
233 } \
234 )
236 #define DEFINE_SCALAR_MOVE_OPCODE(tc, type) \
237 DEFINE_OPCODE( \
238 cat3(OPCODE_,tc,OP) + \
239 cat3(OPCODE_,tc,OP_move) * cat3(OPCODE_,tc,OP_MULT) + \
240 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
241 cat5(tc,move_,type,_,ARG_MODE), \
242 move_scalar(type); \
243 )
245 #define DEFINE_SCALAR_COPY_OPCODE(tc, type) \
246 DEFINE_OPCODE( \
247 cat3(OPCODE_,tc,OP) + \
248 cat3(OPCODE_,tc,OP_copy) * cat3(OPCODE_,tc,OP_MULT) + \
249 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
250 cat5(tc,copy_,type,_,ARG_MODE), \
251 copy_scalar(type); \
252 )
254 #define DEFINE_FIXED_LDC_OPCODE(cls, type, utype, opc, shrt) \
255 DEFINE_OPCODE( \
256 OPCODE_##cls##_OP + \
257 OPCODE_##cls##_OP_##opc * OPCODE_##cls##_OP_MULT + \
258 cat(OPCODE_##cls##_TYPE_,type) * OPCODE_##cls##_TYPE_MULT,\
259 cat6(fixed_,opc,_,type,_,ARG_MODE), \
260 frame_t slot; \
261 utype *val; \
262 ip_t sz; \
263 slot = get_i_param(0); \
264 ajla_assert(!frame_test_flag(fp, slot), (file_line, "ldc fixed: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot));\
265 val = frame_slot(fp, slot, utype); \
266 barrier_aliasing(); \
267 sz = cat(fixed_ldc_,type)(val, ip + 1 + param_size(1), shrt); \
268 barrier_aliasing(); \
269 ADVANCE_I_PARAM(1); \
270 ADVANCE_IP(sz); \
271 )
273 #define DEFINE_FIXED_OPCODES(n, type, utype, sz, bits) \
274 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, add) \
275 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, subtract) \
276 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, multiply) \
277 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, divide) \
278 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, udivide) \
279 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, modulo) \
280 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, umodulo) \
281 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, power) \
282 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, and) \
283 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, or) \
284 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, xor) \
285 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, shl) \
286 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, shr) \
287 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, ushr) \
288 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, rol) \
289 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, ror) \
290 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, bts) \
291 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, btr) \
292 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, btc) \
293 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, equal) \
294 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, not_equal)\
295 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, less) \
296 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, less_equal)\
297 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, uless) \
298 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, uless_equal)\
299 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, bt) \
300 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, not) \
301 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, neg) \
302 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, inc) \
303 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, dec) \
304 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, bswap) \
305 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, brev) \
306 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, bsf) \
307 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, bsr) \
308 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, popcnt) \
309 DEFINE_FIXED_TO_INT(type, utype) \
310 DEFINE_FIXED_UTO_INT(type, utype) \
311 DEFINE_FIXED_FROM_INT(type, utype) \
312 DEFINE_FIXED_UFROM_INT(type, utype) \
313 DEFINE_SCALAR_MOVE_OPCODE(FIXED_, type) \
314 DEFINE_SCALAR_COPY_OPCODE(FIXED_, type) \
315 DEFINE_FIXED_LDC_OPCODE(FIXED, type, utype, ldc, false)
317 for_all_fixed(DEFINE_FIXED_OPCODES)
319 #if TYPE_FIXED_N >= 3
320 DEFINE_FIXED_LDC_OPCODE(FIXED, int32_t, uint32_t, ldc16, true)
321 #if TYPE_FIXED_N >= 4
322 DEFINE_FIXED_LDC_OPCODE(FIXED, int64_t, uint64_t, ldc16, true)
323 #if TYPE_FIXED_N >= 5
324 DEFINE_FIXED_LDC_OPCODE(FIXED, int128_t, uint128_t, ldc16, true)
325 #endif
326 #endif
327 #endif
330 #define DEFINE_REAL_TO_INT(type, op) \
331 DEFINE_OPCODE( \
332 OPCODE_REAL_OP + \
333 cat(OPCODE_REAL_OP_,op) * OPCODE_REAL_OP_MULT + \
334 cat(OPCODE_REAL_TYPE_,type) * OPCODE_REAL_TYPE_MULT, \
335 cat6(REAL_unary_,op,_,type,_,ARG_MODE), \
336 frame_t slot_1 = get_i_param(0); \
337 frame_t slot_r = get_i_param(1); \
338 if (unlikely(frame_test_flag(fp, slot_1))) { \
339 goto cat(escape_convert_thunk_,ARG_MODE); \
340 } else { \
341 bool ret; \
342 barrier_aliasing(); \
343 ret = cat4(REAL_unary_,op,_,type)(frame_slot(fp, slot_1, type), frame_slot(fp, slot_r, int_default_t));\
344 barrier_aliasing(); \
345 if (unlikely(!ret)) \
346 frame_set_pointer(fp, slot_r, convert_real_to_mpint(fp, slot_1, frame_get_type_of_local(fp, slot_1)));\
347 ADVANCE_I_PARAM(3); \
348 } \
349 )
351 #define DEFINE_REAL_FROM_INT(type, op) \
352 DEFINE_OPCODE( \
353 OPCODE_REAL_OP + \
354 cat(OPCODE_REAL_OP_,op) * OPCODE_REAL_OP_MULT + \
355 cat(OPCODE_REAL_TYPE_,type) * OPCODE_REAL_TYPE_MULT, \
356 cat6(REAL_unary_,op,_,type,_,ARG_MODE), \
357 frame_t slot_1 = get_i_param(0); \
358 frame_t slot_r = get_i_param(1); \
359 if (unlikely(frame_test_flag(fp, slot_1))) { \
360 goto cat(escape_convert_thunk_,ARG_MODE); \
361 } else { \
362 barrier_aliasing(); \
363 cat4(REAL_unary_,op,_,type)(frame_slot(fp, slot_1, int_default_t), frame_slot(fp, slot_r, type));\
364 barrier_aliasing(); \
365 ADVANCE_I_PARAM(3); \
366 } \
367 )
370 #define DEFINE_REAL_OPCODES(n, type, ntype, pack, unpack)\
371 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, add) \
372 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, subtract) \
373 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, multiply) \
374 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, divide) \
375 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, modulo) \
376 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, power) \
377 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, ldexp) \
378 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, atan2) \
379 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, equal) \
380 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, not_equal) \
381 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, less) \
382 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, less_equal)\
383 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, neg) \
384 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, sqrt) \
385 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, cbrt) \
386 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, sin) \
387 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, cos) \
388 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, tan) \
389 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, asin) \
390 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, acos) \
391 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, atan) \
392 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, sinh) \
393 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, cosh) \
394 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, tanh) \
395 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, asinh) \
396 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, acosh) \
397 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, atanh) \
398 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exp2) \
399 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exp) \
400 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exp10) \
401 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, log2) \
402 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, log) \
403 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, log10) \
404 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, round) \
405 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, ceil) \
406 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, floor) \
407 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, trunc) \
408 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, fract) \
409 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, mantissa) \
410 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exponent) \
411 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, next_number)\
412 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, prev_number)\
413 DEFINE_REAL_TO_INT(type, to_int) \
414 DEFINE_REAL_FROM_INT(type, from_int) \
415 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, is_exception)\
416 DEFINE_SCALAR_MOVE_OPCODE(REAL_, type) \
417 DEFINE_SCALAR_COPY_OPCODE(REAL_, type) \
418 DEFINE_FIXED_LDC_OPCODE(REAL, type, type, ldc, false)
420 for_all_real(DEFINE_REAL_OPCODES, for_all_empty)
423 DEFINE_LABEL(cat(escape_fixed_binary_thunk_,ARG_MODE),
424 void *ex;
425 ASM_PREVENT_CSE;
426 ex = thunk_fixed_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3));
427 if (ex == POINTER_FOLLOW_THUNK_GO)
428 ADVANCE_I_PARAM(4);
429 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
430 RELOAD_EX_POSITION(ex);
431 )
434 DEFINE_LABEL(cat(escape_fixed_unary_thunk_,ARG_MODE),
435 void *ex;
436 ASM_PREVENT_CSE;
437 ex = thunk_fixed_operator(fp, ip, get_i_param(0), NO_FRAME_T, get_i_param(1), get_i_param(2));
438 if (ex == POINTER_FOLLOW_THUNK_GO)
439 ADVANCE_I_PARAM(3);
440 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
441 RELOAD_EX_POSITION(ex);
442 )
445 DEFINE_LABEL(cat(escape_convert_thunk_,ARG_MODE),
446 void *ex;
447 ASM_PREVENT_CSE;
448 ex = thunk_convert(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2));
449 if (ex == POINTER_FOLLOW_THUNK_GO)
450 ADVANCE_I_PARAM(3);
451 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
452 RELOAD_EX_POSITION(ex);
453 )
456 START_BLOCK(
457 bool (attr_fastcall *function_int_binary)(const mpint_t *s1, const mpint_t *s2, mpint_t *r, ajla_error_t *err);
458 bool (attr_fastcall *function_int_binary_logical)(const mpint_t *s1, const mpint_t *s2, ajla_flat_option_t *r, ajla_error_t *err);
459 bool (attr_fastcall *function_int_unary)(const mpint_t *s1, mpint_t *r, ajla_error_t *err);
460 )
462 #define DEFINE_INT_BINARY_OPCODE(type, op) \
463 DEFINE_OPCODE( \
464 OPCODE_INT_OP + \
465 cat(OPCODE_INT_OP_,op) * OPCODE_INT_OP_MULT + \
466 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
467 cat6(int_binary_,op,_,type,_,ARG_MODE), \
468 frame_t slot_1 = get_i_param(0); \
469 frame_t slot_2 = get_i_param(1); \
470 frame_t slot_r = get_i_param(2); \
471 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "int binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
472 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
473 cat6(escape_int_binary_,op,_,type,_,ARG_MODE): \
474 function_int_binary = cat(mpint_,op); \
475 goto cat(escape_int_binary_thunk_,ARG_MODE); \
476 } else { \
477 const void *val1; \
478 const void *val2; \
479 void *result; \
480 val1 = frame_var(fp, slot_1); \
481 val2 = frame_var(fp, slot_2); \
482 result = frame_var(fp, slot_r); \
483 barrier_aliasing(); \
484 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, val2, result))) {\
485 barrier_aliasing(); \
486 goto cat6(escape_int_binary_,op,_,type,_,ARG_MODE);\
487 } \
488 barrier_aliasing(); \
489 ADVANCE_I_PARAM(4); \
490 } \
491 )
493 #define DEFINE_INT_BINARY_LOGICAL_OPCODE(type, op) \
494 DEFINE_OPCODE( \
495 OPCODE_INT_OP + \
496 cat(OPCODE_INT_OP_,op) * OPCODE_INT_OP_MULT + \
497 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
498 cat6(int_binary_,op,_,type,_,ARG_MODE), \
499 frame_t slot_1 = get_i_param(0); \
500 frame_t slot_2 = get_i_param(1); \
501 frame_t slot_r = get_i_param(2); \
502 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "int logical: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
503 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
504 cat6(escape_int_binary_logical_,op,_,type,_,ARG_MODE): \
505 function_int_binary_logical = cat(mpint_,op); \
506 goto cat(escape_int_binary_logical_thunk_,ARG_MODE); \
507 } else { \
508 const void *val1; \
509 const void *val2; \
510 void *result; \
511 val1 = frame_var(fp, slot_1); \
512 val2 = frame_var(fp, slot_2); \
513 result = frame_var(fp, slot_r); \
514 barrier_aliasing(); \
515 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, val2, result))) {\
516 barrier_aliasing(); \
517 goto cat6(escape_int_binary_logical_,op,_,type,_,ARG_MODE);\
518 } \
519 barrier_aliasing(); \
520 ADVANCE_I_PARAM(4); \
521 } \
522 )
524 #define DEFINE_INT_UNARY_OPCODE(type, op) \
525 DEFINE_OPCODE( \
526 OPCODE_INT_OP + \
527 cat(OPCODE_INT_OP_,op) * OPCODE_INT_OP_MULT + \
528 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
529 cat6(int_binary_,op,_,type,_,ARG_MODE), \
530 frame_t slot_1 = get_i_param(0); \
531 frame_t slot_r = get_i_param(1); \
532 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "int unary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
533 if (unlikely(frame_test_flag(fp, slot_1))) { \
534 cat6(escape_int_unary_,op,_,type,_,ARG_MODE): \
535 function_int_unary = cat(mpint_,op); \
536 goto cat(escape_int_unary_thunk_,ARG_MODE); \
537 } else { \
538 const void *val1; \
539 void *result; \
540 val1 = frame_var(fp, slot_1); \
541 result = frame_var(fp, slot_r); \
542 barrier_aliasing(); \
543 if (unlikely(!cat4(INT_unary_,op,_,type)(val1, result))) {\
544 barrier_aliasing(); \
545 goto cat6(escape_int_unary_,op,_,type,_,ARG_MODE);\
546 } \
547 barrier_aliasing(); \
548 ADVANCE_I_PARAM(3); \
549 } \
550 )
552 #define DEFINE_INT_TO_INT(type) \
553 DEFINE_OPCODE( \
554 OPCODE_INT_OP + \
555 OPCODE_INT_OP_to_int * OPCODE_INT_OP_MULT + \
556 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
557 cat4(INT_unary_to_int_,type,_,ARG_MODE), \
558 frame_t slot_1 = get_i_param(0); \
559 frame_t slot_r = get_i_param(1); \
560 if (unlikely(frame_test_flag(fp, slot_1))) { \
561 goto cat(escape_convert_thunk_,ARG_MODE); \
562 } else { \
563 type val1; \
564 int_default_t r; \
565 barrier_aliasing(); \
566 val1 = *frame_slot(fp, slot_1, type); \
567 r = (int_default_t)val1; \
568 if (unlikely(r != val1)) \
569 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, false));\
570 else \
571 *frame_slot(fp, slot_r, int_default_t) = r; \
572 barrier_aliasing(); \
573 ADVANCE_I_PARAM(3); \
574 } \
575 )
577 #define DEFINE_INT_FROM_INT(typeid, type) \
578 DEFINE_OPCODE( \
579 OPCODE_INT_OP + \
580 OPCODE_INT_OP_from_int * OPCODE_INT_OP_MULT + \
581 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
582 cat4(INT_unary_from_int_,type,_,ARG_MODE), \
583 frame_t slot_1 = get_i_param(0); \
584 frame_t slot_r = get_i_param(1); \
585 if (unlikely(frame_test_flag(fp, slot_1))) { \
586 goto cat(escape_convert_thunk_,ARG_MODE); \
587 } else { \
588 int_default_t val1; \
589 type r; \
590 barrier_aliasing(); \
591 val1 = *frame_slot(fp, slot_1, int_default_t); \
592 r = (type)val1; \
593 if (unlikely(r != val1)) \
594 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, false));\
595 else \
596 *frame_slot(fp, slot_r, type) = r; \
597 barrier_aliasing(); \
598 ADVANCE_I_PARAM(3); \
599 } \
600 )
602 #define DEFINE_INT_LDC_OPCODE(type, opc, shrt) \
603 DEFINE_OPCODE( \
604 OPCODE_INT_OP + \
605 OPCODE_INT_OP_##opc * OPCODE_INT_OP_MULT + \
606 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
607 cat6(int_,opc,_,type,_,ARG_MODE), \
608 frame_t slot; \
609 type *val; \
610 ip_t sz; \
611 slot = get_i_param(0); \
612 ajla_assert(!frame_test_flag(fp, slot), (file_line, "ldc int: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot));\
613 val = frame_slot(fp, slot, type); \
614 barrier_aliasing(); \
615 sz = cat(int_ldc_,type)(val, ip + 1 + param_size(1), shrt); \
616 barrier_aliasing(); \
617 ADVANCE_I_PARAM(1); \
618 ADVANCE_IP(sz); \
619 )
621 #define DEFINE_INT_OPCODES(typeid, s, u, sz, bits) \
622 DEFINE_INT_BINARY_OPCODE(s, add) \
623 DEFINE_INT_BINARY_OPCODE(s, subtract) \
624 DEFINE_INT_BINARY_OPCODE(s, multiply) \
625 DEFINE_INT_BINARY_OPCODE(s, divide) \
626 DEFINE_INT_BINARY_OPCODE(s, modulo) \
627 DEFINE_INT_BINARY_OPCODE(s, power) \
628 DEFINE_INT_BINARY_OPCODE(s, and) \
629 DEFINE_INT_BINARY_OPCODE(s, or) \
630 DEFINE_INT_BINARY_OPCODE(s, xor) \
631 DEFINE_INT_BINARY_OPCODE(s, shl) \
632 DEFINE_INT_BINARY_OPCODE(s, shr) \
633 DEFINE_INT_BINARY_OPCODE(s, bts) \
634 DEFINE_INT_BINARY_OPCODE(s, btr) \
635 DEFINE_INT_BINARY_OPCODE(s, btc) \
636 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, equal) \
637 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, not_equal) \
638 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, less) \
639 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, less_equal) \
640 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, bt) \
641 DEFINE_INT_UNARY_OPCODE(s, not) \
642 DEFINE_INT_UNARY_OPCODE(s, neg) \
643 DEFINE_INT_UNARY_OPCODE(s, inc) \
644 DEFINE_INT_UNARY_OPCODE(s, dec) \
645 DEFINE_INT_UNARY_OPCODE(s, bsf) \
646 DEFINE_INT_UNARY_OPCODE(s, bsr) \
647 DEFINE_INT_UNARY_OPCODE(s, popcnt) \
648 DEFINE_INT_TO_INT(s) \
649 DEFINE_INT_FROM_INT(typeid, s) \
650 DEFINE_SCALAR_MOVE_OPCODE(INT_, s) \
651 DEFINE_SCALAR_COPY_OPCODE(INT_, s) \
652 DEFINE_INT_LDC_OPCODE(s, ldc, false) \
653 DEFINE_INT_LDC_OPCODE(s, ldc16, true)
655 for_all_int(DEFINE_INT_OPCODES, for_all_empty)
657 DEFINE_LABEL(cat(escape_int_binary_thunk_,ARG_MODE),
658 void *ex;
659 ASM_PREVENT_CSE;
660 ex = thunk_int_binary_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3), function_int_binary);
661 if (ex == POINTER_FOLLOW_THUNK_GO)
662 ADVANCE_I_PARAM(4);
663 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
664 RELOAD_EX_POSITION(ex);
665 )
667 DEFINE_LABEL(cat(escape_int_binary_logical_thunk_,ARG_MODE),
668 void *ex;
669 ASM_PREVENT_CSE;
670 ex = thunk_int_binary_logical_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3), function_int_binary_logical);
671 if (ex == POINTER_FOLLOW_THUNK_GO)
672 ADVANCE_I_PARAM(4);
673 else if (ex != POINTER_FOLLOW_THUNK_RETRY)
674 RELOAD_EX_POSITION(ex);
675 )
677 DEFINE_LABEL(cat(escape_int_unary_thunk_,ARG_MODE),
678 void *ex;
679 ASM_PREVENT_CSE;
680 ex = thunk_int_unary_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), function_int_unary);
681 if (ex == POINTER_FOLLOW_THUNK_GO)
682 ADVANCE_I_PARAM(3);
683 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
684 RELOAD_EX_POSITION(ex);
685 )
687 #define file_inc "ipret-a1.inc"
688 #include "for-fix.inc"
690 #define file_inc "ipret-a2.inc"
691 #include "for-int.inc"
693 #define file_inc "ipret-a3.inc"
694 #include "for-real.inc"
696 END_BLOCK()
699 #define DEFINE_BOOL_BINARY_OPCODE(op, operator) \
700 DEFINE_OPCODE( \
701 OPCODE_BOOL_OP + \
702 cat(OPCODE_BOOL_OP_,op) * OPCODE_BOOL_OP_MULT, \
703 cat4(bool_binary_,op,_,ARG_MODE), \
704 frame_t slot_1 = get_i_param(0); \
705 frame_t slot_2 = get_i_param(1); \
706 frame_t slot_r = get_i_param(2); \
707 \
708 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
709 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "bool binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
710 goto cat(escape_bool_binary_thunk_,ARG_MODE); \
711 } else { \
712 ajla_flat_option_t val1; \
713 ajla_flat_option_t val2; \
714 ajla_flat_option_t result; \
715 barrier_aliasing(); \
716 val1 = *frame_slot(fp, slot_1, ajla_flat_option_t); \
717 val2 = *frame_slot(fp, slot_2, ajla_flat_option_t); \
718 result = val1 operator val2; \
719 *frame_slot(fp, slot_r, ajla_flat_option_t) = result; \
720 barrier_aliasing(); \
721 ADVANCE_I_PARAM(4); \
722 } \
723 )
725 #define DEFINE_BOOL_UNARY_OPCODE(op, operator) \
726 DEFINE_OPCODE( \
727 OPCODE_BOOL_OP + \
728 cat(OPCODE_BOOL_OP_,op) * OPCODE_BOOL_OP_MULT, \
729 cat4(bool_unary_,op,_,ARG_MODE), \
730 frame_t slot_1 = get_i_param(0); \
731 frame_t slot_r = get_i_param(1); \
732 \
733 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "bool unary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
734 if (unlikely(frame_test_flag(fp, slot_1))) { \
735 goto cat(escape_bool_unary_thunk_,ARG_MODE); \
736 } else { \
737 ajla_flat_option_t val1; \
738 ajla_flat_option_t result; \
739 barrier_aliasing(); \
740 val1 = *frame_slot(fp, slot_1, ajla_flat_option_t); \
741 result = val1 operator; \
742 *frame_slot(fp, slot_r, ajla_flat_option_t) = result; \
743 barrier_aliasing(); \
744 ADVANCE_I_PARAM(3); \
745 } \
746 )
748 DEFINE_BOOL_BINARY_OPCODE(and, &)
749 DEFINE_BOOL_BINARY_OPCODE(or, |)
750 DEFINE_BOOL_BINARY_OPCODE(equal, ==)
751 DEFINE_BOOL_BINARY_OPCODE(not_equal, ^)
752 DEFINE_BOOL_BINARY_OPCODE(less, <)
753 DEFINE_BOOL_BINARY_OPCODE(less_equal, <=)
754 DEFINE_BOOL_UNARY_OPCODE(not, ^ 1)
756 DEFINE_LABEL(cat(escape_bool_binary_thunk_,ARG_MODE),
757 void *ex;
758 ASM_PREVENT_CSE;
759 ex = thunk_bool_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3));
760 if (ex == POINTER_FOLLOW_THUNK_GO)
761 ADVANCE_I_PARAM(4);
762 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
763 RELOAD_EX_POSITION(ex);
764 )
766 DEFINE_LABEL(cat(escape_bool_unary_thunk_,ARG_MODE),
767 void *ex;
768 ASM_PREVENT_CSE;
769 ex = thunk_bool_operator(fp, ip, get_i_param(0), NO_FRAME_T, get_i_param(1), get_i_param(2));
770 if (ex == POINTER_FOLLOW_THUNK_GO)
771 ADVANCE_I_PARAM(3);
772 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
773 RELOAD_EX_POSITION(ex);
774 )
777 DEFINE_OPCODE_START_LBL(OPCODE_BOOL_OP + OPCODE_BOOL_OP_move, cat(bool_move_,ARG_MODE))
778 #ifdef EMIT_CODE
779 move_scalar(ajla_flat_option_t);
780 #endif
781 DEFINE_OPCODE_END(OPCODE_BOOL_OP + OPCODE_BOOL_OP_move)
783 DEFINE_OPCODE_START_LBL(OPCODE_BOOL_OP + OPCODE_BOOL_OP_copy, cat(bool_copy_,ARG_MODE))
784 #ifdef EMIT_CODE
785 copy_scalar(ajla_flat_option_t);
786 #endif
787 DEFINE_OPCODE_END(OPCODE_BOOL_OP + OPCODE_BOOL_OP_copy)
790 DEFINE_OPCODE_START(OPCODE_INT_LDC_LONG)
791 #ifdef EMIT_CODE
792 frame_t slot;
793 ip_t sz;
794 slot = get_i_param(0);
795 sz = ipret_int_ldc_long(fp, slot, ip + 1 + param_size(1));
796 ADVANCE_I_PARAM(1);
797 ADVANCE_IP(sz);
798 #endif
799 DEFINE_OPCODE_END(OPCODE_INT_LDC_LONG)
802 DEFINE_OPCODE_START(OPCODE_IS_EXCEPTION)
803 #ifdef EMIT_CODE
804 frame_t slot_1 = get_i_param(0);
805 frame_t slot_r = get_i_param(1);
806 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "is exception: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
807 if (!frame_variable_is_flat(fp, slot_1)) {
808 void *ex = is_thunk_operator(fp, ip, slot_1, slot_r, get_i_param(2));
809 if (ex == POINTER_FOLLOW_THUNK_GO)
810 ADVANCE_I_PARAM(3);
811 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
812 RELOAD_EX_POSITION(ex);
813 } else {
814 barrier_aliasing();
815 *frame_slot(fp, slot_r, ajla_flat_option_t) = 0;
816 barrier_aliasing();
817 ADVANCE_I_PARAM(3);
818 }
819 #endif
820 DEFINE_OPCODE_END(OPCODE_IS_EXCEPTION)
822 DEFINE_OPCODE_START(OPCODE_EXCEPTION_CLASS)
823 #ifdef EMIT_CODE
824 void *ex;
825 ex = thunk_get_param(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), 0);
826 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
827 ADVANCE_I_PARAM(3);
828 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
829 RELOAD_EX_POSITION(ex);
830 #endif
831 DEFINE_OPCODE_END(OPCODE_EXCEPTION_CLASS)
833 DEFINE_OPCODE_START(OPCODE_EXCEPTION_TYPE)
834 #ifdef EMIT_CODE
835 void *ex;
836 ex = thunk_get_param(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), 1);
837 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
838 ADVANCE_I_PARAM(3);
839 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
840 RELOAD_EX_POSITION(ex);
841 #endif
842 DEFINE_OPCODE_END(OPCODE_EXCEPTION_TYPE)
844 DEFINE_OPCODE_START(OPCODE_EXCEPTION_AUX)
845 #ifdef EMIT_CODE
846 void *ex;
847 ex = thunk_get_param(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), 2);
848 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
849 ADVANCE_I_PARAM(3);
850 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
851 RELOAD_EX_POSITION(ex);
852 #endif
853 DEFINE_OPCODE_END(OPCODE_EXCEPTION_AUX)
855 DEFINE_OPCODE_START(OPCODE_SYSTEM_PROPERTY)
856 #ifdef EMIT_CODE
857 void *ex;
858 ex = ipret_get_system_property(fp, ip, get_i_param(0), get_i_param(1));
859 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
860 ADVANCE_I_PARAM(3);
861 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
862 RELOAD_EX_POSITION(ex);
863 #endif
864 DEFINE_OPCODE_END(OPCODE_SYSTEM_PROPERTY)
867 DEFINE_OPCODE_START(OPCODE_FLAT_MOVE)
868 #ifdef EMIT_CODE
869 frame_t slot_1 = get_i_param(0);
870 frame_t slot_r = get_i_param(1);
871 if (likely(frame_variable_is_flat(fp, slot_1))) {
872 memcpy(frame_var(fp, slot_r), frame_var(fp, slot_1), frame_get_type_of_local(fp, slot_1)->size);
873 } else {
874 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, true);
875 frame_set_pointer(fp, slot_r, ptr);
876 }
877 ADVANCE_I_PARAM(2);
878 #endif
879 DEFINE_OPCODE_END(OPCODE_FLAT_MOVE)
881 DEFINE_OPCODE_START(OPCODE_FLAT_COPY)
882 #ifdef EMIT_CODE
883 frame_t slot_1 = get_i_param(0);
884 frame_t slot_r = get_i_param(1);
885 if (likely(frame_variable_is_flat(fp, slot_1))) {
886 memcpy(frame_var(fp, slot_r), frame_var(fp, slot_1), frame_get_type_of_local(fp, slot_1)->size);
887 } else {
888 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, false);
889 frame_set_pointer(fp, slot_r, ptr);
890 }
891 ADVANCE_I_PARAM(2);
892 #endif
893 DEFINE_OPCODE_END(OPCODE_FLAT_COPY)
895 DEFINE_OPCODE_START(OPCODE_REF_MOVE)
896 #ifdef EMIT_CODE
897 frame_t slot_1 = get_i_param(0);
898 frame_t slot_r = get_i_param(1);
899 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, true);
900 pointer_poison(frame_pointer(fp, slot_1));
901 frame_set_pointer(fp, slot_r, ptr);
902 ADVANCE_I_PARAM(2);
903 #endif
904 DEFINE_OPCODE_END(OPCODE_REF_MOVE)
906 DEFINE_OPCODE_START(OPCODE_REF_MOVE_CLEAR)
907 #ifdef EMIT_CODE
908 frame_t slot_1 = get_i_param(0);
909 frame_t slot_r = get_i_param(1);
910 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, true);
911 frame_set_pointer(fp, slot_r, ptr);
912 ADVANCE_I_PARAM(2);
913 #endif
914 DEFINE_OPCODE_END(OPCODE_REF_MOVE_CLEAR)
916 DEFINE_OPCODE_START(OPCODE_REF_COPY)
917 #ifdef EMIT_CODE
918 frame_t slot_1 = get_i_param(0);
919 frame_t slot_r = get_i_param(1);
920 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, false);
921 frame_set_pointer(fp, slot_r, ptr);
922 ADVANCE_I_PARAM(2);
923 #endif
924 DEFINE_OPCODE_END(OPCODE_REF_COPY)
926 DEFINE_OPCODE_START(OPCODE_BOX_MOVE_CLEAR)
927 #ifdef EMIT_CODE
928 frame_t slot_1 = get_i_param(0);
929 frame_t slot_r = get_i_param(1);
930 pointer_t ptr = ipret_copy_variable_to_pointer(fp, slot_1, true);
931 *frame_pointer(fp, slot_1) = pointer_empty();
932 frame_set_pointer(fp, slot_r, ptr);
933 ADVANCE_I_PARAM(2);
934 #endif
935 DEFINE_OPCODE_END(OPCODE_BOX_MOVE_CLEAR)
937 DEFINE_OPCODE_START(OPCODE_BOX_COPY)
938 #ifdef EMIT_CODE
939 frame_t slot_1 = get_i_param(0);
940 frame_t slot_r = get_i_param(1);
941 pointer_t ptr = ipret_copy_variable_to_pointer(fp, slot_1, false);
942 frame_set_pointer(fp, slot_r, ptr);
943 ADVANCE_I_PARAM(2);
944 #endif
945 DEFINE_OPCODE_END(OPCODE_BOX_COPY)
947 DEFINE_OPCODE_START(OPCODE_TAKE_BORROWED)
948 #ifdef EMIT_CODE
949 frame_t slot = get_i_param(0);
950 if (!frame_test_and_set_flag(fp, slot)) {
951 pointer_t ptr = *frame_pointer(fp, slot);
952 pointer_reference_owned(ptr);
953 }
954 ADVANCE_I_PARAM(1);
955 #endif
956 DEFINE_OPCODE_END(OPCODE_TAKE_BORROWED)
959 DEFINE_OPCODE_START(OPCODE_DEREFERENCE)
960 #ifdef EMIT_CODE
961 frame_t slot = get_i_param(0);
962 #if 0
963 {
964 struct data *fn = get_frame(fp)->function;
965 const struct type *type = frame_get_type_of_local(fp, slot);
966 bool may_be_borrowed = da(fn,function)->local_variables[slot].may_be_borrowed;
967 if (!frame_test_flag(fp, slot) && !may_be_borrowed && !TYPE_IS_FLAT(type))
968 internal(file_line, "dereference: %s: the value is unexpectedly borrowed", da(fn,function)->function_name);
969 }
970 #endif
971 frame_free(fp, slot);
972 pointer_poison(frame_pointer(fp, slot));
973 ADVANCE_I_PARAM(1);
974 #endif
975 DEFINE_OPCODE_END(OPCODE_DEREFERENCE)
977 DEFINE_OPCODE_START(OPCODE_DEREFERENCE_CLEAR)
978 #ifdef EMIT_CODE
979 frame_t slot = get_i_param(0);
980 #if 0
981 {
982 struct data *fn = get_frame(fp)->function;
983 const struct type *type = frame_get_type_of_local(fp, slot);
984 bool may_be_borrowed = da(fn,function)->local_variables[slot].may_be_borrowed;
985 if (!frame_test_flag(fp, slot) && !may_be_borrowed && !TYPE_IS_FLAT(type))
986 internal(file_line, "dereference clear: %s: the value is unexpectedly borrowed", da(fn,function)->function_name);
987 }
988 #endif
989 frame_free_and_clear(fp, slot);
990 ADVANCE_I_PARAM(1);
991 #endif
992 DEFINE_OPCODE_END(OPCODE_DEREFERENCE_CLEAR)
994 DEFINE_OPCODE_START(OPCODE_EVAL)
995 #ifdef EMIT_CODE
996 frame_t slot = get_i_param(0);
997 if (frame_test_flag(fp, slot)) {
998 pointer_t *ptr = frame_pointer(fp, slot);
999 struct data attr_unused *result;
1000 /*cat(eval_again_,ARG_MODE):*/
1001 pointer_follow(ptr, true, result, PF_WAIT, fp, ip,
1002 RELOAD_EX_POSITION(ex_),
1003 goto cat(eval_skip_,ARG_MODE));
1004 /*
1005 if (unlikely(da_tag(result) == DATA_TAG_array_incomplete)) {
1006 ptr = &da(result,array_incomplete)->next;
1007 goto cat(eval_again_,ARG_MODE);
1008 }
1009 */
1010 cat(eval_skip_,ARG_MODE):;
1011 }
1012 ADVANCE_I_PARAM(1);
1013 #endif
1014 DEFINE_OPCODE_END(OPCODE_EVAL)
1017 #if ARG_MODE >= 1
1018 DEFINE_OPCODE_START(OPCODE_ESCAPE_NONFLAT)
1019 #ifdef EMIT_CODE
1020 frame_t n = get_i_param(0);
1021 ADVANCE_I_PARAM(1);
1022 ADVANCE_IP(n * ARG_MODE);
1023 #endif
1024 DEFINE_OPCODE_END(OPCODE_ESCAPE_NONFLAT)
1025 #endif
1027 #if ARG_MODE >= 1
1028 DEFINE_OPCODE_START(OPCODE_CHECKPOINT)
1029 #ifdef EMIT_CODE
1030 const code_t attr_unused *orig_ip = ip;
1031 frame_t attr_unused id;
1032 ip_t len;
1033 if (SIZEOF_IP_T == 2) {
1034 id = ip[1];
1035 ADVANCE_IP(2);
1036 } else if (SIZEOF_IP_T == 4) {
1037 id = get_unaligned_32(&ip[1]);
1038 ADVANCE_IP(3);
1039 } else {
1040 not_reached();
1041 }
1042 len = get_param(ip, 0);
1043 ADVANCE_IP((len + 1) * ARG_MODE);
1045 if (unlikely(tick_elapsed(&ts)) && likely(frame_execution_control(fp) != NULL)) {
1046 ipret_checkpoint_forced;
1047 } else {
1048 #ifdef HAVE_CODEGEN
1049 struct data *fn = get_frame(fp)->function;
1050 code_return_t r;
1051 void *new_fp;
1052 ip_t new_ip;
1053 struct data *codegen;
1054 if (unlikely(load_relaxed(&da(fn,function)->codegen_failed)))
1055 goto cat(checkpoint_exit_,ARG_MODE);
1056 #if 1
1057 pointer_follow(&da(fn,function)->codegen, false, codegen, PF_WAIT, fp, orig_ip,
1058 RELOAD_EX_POSITION(ex_),
1059 store_relaxed(&da(fn,function)->codegen_failed, 1);
1060 goto cat(checkpoint_exit_,ARG_MODE);
1061 );
1062 #else
1063 pointer_follow(&da(fn,function)->codegen, false, codegen, PF_SPARK, NULL, 0,
1064 /*debug("sparked: %s %p", da(fn,function)->function_name, ex_);*/
1065 SUBMIT_EX(ex_);
1066 goto cat(checkpoint_exit_,ARG_MODE),
1067 store_relaxed(&da(fn,function)->codegen_failed, 1);
1068 goto cat(checkpoint_exit_,ARG_MODE);
1069 );
1070 #endif
1072 /*for (frame_t l = MIN_USEABLE_SLOT; l < function_n_variables(fn); l++) {
1073 if (da(fn,function)->local_variables_flags[l].must_be_flat) {
1074 if (unlikely(frame_test_flag(fp, l)))
1075 goto cat(checkpoint_exit_,ARG_MODE);
1076 }
1077 }*/
1079 /*debug("running optimized code: %s, %p, %x, %p, %p, %p, %u, %p", da(fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, id, da(codegen,codegen)->unoptimized_code[id], fp, &cg_upcall_vector, ts, da(codegen,codegen)->unoptimized_code[id]);*/
1080 /*debug("fp[0] = %lx", *(long *)((char *)fp + 0));
1081 debug("fp[8] = %lx", *(long *)((char *)fp + 8));
1082 debug("fp[16] = %lx", *(long *)((char *)fp + 16));
1083 debug("fp[24] = %lx", *(long *)((char *)fp + 24));
1084 debug("fp[32] = %lx", *(long *)((char *)fp + 32));
1085 debug("fp[40] = %lx", *(long *)((char *)fp + 40));*/
1086 /*{
1087 uintptr_t *stub = (void *)da(codegen,codegen)->unoptimized_code[id];
1088 debug("entry: %p, %lx %lx %lx %lx", stub, stub[0], stub[1], stub[2], stub[3]);
1089 }*/
1090 /*__asm__ volatile("nopr %r6");*/
1091 /*__asm__ volatile("xnop");*/
1092 /*__asm__ volatile("cover");*/
1093 /*debug("calling: %p, %p, %lx, %lx", da(codegen,codegen)->unoptimized_code[id], codegen_entry, ((long *)codegen_entry)[0], ((long *)codegen_entry)[1]);*/
1094 /*debug("calling: %p, %p, %lx, %lx", da(codegen,codegen)->unoptimized_code[id], codegen_entry, ((long *)((char *)codegen_entry - 2))[0], ((long *)((char *)codegen_entry - 2))[1]);*/
1095 r = codegen_entry(fp, &cg_upcall_vector, ts, da(codegen,codegen)->unoptimized_code[id]);
1096 #if defined(ARCH_X86_32) || defined(ARCH_ARM32) || defined(ARCH_MIPS32) || defined(ARCH_POWER32) || defined(ARCH_SPARC32)
1097 #if defined(C_LITTLE_ENDIAN) || defined(ARCH_MIPS_N32)
1098 new_fp = num_to_ptr(r & 0xffffffffU);
1099 new_ip = r >> 32;
1100 #else
1101 new_fp = num_to_ptr(r >> 32);
1102 new_ip = r & 0xffffffffU;
1103 #endif
1104 #else
1105 new_fp = r.fp;
1106 new_ip = r.ip;
1107 #endif
1108 /*debug("ran optimized code: %s, %p, %lx -> %p, %x", da(fn,function)->function_name, fp, (unsigned long)(ip - da(fn,function)->code), new_fp, new_ip);*/
1109 if (unlikely(new_ip == (ip_t)-1)) {
1110 /*debug("reload on optimized code: %p", new_fp);*/
1111 RELOAD_EX_POSITION(new_fp);
1112 } else {
1113 /*if (fp != new_fp) debug("switching function: %s -> %s", da(fn,function)->function_name, da(get_frame(new_fp)->function,function)->function_name);*/
1114 fp = new_fp;
1115 ip = da(get_frame(fp)->function,function)->code + new_ip;
1116 if (unlikely(profiling_escapes)) {
1117 profile_counter_t profiling_counter;
1118 fn = get_frame(fp)->function;
1119 profiling_counter = load_relaxed(&da(fn,function)->escape_data[new_ip].counter);
1120 profiling_counter++;
1121 store_relaxed(&da(fn,function)->escape_data[new_ip].counter, profiling_counter);
1122 }
1123 }
1124 #endif
1125 goto cat(checkpoint_exit_,ARG_MODE);
1126 cat(checkpoint_exit_,ARG_MODE):;
1127 }
1128 #endif
1129 DEFINE_OPCODE_END(OPCODE_CHECKPOINT)
1130 #endif
1133 #if ARG_MODE == 0
1134 DEFINE_OPCODE_START(OPCODE_JMP)
1135 #ifdef EMIT_CODE
1136 if (SIZEOF_IP_T == 2) {
1137 int16_t offset = ip[1];
1138 ADVANCE_IP(2);
1139 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1140 } else if (SIZEOF_IP_T == 4) {
1141 int32_t offset = get_unaligned_32(&ip[1]);
1142 ADVANCE_IP(3);
1143 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1144 } else {
1145 not_reached();
1146 }
1147 #endif
1148 DEFINE_OPCODE_END(OPCODE_JMP)
1149 #endif
1151 #if ARG_MODE == 0 && SIZEOF_IP_T > 2
1152 DEFINE_OPCODE_START(OPCODE_JMP_BACK_16)
1153 #ifdef EMIT_CODE
1154 uint16_t offset;
1156 offset = ip[1];
1157 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) - offset);
1158 #endif
1159 DEFINE_OPCODE_END(OPCODE_JMP_BACK_16)
1160 #endif
1162 DEFINE_OPCODE_START(OPCODE_JMP_FALSE)
1163 #ifdef EMIT_CODE
1164 int32_t offset;
1165 frame_t slot;
1166 ajla_flat_option_t val1;
1168 slot = get_i_param(0);
1169 if (unlikely(frame_test_flag(fp, slot))) {
1170 void *ex = thunk_bool_jump(fp, ip, slot);
1171 if (ex != POINTER_FOLLOW_THUNK_RETRY) {
1172 if (likely(ex != POINTER_FOLLOW_THUNK_EXCEPTION)) {
1173 RELOAD_EX_POSITION(ex);
1174 } else {
1175 if (SIZEOF_IP_T == 2) {
1176 offset = ip[1 + param_size(1) + 1];
1177 ADVANCE_I_PARAM(1);
1178 ADVANCE_IP(2);
1179 } else if (SIZEOF_IP_T == 4) {
1180 offset = get_unaligned_32(&ip[1 + param_size(1) + 2]);
1181 ADVANCE_I_PARAM(1);
1182 ADVANCE_IP(4);
1183 } else {
1184 not_reached();
1185 }
1186 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1187 goto cat(jmp_false_exception_,ARG_MODE);
1188 }
1189 }
1190 }
1191 barrier_aliasing();
1192 val1 = *frame_slot(fp, slot, ajla_flat_option_t);
1193 barrier_aliasing();
1194 if (SIZEOF_IP_T == 2) {
1195 offset = ip[1 + param_size(1)];
1196 ADVANCE_I_PARAM(1);
1197 ADVANCE_IP(2);
1198 } else if (SIZEOF_IP_T == 4) {
1199 offset = get_unaligned_32(&ip[1 + param_size(1)]);
1200 ADVANCE_I_PARAM(1);
1201 ADVANCE_IP(4);
1202 } else {
1203 not_reached();
1204 }
1205 if (!val1)
1206 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1208 cat(jmp_false_exception_,ARG_MODE):;
1209 #endif
1210 DEFINE_OPCODE_END(OPCODE_JMP_FALSE)
1212 #if ARG_MODE == 0
1213 DEFINE_OPCODE_START(OPCODE_LABEL)
1214 #ifdef EMIT_CODE
1215 ADVANCE_IP(1);
1216 #endif
1217 DEFINE_OPCODE_END(OPCODE_LABEL)
1218 #endif
1221 START_BLOCK(
1222 pointer_t *direct_data;
1223 pointer_t indirect_data;
1224 arg_t n_arguments;
1225 frame_t result_slot;
1226 struct thunk *function_error;
1227 )
1229 DEFINE_OPCODE_START(OPCODE_LOAD_FN)
1230 #ifdef EMIT_CODE
1231 struct data *new_reference;
1232 pointer_t result_ptr;
1234 n_arguments = get_i_param(0);
1235 result_slot = get_i_param(1);
1236 direct_data = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1237 indirect_data = pointer_empty(); /* avoid warning */
1239 ADVANCE_I_PARAM(3);
1241 cat(fn_copy_arguments_,ARG_MODE):
1243 new_reference = data_alloc_function_reference_mayfail(n_arguments, &ajla_error pass_file_line);
1244 if (unlikely(!new_reference)) {
1245 function_error = thunk_alloc_exception_error(ajla_error, NULL, fp, ip pass_file_line);
1246 if (!direct_data)
1247 pointer_dereference(indirect_data);
1248 /*cat(fn_set_error_,ARG_MODE):*/
1249 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_arguments);
1250 result_ptr = pointer_thunk(function_error);
1251 } else {
1252 arg_t i;
1253 if (direct_data) {
1254 da(new_reference,function_reference)->is_indirect = false;
1255 da(new_reference,function_reference)->u.direct = direct_data;
1256 } else {
1257 da(new_reference,function_reference)->is_indirect = true;
1258 da(new_reference,function_reference)->u.indirect = indirect_data;
1259 }
1260 for (i = 0; i < n_arguments; i++) {
1261 ipret_fill_function_reference_from_slot(new_reference, i, fp, get_param(ip, 0), (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1262 ADVANCE_IP(param_size(2));
1263 }
1264 result_ptr = pointer_data(new_reference);
1265 }
1267 frame_set_pointer(fp, result_slot, result_ptr);
1268 #endif
1269 DEFINE_OPCODE_END(OPCODE_LOAD_FN)
1271 DEFINE_OPCODE_START(OPCODE_CURRY)
1272 #ifdef EMIT_CODE
1273 frame_t fn_ref_slot;
1275 n_arguments = get_i_param(0);
1276 result_slot = get_i_param(1);
1278 fn_ref_slot = get_i_param(2);
1279 direct_data = NULL;
1280 indirect_data = frame_get_pointer_reference(fp, fn_ref_slot, (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1282 ADVANCE_I_PARAM(4);
1284 goto cat(fn_copy_arguments_,ARG_MODE);
1285 #endif
1286 DEFINE_OPCODE_END(OPCODE_CURRY)
1288 END_BLOCK()
1290 START_BLOCK(
1291 arg_t n_arguments;
1292 arg_t n_return_values;
1293 struct thunk *function_error;
1294 struct data *function;
1295 const code_t *orig_ip;
1296 frame_s *new_fp;
1297 arg_t i;
1298 pointer_t *direct_function;
1299 pointer_t indirect_function;
1300 unsigned char mode;
1301 )
1302 DEFINE_OPCODE_START(OPCODE_CALL)
1303 #ifdef EMIT_CODE
1304 pointer_t *function_ptr;
1306 mode = CALL_MODE_NORMAL;
1307 cat(call_from_mode_,ARG_MODE):
1309 n_arguments = get_i_param(0);
1310 n_return_values = get_i_param(1);
1311 function_ptr = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1313 pointer_follow(function_ptr, false, function, PF_WAIT, fp, ip,
1314 RELOAD_EX_POSITION(ex_),
1315 function_error = thunk_;
1316 thunk_reference(function_error);
1317 ADVANCE_I_PARAM(3);
1318 goto cat(call_set_error_,ARG_MODE);
1319 );
1321 ajla_assert(da(function,function)->n_arguments == n_arguments && da(function,function)->n_return_values == n_return_values,
1322 (file_line, "call %s->%s: the number of arguments does not match: %lu != %lu || %lu != %lu",
1323 da(get_frame(fp)->function,function)->function_name, da(function,function)->function_name,
1324 (unsigned long)da(function,function)->n_arguments, (unsigned long)n_arguments,
1325 (unsigned long)da(function,function)->n_return_values, (unsigned long)n_return_values));
1327 ADVANCE_I_PARAM(3);
1329 new_fp = frame_build(fp, function, &ajla_error);
1330 if (unlikely(!new_fp)) {
1331 cat(call_allocation_error_,ARG_MODE):
1332 function_error = thunk_alloc_exception_error(ajla_error, NULL, fp, ip pass_file_line);
1333 cat(call_set_error_,ARG_MODE):
1334 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_arguments);
1335 pointer_reference_owned_multiple(pointer_thunk(function_error), n_return_values - 1);
1336 i = n_return_values;
1337 while (1) {
1338 frame_t slot_r = get_max_param(ip, 0);
1339 ADVANCE_IP(max_param_size(1) + 1);
1340 frame_set_pointer(fp, slot_r, pointer_thunk(function_error));
1341 if (!--i)
1342 break;
1343 }
1344 } else {
1345 fp = frame_up(new_fp);
1346 frame_init(new_fp, function, get_frame(fp)->timestamp, mode);
1348 i = 0;
1349 cat(call_copy_arguments_,ARG_MODE):
1350 for (; i < da(function,function)->n_arguments; i++) {
1351 frame_t src_slot = get_param(ip, 0);
1352 frame_t dst_slot = da(function,function)->args[i].slot;
1353 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1354 bool may_be_borrowed = da(function,function)->args[i].may_be_borrowed;
1355 if (may_be_borrowed && src_flag & OPCODE_CALL_MAY_LEND && !pointer_is_thunk(*frame_pointer(fp, src_slot))) {
1356 *frame_pointer(new_fp, dst_slot) = *frame_pointer(fp, src_slot);
1357 } else if (may_be_borrowed && src_flag & OPCODE_CALL_MAY_GIVE && !frame_test_flag(fp, src_slot) && !pointer_is_thunk(*frame_pointer(fp, src_slot))) {
1358 *frame_pointer(new_fp, dst_slot) = *frame_pointer(fp, src_slot);
1359 *frame_pointer(fp, src_slot) = pointer_empty();
1360 } else {
1361 ipret_copy_variable(fp, src_slot, new_fp, dst_slot, (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1362 }
1363 ADVANCE_IP(param_size(2));
1364 }
1365 get_frame(new_fp)->previous_ip = frame_ip(fp, ip);
1367 fp = new_fp;
1368 ip = &da(function,function)->code[0];
1370 if (unlikely(mode == CALL_MODE_SPARK)) {
1371 ipret_checkpoint_forced;
1372 }
1373 }
1374 #endif
1375 DEFINE_OPCODE_END(OPCODE_CALL)
1377 DEFINE_OPCODE_START(OPCODE_CALL_STRICT)
1378 #ifdef EMIT_CODE
1379 mode = CALL_MODE_STRICT;
1380 goto cat(call_from_mode_,ARG_MODE);
1381 #endif
1382 DEFINE_OPCODE_END(OPCODE_CALL_STRICT)
1384 DEFINE_OPCODE_START(OPCODE_CALL_SPARK)
1385 #ifdef EMIT_CODE
1386 mode = CALL_MODE_SPARK;
1387 goto cat(call_from_mode_,ARG_MODE);
1388 #endif
1389 DEFINE_OPCODE_END(OPCODE_CALL_SPARK)
1391 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT)
1392 #ifdef EMIT_CODE
1393 frame_t fn_ref_slot;
1394 bool deref;
1395 pointer_t *ptr;
1397 mode = CALL_MODE_NORMAL;
1398 cat(call_indirect_from_mode_,ARG_MODE):
1400 orig_ip = ip;
1401 n_arguments = get_i_param(0);
1402 n_return_values = get_i_param(1);
1403 fn_ref_slot = get_i_param(2);
1404 deref = (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1405 ADVANCE_I_PARAM(4);
1407 ptr = frame_pointer(fp, fn_ref_slot);
1408 while (1) {
1409 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1410 RELOAD_EX_POSITION(ex_),
1411 function_error = thunk_;
1412 thunk_reference(function_error);
1413 if (deref)
1414 frame_free_and_clear(fp, fn_ref_slot);
1415 goto cat(call_set_error_,ARG_MODE)
1416 );
1417 if (!da(function,function_reference)->is_indirect)
1418 break;
1419 ptr = &da(function,function_reference)->u.indirect;
1420 }
1421 ptr = da(function,function_reference)->u.direct;
1422 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1423 RELOAD_EX_POSITION(ex_),
1424 function_error = thunk_;
1425 thunk_reference(function_error);
1426 if (deref)
1427 frame_free_and_clear(fp, fn_ref_slot);
1428 goto cat(call_set_error_,ARG_MODE)
1429 );
1431 new_fp = frame_build(fp, function, &ajla_error);
1432 if (unlikely(!new_fp)) {
1433 if (deref)
1434 frame_free_and_clear(fp, fn_ref_slot);
1435 goto cat(call_allocation_error_,ARG_MODE);
1436 } else {
1437 fp = frame_up(new_fp);
1438 frame_init(new_fp, function, get_frame(fp)->timestamp, mode);
1440 i = da(function,function)->n_arguments - n_arguments;
1442 copy_from_function_reference_to_frame(new_fp, pointer_get_data(*frame_pointer(fp, fn_ref_slot)), i, deref && frame_test_flag(fp, fn_ref_slot));
1443 if (deref)
1444 frame_free_and_clear(fp, fn_ref_slot);
1446 goto cat(call_copy_arguments_,ARG_MODE);
1447 }
1448 #endif
1449 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT)
1451 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_STRICT)
1452 #ifdef EMIT_CODE
1453 mode = CALL_MODE_STRICT;
1454 goto cat(call_indirect_from_mode_,ARG_MODE);
1455 #endif
1456 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_STRICT)
1458 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_SPARK)
1459 #ifdef EMIT_CODE
1460 mode = CALL_MODE_SPARK;
1461 goto cat(call_indirect_from_mode_,ARG_MODE);
1462 #endif
1463 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_SPARK)
1465 DEFINE_OPCODE_START(OPCODE_CALL_LAZY)
1466 #ifdef EMIT_CODE
1467 struct data *function_reference;
1468 struct thunk **results;
1470 n_arguments = get_i_param(0);
1471 n_return_values = get_i_param(1);
1472 direct_function = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1473 indirect_function = pointer_empty(); /* avoid warning */
1475 ADVANCE_I_PARAM(3);
1477 cat(call_lazy_copy_arguments_,ARG_MODE):
1478 results = mem_alloc_array_mayfail(mem_alloc_mayfail, struct thunk **, 0, 0, n_return_values, sizeof(struct thunk *), &ajla_error);
1479 if (unlikely(!results)) {
1480 if (!direct_function)
1481 pointer_dereference(indirect_function);
1482 goto cat(call_allocation_error_,ARG_MODE);
1483 }
1485 if (!(function_reference = data_alloc_function_reference_mayfail(n_arguments, &ajla_error pass_file_line))) {
1486 if (!direct_function)
1487 pointer_dereference(indirect_function);
1488 mem_free(results);
1489 goto cat(call_allocation_error_,ARG_MODE);
1490 }
1491 if (direct_function) {
1492 da(function_reference,function_reference)->is_indirect = false;
1493 da(function_reference,function_reference)->u.direct = direct_function;
1494 } else {
1495 da(function_reference,function_reference)->is_indirect = true;
1496 da(function_reference,function_reference)->u.indirect = indirect_function;
1497 }
1499 if (!thunk_alloc_function_call(pointer_data(function_reference), n_return_values, results, &ajla_error)) {
1500 if (!direct_function)
1501 pointer_dereference(indirect_function);
1502 mem_free(results);
1503 data_free_r1(function_reference);
1504 goto cat(call_allocation_error_,ARG_MODE);
1505 }
1507 for (i = 0; i < n_arguments; i++) {
1508 frame_t src_slot = get_param(ip, 0);
1509 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1510 ipret_fill_function_reference_from_slot(function_reference, i, fp, src_slot, (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1511 ADVANCE_IP(param_size(2));
1512 }
1514 for (i = 0; i < n_return_values; i++) {
1515 frame_t slot_r = get_max_param(ip, 0);
1516 frame_set_pointer(fp, slot_r, pointer_thunk(results[i]));
1517 ADVANCE_IP(max_param_size(1) + 1);
1518 }
1520 mem_free(results);
1521 #endif
1522 DEFINE_OPCODE_END(OPCODE_CALL_LAZY)
1524 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_LAZY)
1525 #ifdef EMIT_CODE
1526 frame_t fn_ref_slot;
1527 bool deref;
1529 n_arguments = get_i_param(0);
1530 n_return_values = get_i_param(1);
1531 fn_ref_slot = get_i_param(2);
1532 deref = (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1533 ADVANCE_I_PARAM(4);
1535 indirect_function = frame_get_pointer_reference(fp, fn_ref_slot, deref);
1536 direct_function = NULL;
1538 goto cat(call_lazy_copy_arguments_,ARG_MODE);
1539 #endif
1540 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_LAZY)
1542 DEFINE_OPCODE_START(OPCODE_CALL_CACHE)
1543 #ifdef EMIT_CODE
1544 struct ipret_call_cache_arg *arguments;
1545 frame_t *return_values;
1546 void *ex;
1548 cat(call_cache_,ARG_MODE):
1549 orig_ip = ip;
1550 n_arguments = get_i_param(0);
1551 n_return_values = get_i_param(1);
1552 direct_function = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1553 ADVANCE_I_PARAM(3);
1555 pointer_follow(direct_function, false, function, PF_WAIT, fp, orig_ip,
1556 RELOAD_EX_POSITION(ex_),
1557 function_error = thunk_;
1558 thunk_reference(function_error);
1559 goto cat(call_set_error_,ARG_MODE);
1560 );
1561 ajla_assert(da(function,function)->n_arguments == n_arguments && da(function,function)->n_return_values == n_return_values,
1562 (file_line, "call_cache: the number of arguments does not match: %lu != %lu || %lu != %lu",
1563 (unsigned long)da(function,function)->n_arguments, (unsigned long)n_arguments,
1564 (unsigned long)da(function,function)->n_return_values, (unsigned long)n_return_values));
1566 arguments = mem_alloc_array_mayfail(mem_alloc_mayfail, struct ipret_call_cache_arg *, 0, 0, n_arguments, sizeof(struct ipret_call_cache_arg), &ajla_error);
1567 if (!arguments) {
1568 goto cat(call_allocation_error_,ARG_MODE);
1569 }
1570 return_values = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n_return_values, sizeof(frame_t), &ajla_error);
1571 if (!return_values) {
1572 mem_free(arguments);
1573 goto cat(call_allocation_error_,ARG_MODE);
1574 }
1576 for (i = 0; i < n_arguments; i++) {
1577 frame_t src_slot = get_param(ip, 0);
1578 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1579 arguments[i].f_arg = NULL;
1580 arguments[i].slot = src_slot;
1581 arguments[i].deref = (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1582 ADVANCE_IP(param_size(2));
1583 }
1585 for (i = 0; i < n_return_values; i++) {
1586 frame_t slot_r = get_max_param(ip, 0);
1587 return_values[i] = slot_r;
1588 ADVANCE_IP(max_param_size(1) + 1);
1589 }
1591 ex = ipret_call_cache(fp, orig_ip, direct_function, arguments, return_values, NO_FRAME_T);
1592 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
1593 RELOAD_EX_POSITION(ex);
1594 }
1595 #endif
1596 DEFINE_OPCODE_END(OPCODE_CALL_CACHE)
1598 DEFINE_OPCODE_START(OPCODE_CALL_SAVE)
1599 #ifdef EMIT_CODE
1600 goto cat(call_cache_,ARG_MODE);
1601 #endif
1602 DEFINE_OPCODE_END(OPCODE_CALL_SAVE)
1604 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_CACHE)
1605 #ifdef EMIT_CODE
1606 struct ipret_call_cache_arg *arguments;
1607 frame_t *return_values;
1608 void *ex;
1610 frame_t fn_ref_slot;
1611 bool deref;
1612 pointer_t *ptr;
1613 arg_t n_curried_arguments;
1615 cat(call_indirect_cache_,ARG_MODE):
1616 orig_ip = ip;
1617 n_arguments = get_i_param(0);
1618 n_return_values = get_i_param(1);
1619 fn_ref_slot = get_i_param(2);
1620 deref = (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1621 ADVANCE_I_PARAM(4);
1623 n_curried_arguments = 0;
1624 ptr = frame_pointer(fp, fn_ref_slot);
1625 while (1) {
1626 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1627 RELOAD_EX_POSITION(ex_),
1628 function_error = thunk_;
1629 thunk_reference(function_error);
1630 if (deref)
1631 frame_free_and_clear(fp, fn_ref_slot);
1632 goto cat(call_set_error_,ARG_MODE)
1633 );
1634 n_curried_arguments += da(function,function_reference)->n_curried_arguments;
1635 if (!da(function,function_reference)->is_indirect)
1636 break;
1637 ptr = &da(function,function_reference)->u.indirect;
1638 }
1639 ptr = da(function,function_reference)->u.direct;
1640 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1641 RELOAD_EX_POSITION(ex_),
1642 function_error = thunk_;
1643 thunk_reference(function_error);
1644 if (deref)
1645 frame_free_and_clear(fp, fn_ref_slot);
1646 goto cat(call_set_error_,ARG_MODE)
1647 );
1649 arguments = mem_alloc_array_mayfail(mem_alloc_mayfail, struct ipret_call_cache_arg *, 0, 0, n_curried_arguments + n_arguments, sizeof(struct ipret_call_cache_arg), &ajla_error);
1650 if (!arguments) {
1651 if (deref)
1652 frame_free_and_clear(fp, fn_ref_slot);
1653 goto cat(call_allocation_error_,ARG_MODE);
1654 }
1655 return_values = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n_return_values, sizeof(frame_t), &ajla_error);
1656 if (!return_values) {
1657 mem_free(arguments);
1658 if (deref)
1659 frame_free_and_clear(fp, fn_ref_slot);
1660 goto cat(call_allocation_error_,ARG_MODE);
1661 }
1663 i = n_curried_arguments;
1664 function = pointer_get_data(*frame_pointer(fp, fn_ref_slot));
1665 while (1) {
1666 arg_t j;
1667 j = da(function,function_reference)->n_curried_arguments;
1668 while (j--) {
1669 i--;
1670 arguments[i].f_arg = &da(function,function_reference)->arguments[j];
1671 arguments[i].deref = false;
1672 }
1673 if (!da(function,function_reference)->is_indirect)
1674 break;
1675 function = pointer_get_data(da(function,function_reference)->u.indirect);
1676 }
1678 for (i = n_curried_arguments; i < n_curried_arguments + n_arguments; i++) {
1679 frame_t src_slot = get_param(ip, 0);
1680 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1681 arguments[i].f_arg = NULL;
1682 arguments[i].slot = src_slot;
1683 arguments[i].deref = (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1684 ADVANCE_IP(param_size(2));
1685 }
1687 for (i = 0; i < n_return_values; i++) {
1688 frame_t slot_r = get_max_param(ip, 0);
1689 return_values[i] = slot_r;
1690 ADVANCE_IP(max_param_size(1) + 1);
1691 }
1693 ex = ipret_call_cache(fp, orig_ip, ptr, arguments, return_values, deref ? fn_ref_slot : NO_FRAME_T);
1694 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
1695 RELOAD_EX_POSITION(ex);
1696 }
1697 #endif
1698 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_CACHE)
1700 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_SAVE)
1701 #ifdef EMIT_CODE
1702 goto cat(call_indirect_cache_,ARG_MODE);
1703 #endif
1704 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_SAVE)
1706 END_BLOCK()
1708 DEFINE_OPCODE_START(OPCODE_RETURN)
1709 #ifdef EMIT_CODE
1710 struct data *function;
1711 frame_s *previous_fp;
1712 const code_t *previous_ip;
1713 arg_t i, n;
1715 /*ADVANCE_I_PARAM(0);*/
1717 function = get_frame(fp)->function;
1718 previous_fp = frame_up(fp);
1719 if (unlikely(frame_is_top(previous_fp))) {
1720 struct execution_control *ex;
1721 void *exx;
1722 struct thunk *t;
1723 struct stack_bottom *sb = frame_stack_bottom(previous_fp);
1724 ex = sb->ex;
1725 if (!ex) {
1726 frame_t src_slot = get_i_param(0);
1727 sb->ret = ipret_copy_variable_to_pointer(fp, src_slot, true);
1728 goto exit_ipret;
1729 }
1730 t = ex->thunk;
1731 n = da(function,function)->n_return_values;
1732 i = 0;
1733 do {
1734 frame_t src_slot = get_i_param(i * 2);
1735 frame_t flags = get_i_param(i * 2 + 1);
1736 t->u.function_call.results[i].ptr = ipret_copy_variable_to_pointer(fp, src_slot, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1737 } while (++i < n);
1738 exx = thunk_terminate(t, n);
1739 #if 0
1740 if (exx != POINTER_FOLLOW_THUNK_EXIT)
1741 task_submit(exx, true);
1742 goto exit_ipret;
1743 #else
1744 RELOAD_EX_POSITION(exx);
1745 #endif
1746 }
1748 get_frame(previous_fp)->timestamp = get_frame(fp)->timestamp;
1749 previous_ip = &da(get_frame(previous_fp)->function,function)->code[get_frame(fp)->previous_ip];
1751 n = da(function,function)->n_return_values;
1752 i = 0;
1753 do {
1754 frame_t src_slot, dst_slot, flags;
1755 src_slot = get_i_param(i * 2);
1756 flags = get_i_param(i * 2 + 1);
1757 dst_slot = (frame_t)get_max_param(previous_ip, 0);
1758 previous_ip += max_param_size(1) + 1;
1759 ipret_copy_variable(fp, src_slot, previous_fp, dst_slot, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1760 } while (++i < n);
1762 fp = previous_fp;
1763 ip = previous_ip;
1764 #endif
1765 DEFINE_OPCODE_END(OPCODE_RETURN)
1768 DEFINE_OPCODE_START(OPCODE_STRUCTURED)
1769 #ifdef EMIT_CODE
1770 const code_t *orig_ip = ip;
1772 frame_t struc, elem;
1773 pointer_t *struc_ptr;
1774 unsigned char *struc_flat;
1775 const struct type *t;
1776 bool optimize_elem_is_flat;
1778 unsigned char type;
1779 frame_t arg;
1781 pointer_t thunk_ptr;
1783 elem = get_i_param(1);
1784 optimize_elem_is_flat = frame_variable_is_flat(fp, elem);
1786 cat(structured_retry_,ARG_MODE):
1787 struc = get_i_param(0);
1788 ADVANCE_I_PARAM(2);
1790 t = frame_get_type_of_local(fp, struc);
1791 if (frame_variable_is_flat(fp, struc)) {
1792 if (!optimize_elem_is_flat) {
1793 frame_set_pointer(fp, struc, flat_to_data(frame_get_type_of_local(fp, struc), frame_var(fp, struc)));
1794 struc_flat = NULL;
1795 struc_ptr = frame_pointer(fp, struc);
1796 } else {
1797 struc_flat = frame_var(fp, struc);
1798 struc_ptr = NULL; /* avoid warning */
1799 }
1800 } else {
1801 struc_flat = NULL;
1802 struc_ptr = frame_pointer(fp, struc);
1803 }
1805 do {
1806 array_index_t array_index = index_invalid(); /* avoid warning */
1807 struct data *data = NULL; /* avoid warning */
1809 type = (unsigned char)get_param(ip, 0);
1810 arg = get_param(ip, 1);
1811 ADVANCE_IP(param_size(3));
1813 if ((type & OPCODE_STRUCTURED_MASK) == OPCODE_STRUCTURED_ARRAY) {
1814 void *ex = ipret_get_index(fp, orig_ip, fp, arg, NULL, &array_index, &thunk_ptr pass_file_line);
1815 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
1816 if (ex == POINTER_FOLLOW_THUNK_EXCEPTION)
1817 goto cat(structured_error_,ARG_MODE);
1818 RELOAD_EX_POSITION(ex);
1819 }
1820 }
1822 if (struc_flat) {
1823 switch (type & OPCODE_STRUCTURED_MASK) {
1824 case OPCODE_STRUCTURED_RECORD: {
1825 const struct flat_record_definition_entry *e;
1826 ajla_assert(arg < flat_record_n_slots(type_def(t,flat_record)), (file_line, "structured flat record: invalid index: %"PRIuMAX" >= %"PRIuMAX"", (uintmax_t)arg, (uintmax_t)flat_record_n_slots(type_def(t,flat_record))));
1827 e = &type_def(t,flat_record)->entries[arg];
1828 t = e->subtype;
1829 struc_flat += e->flat_offset;
1830 break;
1831 }
1832 case OPCODE_STRUCTURED_OPTION: {
1833 thunk_ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_OPTION_DOESNT_MATCH), fp, ip pass_file_line);
1834 goto cat(structured_error_,ARG_MODE);
1835 }
1836 case OPCODE_STRUCTURED_ARRAY: {
1837 if (unlikely(index_ge_int(array_index, type_def(t,flat_array)->n_elements))) {
1838 index_free(&array_index);
1839 thunk_ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_INDEX_OUT_OF_RANGE), fp, ip pass_file_line);
1840 goto cat(structured_error_,ARG_MODE);
1841 }
1842 t = type_def(t,flat_array)->base;
1843 struc_flat += t->size * index_to_int(array_index);
1844 index_free(&array_index);
1845 break;
1846 }
1847 default:
1848 internal(file_line, "structured flat: invalid structure type %02x", type);
1849 }
1850 } else {
1851 cat(structured_restart_pointer_follow_,ARG_MODE):
1852 pointer_follow(struc_ptr, false, data, PF_WAIT, fp, orig_ip,
1853 if ((type & OPCODE_STRUCTURED_MASK) == OPCODE_STRUCTURED_ARRAY)
1854 index_free(&array_index);
1855 RELOAD_EX_POSITION(ex_),
1856 if ((type & OPCODE_STRUCTURED_MASK) == OPCODE_STRUCTURED_ARRAY)
1857 index_free(&array_index);
1858 goto cat(structured_read_to_the_end_,ARG_MODE);
1859 );
1860 if (unlikely(!data_is_writable(data))) {
1861 struct_clone(struc_ptr);
1862 goto cat(structured_restart_pointer_follow_,ARG_MODE);
1863 }
1864 switch (type & OPCODE_STRUCTURED_MASK) {
1865 case OPCODE_STRUCTURED_RECORD: {
1866 const struct record_definition *def = type_def(da(data,record)->definition,record);
1867 ajla_assert(arg < def->n_slots, (file_line, "structured record: invalid index: %"PRIuMAX" >= %"PRIuMAX"", (uintmax_t)arg, (uintmax_t)def->n_slots));
1868 t = def->types[arg];
1869 struc_ptr = frame_pointer(da_record_frame(data), arg);
1871 if (type & OPCODE_STRUCTURED_FLAG_END) {
1872 if (frame_variable_is_flat(fp, elem)) {
1873 if (TYPE_IS_FLAT(t)) {
1874 frame_free(da_record_frame(data), arg);
1875 struc_flat = frame_var(da_record_frame(data), arg);
1876 goto cat(structured_write_flat_,ARG_MODE);
1877 }
1878 } else {
1879 if (!frame_test_and_set_flag(da_record_frame(data), arg))
1880 goto cat(structured_write_ptr_,ARG_MODE);
1881 }
1882 }
1884 if (!frame_test_flag(da_record_frame(data), arg)) {
1885 struc_flat = frame_var(da_record_frame(data), arg);
1886 if (unlikely(!optimize_elem_is_flat)) {
1887 frame_set_pointer(da_record_frame(data), arg, flat_to_data(t, struc_flat));
1888 struc_flat = NULL;
1889 }
1890 }
1891 break;
1892 }
1893 case OPCODE_STRUCTURED_OPTION: {
1894 if (unlikely(da(data,option)->option != arg)) {
1895 if (likely((type & OPCODE_STRUCTURED_FLAG_END) != 0)) {
1896 da(data,option)->option = arg;
1897 } else {
1898 thunk_ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_OPTION_DOESNT_MATCH), fp, ip pass_file_line);
1899 cat(structured_error_,ARG_MODE):
1900 if (struc_flat) {
1901 pointer_dereference(thunk_ptr);
1902 optimize_elem_is_flat = false;
1903 ip = orig_ip;
1904 goto cat(structured_retry_,ARG_MODE);
1905 } else {
1906 pointer_dereference(*struc_ptr);
1907 *struc_ptr = thunk_ptr;
1908 goto cat(structured_read_to_the_end_,ARG_MODE);
1909 }
1910 }
1911 }
1912 struc_ptr = &da(data,option)->pointer;
1913 break;
1914 }
1915 case OPCODE_STRUCTURED_ARRAY: {
1916 unsigned array_flags;
1917 if (unlikely(da_tag(data) == DATA_TAG_array_incomplete)) {
1918 array_index_t len_first;
1920 if (array_incomplete_collapse(struc_ptr))
1921 goto cat(structured_restart_pointer_follow_,ARG_MODE);
1923 len_first = array_len(pointer_get_data(da(data,array_incomplete)->first));
1924 if (!index_ge_index(array_index, len_first)) {
1925 index_free(&len_first);
1926 struc_ptr = &da(data,array_incomplete)->first;
1927 goto cat(structured_restart_pointer_follow_,ARG_MODE);
1928 } else {
1929 index_sub(&array_index, len_first);
1930 index_free(&len_first);
1931 struc_ptr = &da(data,array_incomplete)->next;
1932 goto cat(structured_restart_pointer_follow_,ARG_MODE);
1933 }
1934 }
1935 if (!optimize_elem_is_flat) {
1936 array_flags = ARRAY_MODIFY_NEED_PTR;
1937 } else if (type & OPCODE_STRUCTURED_FLAG_END) {
1938 array_flags = ARRAY_MODIFY_NEED_FLAT;
1939 t = frame_get_type_of_local(fp, elem);
1940 } else {
1941 array_flags = 0;
1942 }
1943 if (unlikely(!array_modify(struc_ptr, array_index, array_flags, &struc_ptr, &struc_flat, &t, fp, ip))) {
1944 goto cat(structured_read_to_the_end_,ARG_MODE);
1945 }
1946 break;
1947 }
1948 default:
1949 internal(file_line, "structured: invalid structure type %02x", type);
1950 }
1951 }
1952 } while (!(type & OPCODE_STRUCTURED_FLAG_END));
1953 if (struc_flat) {
1954 cat(structured_write_flat_,ARG_MODE):
1955 memcpy_fast(struc_flat, frame_var(fp, elem), frame_get_type_of_local(fp, elem)->size);
1956 } else {
1957 /* The pointer may be empty if we moved from an option without data to an option with data */
1958 if (likely(!pointer_is_empty(*struc_ptr)))
1959 pointer_dereference(*struc_ptr);
1960 if (frame_variable_is_flat(fp, elem)) {
1961 *struc_ptr = flat_to_data(frame_get_type_of_local(fp, elem), frame_var(fp, elem));
1962 } else {
1963 cat(structured_write_ptr_,ARG_MODE):
1964 *struc_ptr = frame_get_pointer_reference(fp, elem, (type & OPCODE_STRUCTURED_FREE_VARIABLE) != 0);
1965 }
1966 }
1967 if (false) {
1968 do {
1969 type = (unsigned char)get_param(ip, 0);
1970 arg = get_param(ip, 1);
1971 ADVANCE_IP(param_size(3));
1972 cat(structured_read_to_the_end_,ARG_MODE):;
1973 } while (!(type & OPCODE_STRUCTURED_FLAG_END));
1974 if (type & OPCODE_STRUCTURED_FREE_VARIABLE)
1975 frame_free(fp, elem);
1976 }
1977 #endif
1978 DEFINE_OPCODE_END(OPCODE_STRUCTURED)
1980 DEFINE_OPCODE_START(OPCODE_RECORD_CREATE)
1981 #ifdef EMIT_CODE
1982 frame_t result_slot;
1983 arg_t n_entries, i, ii;
1984 const struct type *t;
1985 const struct record_definition *def;
1987 result_slot = get_i_param(0);
1988 n_entries = get_i_param(1);
1989 ADVANCE_I_PARAM(2);
1991 t = frame_get_type_of_local(fp, result_slot);
1992 if (t->tag == TYPE_TAG_flat_record) {
1993 const code_t *backup_ip;
1994 def = type_def(type_def(t,flat_record)->base,record);
1995 backup_ip = ip;
1996 for (i = 0, ii = 0; i < n_entries; i++, ii++) {
1997 frame_t var_slot, record_slot;
1998 flat_size_t flat_offset;
1999 const struct type *flat_type;
2001 while (unlikely(record_definition_is_elided(def, ii)))
2002 ii++;
2004 var_slot = get_param(ip, 0);
2006 if (unlikely(!frame_variable_is_flat(fp, var_slot))) {
2007 ip = backup_ip;
2008 goto cat(create_record_no_flat_,ARG_MODE);
2009 }
2010 record_slot = record_definition_slot(def, ii);
2011 flat_offset = type_def(t,flat_record)->entries[record_slot].flat_offset;
2012 flat_type = type_def(t,flat_record)->entries[record_slot].subtype;
2013 ajla_assert(type_is_equal(frame_get_type_of_local(fp, var_slot), flat_type), (file_line, "record create (flat): copying between different types (%u,%u,%u) -> (%u,%u,%u)", frame_get_type_of_local(fp, var_slot)->tag, frame_get_type_of_local(fp, var_slot)->size, frame_get_type_of_local(fp, var_slot)->align, flat_type->tag, flat_type->size, flat_type->align));
2014 memcpy_fast(frame_var(fp, result_slot) + flat_offset, frame_var(fp, var_slot), flat_type->size);
2015 ADVANCE_IP(param_size(2));
2016 }
2017 ajla_assert(!frame_test_flag(fp, result_slot), (file_line, "record create (flat): flag already set for destination slot %"PRIuMAX"", (uintmax_t)result_slot));
2018 } else {
2019 struct data *result;
2020 def = type_def(t,record);
2021 cat(create_record_no_flat_,ARG_MODE):
2022 result = data_alloc_record_mayfail(def, &ajla_error pass_file_line);
2023 if (unlikely(!result)) {
2024 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_entries);
2025 frame_set_pointer(fp, result_slot, pointer_error(ajla_error, fp, ip pass_file_line));
2026 break;
2027 }
2028 memset(da_record_frame(result), 0, bitmap_slots(def->n_slots) * slot_size);
2029 for (i = 0, ii = 0; i < n_entries; i++, ii++) {
2030 frame_t var_slot, record_slot;
2031 const struct type *rec_type, *var_type;
2033 while (unlikely(record_definition_is_elided(def, ii)))
2034 ii++;
2036 var_slot = get_param(ip, 0);
2037 record_slot = record_definition_slot(def, ii);
2038 rec_type = def->types[record_slot];
2039 var_type = frame_get_type_of_local(fp, var_slot);
2041 if (!frame_variable_is_flat(fp, var_slot)) {
2042 pointer_t ptr = frame_get_pointer_reference(fp, var_slot, (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2043 frame_set_pointer(da_record_frame(result), record_slot, ptr);
2044 } else if (TYPE_IS_FLAT(rec_type)) {
2045 ajla_assert(type_is_equal(var_type, rec_type), (file_line, "record create: copying between different types (%u,%u,%u) -> (%u,%u,%u)", var_type->tag, var_type->size, var_type->align, rec_type->tag, rec_type->size, rec_type->align));
2046 memcpy_fast(frame_var(da_record_frame(result), record_slot), frame_var(fp, var_slot), rec_type->size);
2047 } else {
2048 pointer_t ptr = flat_to_data(var_type, frame_var(fp, var_slot));
2049 frame_set_pointer(da_record_frame(result), record_slot, ptr);
2050 }
2052 ADVANCE_IP(param_size(2));
2053 }
2054 frame_set_pointer(fp, result_slot, pointer_data(result));
2055 }
2056 #endif
2057 DEFINE_OPCODE_END(OPCODE_RECORD_CREATE)
2059 DEFINE_OPCODE_START(OPCODE_RECORD_LOAD)
2060 #ifdef EMIT_CODE
2061 frame_t record, record_slot, result;
2062 unsigned flags;
2063 const struct type *t;
2064 record = get_i_param(0);
2065 record_slot = get_i_param(1);
2066 result = get_i_param(2);
2067 flags = get_i_param(3);
2069 t = frame_get_type_of_local(fp, record);
2070 if (t->tag == TYPE_TAG_flat_record && !frame_test_flag(fp, record)) {
2071 const struct flat_record_definition_entry *ft = &type_def(t,flat_record)->entries[record_slot];
2072 if (likely(TYPE_IS_FLAT(frame_get_type_of_local(fp, result)))) {
2073 memcpy_fast(frame_var(fp, result), frame_var(fp, record) + ft->flat_offset, ft->subtype->size);
2074 } else {
2075 pointer_t ptr = flat_to_data(ft->subtype, frame_var(fp, record) + ft->flat_offset);
2076 frame_set_pointer(fp, result, ptr);
2077 }
2078 } else {
2079 const struct type *rtype, *etype;
2080 struct data *data;
2081 frame_s *fr;
2082 pointer_follow(frame_pointer(fp, record), true, data, unlikely(flags & OPCODE_OP_FLAG_STRICT) ? PF_WAIT : PF_NOEVAL, fp, ip,
2083 if (!(flags & OPCODE_OP_FLAG_STRICT)) {
2084 ex_ = ipret_record_load_create_thunk(fp, ip, record, record_slot, result);
2085 if (ex_ == POINTER_FOLLOW_THUNK_GO)
2086 goto cat(record_load_end_,ARG_MODE);
2087 }
2088 RELOAD_EX_POSITION(ex_),
2089 thunk_reference(thunk_);
2090 frame_set_pointer(fp, result, pointer_thunk(thunk_));
2091 goto cat(record_load_end_,ARG_MODE);
2092 );
2093 rtype = da(data,record)->definition;
2094 etype = type_def(rtype,record)->types[record_slot];
2095 fr = da_record_frame(data);
2096 if (!frame_test_flag(fr, record_slot)) {
2097 if (likely(TYPE_IS_FLAT(frame_get_type_of_local(fp, result)))) {
2098 memcpy_fast(frame_var(fp, result), frame_var(fr, record_slot), etype->size);
2099 } else {
2100 pointer_t ptr = flat_to_data(etype, frame_var(fr, record_slot));
2101 frame_set_pointer(fp, result, ptr);
2102 }
2103 } else {
2104 pointer_reference_maybe(fp, result, frame_pointer(fr, record_slot), flags);
2105 }
2106 }
2108 cat(record_load_end_,ARG_MODE):
2109 ADVANCE_I_PARAM(4);
2110 #endif
2111 DEFINE_OPCODE_END(OPCODE_RECORD_CREATE)
2113 DEFINE_OPCODE_START(OPCODE_OPTION_CREATE_EMPTY_FLAT)
2114 #ifdef EMIT_CODE
2115 frame_t slot_r = get_i_param(0);
2116 ajla_flat_option_t opt = (ajla_flat_option_t)get_i_param(1);
2117 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "option create empty flat: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
2118 barrier_aliasing();
2119 *frame_slot(fp, slot_r, ajla_flat_option_t) = opt;
2120 barrier_aliasing();
2121 ADVANCE_I_PARAM(2);
2122 #endif
2123 DEFINE_OPCODE_END(OPCODE_OPTION_CREATE_EMPTY_FLAT)
2125 DEFINE_OPCODE_START(OPCODE_OPTION_CREATE_EMPTY)
2126 #ifdef EMIT_CODE
2127 struct data *d;
2128 frame_t slot_r;
2130 d = data_alloc(option, &ajla_error);
2131 slot_r = get_i_param(0);
2132 if (likely(d != NULL)) {
2133 da(d,option)->option = get_i_param(1);
2134 da(d,option)->pointer = pointer_empty();
2135 frame_set_pointer(fp, slot_r, pointer_data(d));
2136 } else {
2137 frame_set_pointer(fp, slot_r, pointer_error(ajla_error, fp, ip pass_file_line));
2138 }
2139 ADVANCE_I_PARAM(2);
2140 #endif
2141 DEFINE_OPCODE_END(OPCODE_OPTION_CREATE_EMPTY)
2143 DEFINE_OPCODE_START(OPCODE_OPTION_CREATE)
2144 #ifdef EMIT_CODE
2145 struct data *d;
2146 frame_t slot_1, slot_r;
2147 pointer_t ptr;
2149 slot_1 = get_i_param(2);
2150 if (!frame_variable_is_flat(fp, slot_1)) {
2151 ptr = frame_get_pointer_reference(fp, slot_1, (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2152 } else {
2153 ptr = flat_to_data(frame_get_type_of_local(fp, slot_1), frame_var(fp, slot_1));
2154 }
2156 d = data_alloc(option, &ajla_error);
2157 slot_r = get_i_param(0);
2158 if (likely(d != NULL)) {
2159 da(d,option)->option = get_i_param(1);
2160 da(d,option)->pointer = ptr;
2161 frame_set_pointer(fp, slot_r, pointer_data(d));
2162 } else {
2163 pointer_dereference(ptr);
2164 frame_set_pointer(fp, slot_r, pointer_error(ajla_error, fp, ip pass_file_line));
2165 }
2166 ADVANCE_I_PARAM(4);
2167 #endif
2168 DEFINE_OPCODE_END(OPCODE_OPTION_CREATE)
2170 DEFINE_OPCODE_START(OPCODE_OPTION_LOAD)
2171 #ifdef EMIT_CODE
2172 frame_t option, option_idx, result;
2173 unsigned flags;
2174 struct data *data;
2175 pointer_t ptr;
2177 option = get_i_param(0);
2178 option_idx = get_i_param(1);
2179 result = get_i_param(2);
2180 flags = get_i_param(3);
2182 if (unlikely(frame_variable_is_flat(fp, option)))
2183 goto cat(option_load_mismatch,ARG_MODE);
2185 pointer_follow(frame_pointer(fp, option), true, data, unlikely(flags & OPCODE_OP_FLAG_STRICT) ? PF_WAIT : PF_NOEVAL, fp, ip,
2186 if (!(flags & OPCODE_OP_FLAG_STRICT)) {
2187 ex_ = ipret_option_load_create_thunk(fp, ip, option, option_idx, result);
2188 if (ex_ == POINTER_FOLLOW_THUNK_GO)
2189 goto cat(option_load_end_,ARG_MODE);
2190 }
2191 RELOAD_EX_POSITION(ex_),
2192 thunk_reference(thunk_);
2193 frame_set_pointer(fp, result, pointer_thunk(thunk_));
2194 goto cat(option_load_end_,ARG_MODE);
2195 );
2197 if (unlikely(da(data,option)->option != option_idx)) {
2198 cat(option_load_mismatch,ARG_MODE):
2199 ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_OPTION_DOESNT_MATCH), fp, ip pass_file_line);
2200 frame_set_pointer(fp, result, ptr);
2201 } else {
2202 pointer_reference_maybe(fp, result, &da(data,option)->pointer, flags);
2203 }
2205 cat(option_load_end_,ARG_MODE):
2206 ADVANCE_I_PARAM(4);
2207 #endif
2208 DEFINE_OPCODE_END(OPCODE_OPTION_LOAD)
2211 START_BLOCK(
2212 frame_t slot_1;
2213 ajla_option_t option;
2214 )
2216 DEFINE_OPCODE_START(OPCODE_OPTION_TEST_FLAT)
2217 #ifdef EMIT_CODE
2218 slot_1 = get_i_param(0);
2220 if (!frame_test_flag(fp, slot_1)) {
2221 barrier_aliasing();
2222 option = *frame_slot(fp, slot_1, ajla_flat_option_t);
2223 barrier_aliasing();
2224 goto cat(option_test_store_result_,ARG_MODE);
2225 }
2226 goto cat(option_test_,ARG_MODE);
2227 #endif
2228 DEFINE_OPCODE_END(OPCODE_OPTION_TEST_FLAT)
2230 DEFINE_OPCODE_START(OPCODE_OPTION_TEST)
2231 #ifdef EMIT_CODE
2232 frame_t slot_r;
2233 pointer_t ptr;
2234 struct data *data;
2236 slot_1 = get_i_param(0);
2237 cat(option_test_,ARG_MODE):
2238 ptr = *frame_pointer(fp, slot_1);
2239 if (unlikely(pointer_is_thunk(ptr))) {
2240 void *ex;
2241 ajla_assert(!frame_test_flag(fp, get_i_param(2)), (file_line, "option test: flag already set for destination slot %"PRIuMAX"", (uintmax_t)get_i_param(2)));
2242 ex = thunk_option_test(fp, ip, slot_1, get_i_param(1), get_i_param(2));
2243 if (ex == POINTER_FOLLOW_THUNK_GO) {
2244 ADVANCE_I_PARAM(3);
2245 break;
2246 } else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY)) {
2247 RELOAD_EX_POSITION(ex);
2248 }
2249 break;
2250 }
2251 data = pointer_get_data(ptr);
2252 option = da(data,option)->option;
2253 cat(option_test_store_result_,ARG_MODE):
2254 slot_r = get_i_param(2);
2255 barrier_aliasing();
2256 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "option test: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
2257 *frame_slot(fp, slot_r, ajla_flat_option_t) = option == (ajla_option_t)get_i_param(1);
2258 barrier_aliasing();
2259 ADVANCE_I_PARAM(3);
2260 #endif
2261 DEFINE_OPCODE_END(OPCODE_OPTION_TEST)
2263 DEFINE_OPCODE_START(OPCODE_OPTION_ORD_FLAT)
2264 #ifdef EMIT_CODE
2265 slot_1 = get_i_param(0);
2267 if (!frame_test_flag(fp, slot_1)) {
2268 barrier_aliasing();
2269 option = *frame_slot(fp, slot_1, ajla_flat_option_t);
2270 barrier_aliasing();
2271 goto cat(option_ord_store_result_,ARG_MODE);
2272 }
2273 goto cat(option_ord_,ARG_MODE);
2274 #endif
2275 DEFINE_OPCODE_END(OPCODE_OPTION_ORD_FLAT)
2277 DEFINE_OPCODE_START(OPCODE_OPTION_ORD)
2278 #ifdef EMIT_CODE
2279 frame_t slot_r;
2280 pointer_t ptr;
2281 struct data *data;
2283 slot_1 = get_i_param(0);
2284 cat(option_ord_,ARG_MODE):
2285 ptr = *frame_pointer(fp, slot_1);
2286 if (unlikely(pointer_is_thunk(ptr))) {
2287 void *ex;
2288 ajla_assert(!frame_test_flag(fp, get_i_param(1)), (file_line, "option ord: flag already set for destination slot %"PRIuMAX"", (uintmax_t)get_i_param(2)));
2289 ex = thunk_option_ord(fp, ip, slot_1, get_i_param(1));
2290 if (ex == POINTER_FOLLOW_THUNK_GO) {
2291 ADVANCE_I_PARAM(2);
2292 break;
2293 } else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY)) {
2294 RELOAD_EX_POSITION(ex);
2295 }
2296 break;
2297 }
2298 data = pointer_get_data(ptr);
2299 option = da(data,option)->option;
2300 cat(option_ord_store_result_,ARG_MODE):
2301 slot_r = get_i_param(1);
2302 barrier_aliasing();
2303 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "option ord: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
2304 *frame_slot(fp, slot_r, int_default_t) = option;
2305 barrier_aliasing();
2306 ADVANCE_I_PARAM(2);
2307 #endif
2308 DEFINE_OPCODE_END(OPCODE_OPTION_ORD)
2310 END_BLOCK()
2313 DEFINE_OPCODE_START(OPCODE_ARRAY_CREATE)
2314 #ifdef EMIT_CODE
2315 frame_t result_slot;
2316 size_t n_entries;
2317 struct data *total;
2319 result_slot = get_i_param(0);
2320 n_entries = get_i_param(1);
2321 ADVANCE_I_PARAM(2);
2323 ajla_assert(n_entries != 0, (file_line, "array create: zero entries"));
2325 total = NULL;
2326 while (1) {
2327 size_t n, i;
2328 struct data *a;
2329 const struct type *type = frame_get_type_of_local(fp, get_param(ip, 0));
2330 bool flat = frame_variable_is_flat(fp, get_param(ip, 0));
2331 for (n = 1; n < n_entries; n++) {
2332 if (frame_variable_is_flat(fp, get_param(ip, n * 2)) != flat)
2333 break;
2334 if (unlikely((int_default_t)(n + 1) < zero))
2335 break;
2336 }
2337 do {
2338 if (flat)
2339 a = data_alloc_array_flat_mayfail(type, n, n, false, &ajla_error pass_file_line);
2340 else
2341 a = data_alloc_array_pointers_mayfail(n, n, &ajla_error pass_file_line);
2342 } while (unlikely(!a) && (n >>= 1));
2343 if (unlikely(!a)) {
2344 if (total)
2345 data_dereference(total);
2346 goto cat(array_create_error_,ARG_MODE);
2347 }
2348 n_entries -= n;
2349 if (flat) {
2350 unsigned char *flat_ptr = da_array_flat(a);
2351 for (i = 0; i < n; i++) {
2352 frame_t var_slot = get_param(ip, 0);
2353 memcpy_fast(flat_ptr, frame_var(fp, var_slot), type->size);
2354 flat_ptr += type->size;
2355 ADVANCE_IP(param_size(2));
2356 }
2357 } else {
2358 for (i = 0; i < n; i++) {
2359 frame_t var_slot = get_param(ip, 0);
2360 pointer_t ptr = frame_get_pointer_reference(fp, var_slot, (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2361 da(a,array_pointers)->pointer[i] = ptr;
2362 ADVANCE_IP(param_size(2));
2363 }
2364 }
2365 if (likely(!total)) {
2366 total = a;
2367 } else {
2368 total = array_join(total, a, &ajla_error);
2369 if (unlikely(!total)) {
2370 cat(array_create_error_,ARG_MODE):
2371 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_entries);
2372 frame_set_pointer(fp, result_slot, pointer_error(ajla_error, fp, ip pass_file_line));
2373 break;
2374 }
2375 }
2376 if (likely(!n_entries)) {
2377 frame_set_pointer(fp, result_slot, pointer_data(total));
2378 break;
2379 }
2380 }
2381 #endif
2382 DEFINE_OPCODE_END(OPCODE_ARRAY_CREATE)
2384 DEFINE_OPCODE_START(OPCODE_ARRAY_CREATE_EMPTY_FLAT)
2385 #ifdef EMIT_CODE
2386 frame_t result_slot;
2387 struct data *a;
2388 const struct type *type = da_type(get_frame(fp)->function, get_i_param(1));
2389 TYPE_TAG_VALIDATE(type->tag);
2391 a = data_alloc_array_flat_mayfail(type, 0, 0, false, &ajla_error pass_file_line);
2392 result_slot = get_i_param(0);
2393 frame_set_pointer(fp, result_slot, likely(a != NULL) ? pointer_data(a) : pointer_error(ajla_error, fp, ip pass_file_line));
2394 ADVANCE_I_PARAM(2);
2395 #endif
2396 DEFINE_OPCODE_END(OPCODE_ARRAY_CREATE_EMPTY_FLAT)
2398 DEFINE_OPCODE_START(OPCODE_ARRAY_CREATE_EMPTY)
2399 #ifdef EMIT_CODE
2400 frame_t result_slot;
2401 struct data *a;
2403 a = data_alloc_array_pointers_mayfail(0, 0, &ajla_error pass_file_line);
2404 result_slot = get_i_param(0);
2405 frame_set_pointer(fp, result_slot, likely(a != NULL) ? pointer_data(a) : pointer_error(ajla_error, fp, ip pass_file_line));
2406 ADVANCE_I_PARAM(1);
2407 #endif
2408 DEFINE_OPCODE_END(OPCODE_ARRAY_CREATE_EMPTY)
2410 DEFINE_OPCODE_START(OPCODE_ARRAY_FILL)
2411 #ifdef EMIT_CODE
2412 frame_t content_slot, length_slot, result_slot;
2413 unsigned char content_flag;
2414 array_index_t length;
2415 pointer_t result_ptr;
2416 void *ex;
2418 content_slot = get_i_param(0);
2419 content_flag = get_i_param(1);
2420 length_slot = get_i_param(2);
2421 result_slot = get_i_param(3);
2423 ex = ipret_get_index(fp, ip, fp, length_slot, NULL, &length, &result_ptr pass_file_line);
2424 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
2425 if (ex == POINTER_FOLLOW_THUNK_EXCEPTION) {
2426 if (content_flag & OPCODE_FLAG_FREE_ARGUMENT)
2427 frame_free_and_clear(fp, content_slot);
2428 goto cat(array_fill_error_,ARG_MODE);
2429 }
2430 RELOAD_EX_POSITION(ex);
2431 }
2433 if (!frame_variable_is_flat(fp, content_slot)) {
2434 pointer_t ptr = frame_get_pointer_reference(fp, content_slot, (content_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2435 if (unlikely((content_flag & OPCODE_ARRAY_FILL_FLAG_SPARSE) != 0)) {
2436 result_ptr = array_create_sparse(length, ptr);
2437 } else {
2438 result_ptr = array_create(length, NULL, NULL, ptr);
2439 }
2440 } else {
2441 const struct type *type = frame_get_type_of_local(fp, content_slot);
2442 barrier_aliasing();
2443 if (unlikely((content_flag & OPCODE_ARRAY_FILL_FLAG_SPARSE) != 0) && likely(index_ge_int(length, 1))) {
2444 pointer_t ptr = flat_to_data(type, frame_var(fp, content_slot));
2445 result_ptr = array_create_sparse(length, ptr);
2446 } else {
2447 if (TYPE_IS_FLAT(frame_get_type_of_local(fp, result_slot))) {
2448 unsigned char *result = frame_var(fp, result_slot);
2449 int_default_t l = index_to_int(length);
2450 index_free(&length);
2451 while (l--) {
2452 result = mempcpy(result, frame_var(fp, content_slot), type->size);
2453 }
2454 goto cat(array_exit_,ARG_MODE);
2455 } else {
2456 result_ptr = array_create(length, type, frame_var(fp, content_slot), pointer_empty());
2457 }
2458 }
2459 barrier_aliasing();
2460 }
2462 cat(array_fill_error_,ARG_MODE):
2463 frame_set_pointer(fp, result_slot, result_ptr);
2465 cat(array_exit_,ARG_MODE):
2466 ADVANCE_I_PARAM(4);
2467 #endif
2468 DEFINE_OPCODE_END(OPCODE_ARRAY_FILL)
2470 DEFINE_OPCODE_START(OPCODE_ARRAY_STRING)
2471 #ifdef EMIT_CODE
2472 frame_t result_slot, length;
2473 pointer_t result_ptr;
2475 result_slot = get_i_param(0);
2476 length = get_i_param(1);
2477 ADVANCE_I_PARAM(2);
2479 if (unlikely((int_default_t)length < (int_default_t)zero) ||
2480 unlikely((frame_t)(int_default_t)length != length)) {
2481 result_ptr = pointer_error(error_ajla(EC_ASYNC, AJLA_ERROR_SIZE_OVERFLOW), fp, ip pass_file_line);
2482 } else {
2483 result_ptr = array_string(length, type_get_fixed(0, true), cast_ptr(unsigned char *, ip));
2484 }
2485 frame_set_pointer(fp, result_slot, result_ptr);
2487 ADVANCE_IP((length + 1) >> 1);
2488 #endif
2489 DEFINE_OPCODE_END(OPCODE_ARRAY_STRING)
2491 DEFINE_OPCODE_START(OPCODE_ARRAY_UNICODE)
2492 #ifdef EMIT_CODE
2493 frame_t result_slot, length;
2494 pointer_t result_ptr;
2496 result_slot = get_i_param(0);
2497 length = get_i_param(1);
2498 ADVANCE_I_PARAM(2);
2500 if (unlikely((int_default_t)length < (int_default_t)zero) ||
2501 unlikely((frame_t)(int_default_t)length != length)) {
2502 result_ptr = pointer_error(error_ajla(EC_ASYNC, AJLA_ERROR_SIZE_OVERFLOW), fp, ip pass_file_line);
2503 } else {
2504 result_ptr = array_string(length, type_get_int(2), cast_ptr(unsigned char *, ip));
2505 }
2506 frame_set_pointer(fp, result_slot, result_ptr);
2508 ADVANCE_IP(length * 2);
2509 #endif
2510 DEFINE_OPCODE_END(OPCODE_ARRAY_STRING)
2512 DEFINE_OPCODE_START(OPCODE_ARRAY_LOAD)
2513 #ifdef EMIT_CODE
2514 frame_t array, idx, result;
2515 unsigned flags;
2516 const struct type *t, *t_elem;
2517 void *ex;
2518 array_index_t array_index;
2519 pointer_t ptr;
2520 unsigned char *flat;
2522 array = get_i_param(0);
2523 idx = get_i_param(1);
2524 result = get_i_param(2);
2525 flags = get_i_param(3);
2527 ex = ipret_get_index(fp, ip, fp, idx, NULL, &array_index, &ptr pass_file_line);
2528 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
2529 if (ex == POINTER_FOLLOW_THUNK_EXCEPTION)
2530 goto cat(array_load_set_ptr_,ARG_MODE);
2531 RELOAD_EX_POSITION(ex);
2532 }
2534 t = frame_get_type_of_local(fp, array);
2535 if (t->tag == TYPE_TAG_flat_array && !frame_test_flag(fp, array)) {
2536 const struct flat_array_definition *def = type_def(t,flat_array);
2537 if (unlikely(index_ge_int(array_index, def->n_elements))) {
2538 if (unlikely((flags & OPCODE_ARRAY_INDEX_IN_RANGE) != 0))
2539 internal(file_line, "array_load: flat array index out of range");
2540 index_free(&array_index);
2541 ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_INDEX_OUT_OF_RANGE), fp, ip pass_file_line);
2542 goto cat(array_load_set_ptr_,ARG_MODE);
2543 }
2544 t_elem = def->base;
2545 flat = frame_var(fp, array) + t_elem->size * index_to_int(array_index);
2546 index_free(&array_index);
2547 goto cat(array_load_set_flat_,ARG_MODE);
2548 } else {
2549 pointer_t *array_ptr = frame_pointer(fp, array);
2550 pointer_t *pptr;
2551 struct data *data;
2553 cat(array_restart_pointer_follow_,ARG_MODE):
2554 pointer_follow(array_ptr, false, data, unlikely(flags & OPCODE_OP_FLAG_STRICT) ? PF_WAIT : PF_NOEVAL, fp, ip,
2555 index_free(&array_index);
2556 if (!(flags & OPCODE_OP_FLAG_STRICT)) {
2557 ex_ = ipret_array_load_create_thunk(fp, ip, array, idx, result);
2558 if (ex_ == POINTER_FOLLOW_THUNK_GO)
2559 goto cat(array_load_end_,ARG_MODE);
2560 }
2561 RELOAD_EX_POSITION(ex_),
2562 index_free(&array_index);
2563 thunk_reference(thunk_);
2564 ptr = pointer_thunk(thunk_);
2565 goto cat(array_load_set_ptr_,ARG_MODE);
2566 );
2568 if (unlikely(da_tag(data) == DATA_TAG_array_incomplete)) {
2569 array_index_t len_first;
2571 if (array_ptr == frame_pointer(fp, array)) {
2572 if (!frame_test_and_set_flag(fp, array))
2573 data_reference(data);
2574 if (array_incomplete_collapse(array_ptr))
2575 goto cat(array_restart_pointer_follow_,ARG_MODE);
2576 }
2578 len_first = array_len(pointer_get_data(da(data,array_incomplete)->first));
2579 if (!index_ge_index(array_index, len_first)) {
2580 index_free(&len_first);
2581 data = pointer_get_data(da(data,array_incomplete)->first);
2582 } else {
2583 index_sub(&array_index, len_first);
2584 index_free(&len_first);
2585 array_ptr = &da(data,array_incomplete)->next;
2586 goto cat(array_restart_pointer_follow_,ARG_MODE);
2587 }
2588 }
2590 if (unlikely(!array_read(data, array_index, &pptr, &flat, &t_elem, NULL))) {
2591 if (unlikely((flags & OPCODE_ARRAY_INDEX_IN_RANGE) != 0))
2592 internal(file_line, "array_load: array index out of range");
2593 ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_INDEX_OUT_OF_RANGE), fp, ip pass_file_line);
2594 goto cat(array_load_set_ptr_,ARG_MODE);
2595 }
2596 if (pptr) {
2597 pointer_reference_maybe(fp, result, pptr, flags);
2598 goto cat(array_load_end_,ARG_MODE);
2599 } else {
2600 cat(array_load_set_flat_,ARG_MODE):
2601 if (likely(TYPE_IS_FLAT(frame_get_type_of_local(fp, result)))) {
2602 memcpy_fast(frame_var(fp, result), flat, t_elem->size);
2603 goto cat(array_load_end_,ARG_MODE);
2604 } else {
2605 ptr = flat_to_data(t_elem, flat);
2606 }
2607 }
2608 }
2610 cat(array_load_set_ptr_,ARG_MODE):
2611 frame_set_pointer(fp, result, ptr);
2613 cat(array_load_end_,ARG_MODE):
2614 ADVANCE_I_PARAM(4);
2615 #endif
2616 DEFINE_OPCODE_END(OPCODE_ARRAY_LOAD)
2618 DEFINE_OPCODE_START(OPCODE_ARRAY_LEN)
2619 #ifdef EMIT_CODE
2620 frame_t slot_a, slot_r;
2621 unsigned flags;
2622 void *ex;
2624 slot_a = get_i_param(0);
2625 slot_r = get_i_param(1);
2626 flags = get_i_param(2);
2628 ex = ipret_array_len(fp, ip, slot_r, slot_a, flags);
2629 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2630 RELOAD_EX_POSITION(ex);
2632 ADVANCE_I_PARAM(3);
2633 #endif
2634 DEFINE_OPCODE_END(OPCODE_ARRAY_LEN)
2636 DEFINE_OPCODE_START(OPCODE_ARRAY_LEN_GREATER_THAN)
2637 #ifdef EMIT_CODE
2638 frame_t slot_a, slot_l, slot_r;
2639 unsigned flags;
2640 void *ex;
2642 slot_a = get_i_param(0);
2643 slot_l = get_i_param(1);
2644 slot_r = get_i_param(2);
2645 flags = get_i_param(3);
2647 ex = ipret_array_len_greater_than(fp, ip, slot_r, slot_a, slot_l, flags);
2648 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2649 RELOAD_EX_POSITION(ex);
2651 ADVANCE_I_PARAM(4);
2652 #endif
2653 DEFINE_OPCODE_END(OPCODE_ARRAY_LEN_GREATER_THAN)
2655 DEFINE_OPCODE_START(OPCODE_ARRAY_SUB)
2656 #ifdef EMIT_CODE
2657 frame_t slot_a, slot_start, slot_end, slot_r;
2658 unsigned flags;
2659 void *ex;
2661 slot_a = get_i_param(0);
2662 slot_start = get_i_param(1);
2663 slot_end = get_i_param(2);
2664 slot_r = get_i_param(3);
2665 flags = get_i_param(4);
2667 ex = ipret_array_sub(fp, ip, slot_r, slot_a, slot_start, slot_end, flags);
2668 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2669 RELOAD_EX_POSITION(ex);
2671 ADVANCE_I_PARAM(5);
2672 #endif
2673 DEFINE_OPCODE_END(OPCODE_ARRAY_SUB)
2675 DEFINE_OPCODE_START(OPCODE_ARRAY_SKIP)
2676 #ifdef EMIT_CODE
2677 frame_t slot_a, slot_start, slot_r;
2678 unsigned flags;
2679 void *ex;
2681 slot_a = get_i_param(0);
2682 slot_start = get_i_param(1);
2683 slot_r = get_i_param(2);
2684 flags = get_i_param(3);
2686 ex = ipret_array_skip(fp, ip, slot_r, slot_a, slot_start, flags);
2687 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2688 RELOAD_EX_POSITION(ex);
2690 ADVANCE_I_PARAM(4);
2691 #endif
2692 DEFINE_OPCODE_END(OPCODE_ARRAY_SKIP)
2694 DEFINE_OPCODE_START(OPCODE_ARRAY_APPEND)
2695 #ifdef EMIT_CODE
2696 frame_t slot_r, slot_1, slot_2;
2697 unsigned flags;
2698 void *ex;
2700 slot_r = get_i_param(0);
2701 flags = get_i_param(1);
2702 slot_1 = get_i_param(2);
2703 slot_2 = get_i_param(3);
2705 ex = ipret_array_append(fp, ip, slot_r, slot_1, slot_2, flags);
2706 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2707 RELOAD_EX_POSITION(ex);
2709 ADVANCE_I_PARAM(4);
2710 #endif
2711 DEFINE_OPCODE_END(OPCODE_ARRAY_APPEND)
2713 DEFINE_OPCODE_START(OPCODE_ARRAY_APPEND_ONE_FLAT)
2714 #ifdef EMIT_CODE
2715 frame_t slot_r, slot_1, slot_2;
2716 unsigned flags;
2717 void *ex;
2719 slot_r = get_i_param(0);
2720 flags = get_i_param(1);
2721 slot_1 = get_i_param(2);
2722 slot_2 = get_i_param(3);
2724 ex = ipret_array_append_one_flat(fp, ip, slot_r, slot_1, slot_2, flags);
2725 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2726 RELOAD_EX_POSITION(ex);
2728 ADVANCE_I_PARAM(4);
2729 #endif
2730 DEFINE_OPCODE_END(OPCODE_ARRAY_APPEND_ONE_FLAT)
2732 DEFINE_OPCODE_START(OPCODE_ARRAY_APPEND_ONE)
2733 #ifdef EMIT_CODE
2734 frame_t slot_r, slot_1, slot_2;
2735 unsigned flags;
2736 void *ex;
2738 slot_r = get_i_param(0);
2739 flags = get_i_param(1);
2740 slot_1 = get_i_param(2);
2741 slot_2 = get_i_param(3);
2743 ex = ipret_array_append_one(fp, ip, slot_r, slot_1, slot_2, flags);
2744 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2745 RELOAD_EX_POSITION(ex);
2747 ADVANCE_I_PARAM(4);
2748 #endif
2749 DEFINE_OPCODE_END(OPCODE_ARRAY_APPEND_ONE)
2751 DEFINE_OPCODE_START(OPCODE_ARRAY_FLATTEN)
2752 #ifdef EMIT_CODE
2753 frame_t slot_r, slot_1;
2754 unsigned flags;
2755 void *ex;
2757 slot_r = get_i_param(0);
2758 flags = get_i_param(1);
2759 slot_1 = get_i_param(2);
2761 ex = ipret_array_flatten(fp, ip, slot_r, slot_1, flags);
2762 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2763 RELOAD_EX_POSITION(ex);
2765 ADVANCE_I_PARAM(3);
2766 #endif
2767 DEFINE_OPCODE_END(OPCODE_ARRAY_FLATTEN)
2770 #if ARG_MODE == 0
2771 DEFINE_OPCODE_START(OPCODE_IO)
2772 #ifdef EMIT_CODE
2773 unsigned char io_code, n_outputs, n_inputs, n_params;
2774 void *ex;
2776 io_code = get_i_param(0);
2777 n_outputs = get_i_param(1);
2778 n_inputs = get_i_param(2);
2779 n_params = get_i_param(3);
2781 ex = ipret_io(fp, ip, io_code, n_outputs, n_inputs, n_params);
2782 if (ex != POINTER_FOLLOW_THUNK_GO) {
2783 RELOAD_EX_POSITION(ex);
2784 } else {
2785 ADVANCE_IP(3 + 2 * (n_outputs + n_inputs + n_params));
2786 }
2787 #endif
2788 DEFINE_OPCODE_END(OPCODE_IO)
2789 #endif
2791 #if ARG_MODE == 0
2792 DEFINE_OPCODE_START(OPCODE_INTERNAL_FUNCTION)
2793 #ifdef EMIT_CODE
2794 void *ex = function_call_internal(fp, ip);
2795 if (ex != POINTER_FOLLOW_THUNK_RETRY) {
2796 RELOAD_EX_POSITION(ex);
2797 }
2798 #endif
2799 DEFINE_OPCODE_END(OPCODE_INTERNAL_FUNCTION)
2800 #endif
2802 #if ARG_MODE == 0
2803 DEFINE_OPCODE_START(OPCODE_UNREACHABLE)
2804 #ifdef EMIT_CODE
2805 internal(file_line, "unreachable code");
2806 #endif
2807 DEFINE_OPCODE_END(OPCODE_UNREACHABLE)
2808 #endif
2810 #if ARG_MODE == 0
2811 DEFINE_OPCODE_START(OPCODE_EXIT_THREAD)
2812 #ifdef EMIT_CODE
2813 struct execution_control *ex = frame_execution_control(fp);
2814 pointer_t *var_ptr = frame_pointer(fp, get_i_param(0));
2815 struct data attr_unused *data;
2817 pointer_follow(var_ptr, true, data, PF_WAIT, fp, ip,
2818 RELOAD_EX_POSITION(ex_),
2819 ajla_assert_lo(thunk_tag(thunk_) == THUNK_TAG_EXCEPTION, (file_line, "exit_thread: invalid thunk tag %u", thunk_tag(thunk_)));
2820 goto exit_ipret_thunk;
2821 );
2823 exit_ipret_thunk:
2824 ajla_assert_lo(ex->thunk == NULL, (file_line, "exit_thread: non-NULL thunk %p", ex->thunk));
2825 ex->current_frame = fp;
2826 ex->current_ip = frame_ip(fp, ip);
2827 execution_control_terminate(ex, *var_ptr);
2828 goto exit_ipret;
2829 #endif
2830 DEFINE_OPCODE_END(OPCODE_EXIT_THREAD)
2831 #endif
2834 #undef move_scalar
2835 #undef copy_scalar
2837 #undef get_param
2838 #undef param_size
2839 #undef get_i_param
2840 #undef ADVANCE_I_PARAM
2842 #undef ARG_MODE
2843 #undef DEFINE_OPCODE_START