| blob | c0c9425ea8753aa7ca5ab1398e421f81fc95306c |
1 /*
2 * Copyright (C) 2024 Mikulas Patocka
3 *
4 * This file is part of Ajla.
5 *
6 * Ajla is free software: you can redistribute it and/or modify it under the
7 * terms of the GNU General Public License as published by the Free Software
8 * Foundation, either version 3 of the License, or (at your option) any later
9 * version.
10 *
11 * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * Ajla. If not, see <https://www.gnu.org/licenses/>.
17 */
19 #if ARG_MODE == 0
20 #define get_param(ip, n) (!((n) & 1) ? get_lo((ip) + (n) / 2) : get_hi((ip) + (n) / 2))
21 #define param_size(n) (((n) + 1) / 2)
22 #elif ARG_MODE == 1
23 /* A cast to uintptr_t because of gcc optimizer inefficiency */
24 #define get_param(ip, n) ((uintptr_t)(ip)[n])
25 #define param_size(n) (n)
26 #elif ARG_MODE == 2
27 #define get_param(ip, n) get_unaligned_32(&(ip)[(n) * 2])
28 #define param_size(n) ((n) * 2)
29 #endif
31 #define get_i_param(n) get_param(ip + 1, n)
32 #define ADVANCE_I_PARAM(n) ADVANCE_IP(1 + param_size(n))
35 #ifdef EMIT_FUNCTIONS
36 static const code_t attr_cold *cat(free_parameters_,ARG_MODE)(frame_s *fp, const code_t *ip, frame_t n_entries)
37 {
38 while (n_entries--) {
39 if (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT)
40 frame_free_and_clear(fp, get_param(ip, 0));
41 ADVANCE_IP(param_size(2));
42 }
43 return ip;
44 }
45 #endif
48 #define move_scalar(utype) \
49 do { \
50 frame_t slot_1 = get_i_param(0); \
51 frame_t slot_r = get_i_param(1); \
52 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "scalar move: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
53 barrier_aliasing(); \
54 if (unlikely(frame_test_and_clear_flag(fp, slot_1))) { \
55 frame_set_pointer(fp, slot_r, *frame_pointer(fp, slot_1));\
56 pointer_poison(frame_pointer(fp, slot_1)); \
57 } else { \
58 *frame_slot(fp, slot_r, utype) = \
59 *frame_slot(fp, slot_1, utype); \
60 } \
61 barrier_aliasing(); \
62 ADVANCE_I_PARAM(2); \
63 } while (0)
65 #define copy_scalar(utype) \
66 do { \
67 frame_t slot_1 = get_i_param(0); \
68 frame_t slot_r = get_i_param(1); \
69 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "scalar copy: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
70 barrier_aliasing(); \
71 if (unlikely(frame_test_flag(fp, slot_1))) { \
72 pointer_t ptr = *frame_pointer(fp, slot_1); \
73 frame_set_pointer(fp, slot_r, ptr); \
74 pointer_reference_owned(ptr); \
75 } else { \
76 *frame_slot(fp, slot_r, utype) = \
77 *frame_slot(fp, slot_1, utype); \
78 } \
79 barrier_aliasing(); \
80 ADVANCE_I_PARAM(2); \
81 } while (0)
84 #define DEFINE_FIXED_REAL_BINARY_OPCODE(tc, type, op) \
85 DEFINE_OPCODE( \
86 cat3(OPCODE_,tc,OP) + \
87 cat4(OPCODE_,tc,OP_,op) * cat3(OPCODE_,tc,OP_MULT) + \
88 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
89 cat7(tc,binary_,op,_,type,_,ARG_MODE), \
90 frame_t slot_1 = get_i_param(0); \
91 frame_t slot_2 = get_i_param(1); \
92 frame_t slot_r = get_i_param(2); \
93 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "fixed binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
94 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
95 goto cat(escape_fixed_binary_thunk_,ARG_MODE); \
96 } else { \
97 const void *val1; \
98 const void *val2; \
99 void *result; \
100 val1 = frame_var(fp, slot_1); \
101 val2 = frame_var(fp, slot_2); \
102 result = frame_var(fp, slot_r); \
103 barrier_aliasing(); \
104 if (unlikely(!cat5(tc,binary_,op,_,type)(val1, val2, result))) {\
105 barrier_aliasing(); \
106 goto cat(escape_fixed_binary_thunk_,ARG_MODE); \
107 } \
108 barrier_aliasing(); \
109 ADVANCE_I_PARAM(4); \
110 } \
111 )
113 #define DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(tc, type, op) \
114 DEFINE_OPCODE( \
115 cat3(OPCODE_,tc,OP) + \
116 cat4(OPCODE_,tc,OP_C_,op) * cat3(OPCODE_,tc,OP_MULT) + \
117 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
118 cat7(tc,binary_const_,op,_,type,_,ARG_MODE), \
119 frame_t slot_1 = get_i_param(0); \
120 frame_t slot_2 = get_i_param(1); \
121 frame_t slot_r = get_i_param(2); \
122 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "fixed binary const: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
123 if (unlikely(frame_test_flag(fp, slot_1))) { \
124 goto cat(escape_fixed_binary_const_thunk_,ARG_MODE); \
125 } else { \
126 const void *val1; \
127 type val2; \
128 void *result; \
129 val1 = frame_var(fp, slot_1); \
130 val2 = (int32_t)slot_2; \
131 result = frame_var(fp, slot_r); \
132 barrier_aliasing(); \
133 if (unlikely(!cat5(tc,binary_,op,_,type)(val1, cast_ptr(void *, &val2), result))) {\
134 barrier_aliasing(); \
135 goto cat(escape_fixed_binary_const_thunk_,ARG_MODE);\
136 } \
137 barrier_aliasing(); \
138 ADVANCE_I_PARAM(4); \
139 } \
140 )
142 #define DEFINE_FIXED_REAL_UNARY_OPCODE(tc, type, op) \
143 DEFINE_OPCODE( \
144 cat3(OPCODE_,tc,OP) + \
145 cat4(OPCODE_,tc,OP_,op) * cat3(OPCODE_,tc,OP_MULT) + \
146 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
147 cat7(tc,unary_,op,_,type,_,ARG_MODE), \
148 frame_t slot_1 = get_i_param(0); \
149 frame_t slot_r = get_i_param(1); \
150 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "fixed unary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
151 if (unlikely(frame_test_flag(fp, slot_1))) { \
152 goto cat(escape_fixed_unary_thunk_,ARG_MODE); \
153 } else { \
154 const void *val1; \
155 void *result; \
156 val1 = frame_var(fp, slot_1); \
157 result = frame_var(fp, slot_r); \
158 barrier_aliasing(); \
159 cat5(tc,unary_,op,_,type)(val1, result); \
160 barrier_aliasing(); \
161 ADVANCE_I_PARAM(3); \
162 } \
163 )
165 #define DEFINE_FIXED_TO_INT(type, utype) \
166 DEFINE_OPCODE( \
167 OPCODE_FIXED_OP + \
168 OPCODE_FIXED_OP_to_int * OPCODE_FIXED_OP_MULT + \
169 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
170 cat4(FIXED_unary_to_int_,type,_,ARG_MODE), \
171 frame_t slot_1 = get_i_param(0); \
172 frame_t slot_r = get_i_param(1); \
173 if (unlikely(frame_test_flag(fp, slot_1))) { \
174 goto cat(escape_convert_thunk_,ARG_MODE); \
175 } else { \
176 type val1; \
177 int_default_t r; \
178 barrier_aliasing(); \
179 val1 = *frame_slot(fp, slot_1, type); \
180 r = (int_default_t)val1; \
181 if (unlikely(r != val1)) \
182 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, false));\
183 else \
184 *frame_slot(fp, slot_r, int_default_t) = r; \
185 barrier_aliasing(); \
186 ADVANCE_I_PARAM(3); \
187 } \
188 )
190 #define DEFINE_FIXED_UTO_INT(type, utype) \
191 DEFINE_OPCODE( \
192 OPCODE_FIXED_OP + \
193 OPCODE_FIXED_OP_uto_int * OPCODE_FIXED_OP_MULT + \
194 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
195 cat4(FIXED_unary_uto_int_,type,_,ARG_MODE), \
196 frame_t slot_1 = get_i_param(0); \
197 frame_t slot_r = get_i_param(1); \
198 if (unlikely(frame_test_flag(fp, slot_1))) { \
199 goto cat(escape_convert_thunk_,ARG_MODE); \
200 } else { \
201 utype val1; \
202 int_default_t r; \
203 barrier_aliasing(); \
204 val1 = *frame_slot(fp, slot_1, utype); \
205 r = (int_default_t)val1; \
206 if (unlikely((utype)r != val1) || unlikely(r < 0)) \
207 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, true));\
208 else \
209 *frame_slot(fp, slot_r, int_default_t) = r; \
210 barrier_aliasing(); \
211 ADVANCE_I_PARAM(3); \
212 } \
213 )
215 #define DEFINE_FIXED_FROM_INT(type, utype) \
216 DEFINE_OPCODE( \
217 OPCODE_FIXED_OP + \
218 OPCODE_FIXED_OP_from_int * OPCODE_FIXED_OP_MULT + \
219 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
220 cat4(FIXED_unary_from_int_,type,_,ARG_MODE), \
221 frame_t slot_1 = get_i_param(0); \
222 frame_t slot_r = get_i_param(1); \
223 if (unlikely(frame_test_flag(fp, slot_1))) { \
224 goto cat(escape_convert_thunk_,ARG_MODE); \
225 } else { \
226 int_default_t val1; \
227 type r; \
228 barrier_aliasing(); \
229 val1 = *frame_slot(fp, slot_1, int_default_t); \
230 r = (type)val1; \
231 if (unlikely(r != val1)) \
232 frame_set_pointer(fp, slot_r, pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_DOESNT_FIT), fp, ip pass_file_line));\
233 else \
234 *frame_slot(fp, slot_r, type) = r; \
235 barrier_aliasing(); \
236 ADVANCE_I_PARAM(3); \
237 } \
238 )
240 #define DEFINE_FIXED_UFROM_INT(type, utype) \
241 DEFINE_OPCODE( \
242 OPCODE_FIXED_OP + \
243 OPCODE_FIXED_OP_ufrom_int * OPCODE_FIXED_OP_MULT + \
244 cat(OPCODE_FIXED_TYPE_,type) * OPCODE_FIXED_TYPE_MULT, \
245 cat4(FIXED_unary_ufrom_int_,type,_,ARG_MODE), \
246 frame_t slot_1 = get_i_param(0); \
247 frame_t slot_r = get_i_param(1); \
248 if (unlikely(frame_test_flag(fp, slot_1))) { \
249 goto cat(escape_convert_thunk_,ARG_MODE); \
250 } else { \
251 int_default_t val1; \
252 utype r; \
253 barrier_aliasing(); \
254 val1 = *frame_slot(fp, slot_1, int_default_t); \
255 r = (utype)val1; \
256 if (unlikely((int_default_t)r != val1) || unlikely(val1 < 0))\
257 frame_set_pointer(fp, slot_r, pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_DOESNT_FIT), fp, ip pass_file_line));\
258 else \
259 *frame_slot(fp, slot_r, type) = r; \
260 barrier_aliasing(); \
261 ADVANCE_I_PARAM(3); \
262 } \
263 )
265 #define DEFINE_SCALAR_MOVE_OPCODE(tc, type) \
266 DEFINE_OPCODE( \
267 cat3(OPCODE_,tc,OP) + \
268 cat3(OPCODE_,tc,OP_move) * cat3(OPCODE_,tc,OP_MULT) + \
269 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
270 cat5(tc,move_,type,_,ARG_MODE), \
271 move_scalar(type); \
272 )
274 #define DEFINE_SCALAR_COPY_OPCODE(tc, type) \
275 DEFINE_OPCODE( \
276 cat3(OPCODE_,tc,OP) + \
277 cat3(OPCODE_,tc,OP_copy) * cat3(OPCODE_,tc,OP_MULT) + \
278 cat4(OPCODE_,tc,TYPE_,type) * cat3(OPCODE_,tc,TYPE_MULT),\
279 cat5(tc,copy_,type,_,ARG_MODE), \
280 copy_scalar(type); \
281 )
283 #define DEFINE_FIXED_LDC_OPCODE(cls, type, utype, opc, shrt) \
284 DEFINE_OPCODE( \
285 OPCODE_##cls##_OP + \
286 OPCODE_##cls##_OP_##opc * OPCODE_##cls##_OP_MULT + \
287 cat(OPCODE_##cls##_TYPE_,type) * OPCODE_##cls##_TYPE_MULT,\
288 cat6(fixed_,opc,_,type,_,ARG_MODE), \
289 frame_t slot; \
290 utype *val; \
291 ip_t sz; \
292 slot = get_i_param(0); \
293 ajla_assert(!frame_test_flag(fp, slot), (file_line, "ldc fixed: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot));\
294 val = frame_slot(fp, slot, utype); \
295 barrier_aliasing(); \
296 sz = cat(fixed_ldc_,type)(val, ip + 1 + param_size(1), shrt); \
297 barrier_aliasing(); \
298 ADVANCE_I_PARAM(1); \
299 ADVANCE_IP(sz); \
300 )
302 #define DEFINE_FIXED_OPCODES(n, type, utype, sz, bits) \
303 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, add) \
304 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, subtract) \
305 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, multiply) \
306 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, divide) \
307 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, udivide) \
308 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, modulo) \
309 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, umodulo) \
310 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, power) \
311 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, and) \
312 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, or) \
313 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, xor) \
314 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, shl) \
315 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, shr) \
316 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, ushr) \
317 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, rol) \
318 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, ror) \
319 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, bts) \
320 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, btr) \
321 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, btc) \
322 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, equal) \
323 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, not_equal)\
324 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, less) \
325 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, less_equal)\
326 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, greater) \
327 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, greater_equal)\
328 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, uless) \
329 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, uless_equal)\
330 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, ugreater) \
331 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, ugreater_equal)\
332 DEFINE_FIXED_REAL_BINARY_OPCODE(FIXED_, type, bt) \
333 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, add) \
334 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, subtract) \
335 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, multiply) \
336 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, divide) \
337 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, udivide) \
338 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, modulo) \
339 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, umodulo) \
340 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, power) \
341 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, and) \
342 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, or) \
343 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, xor) \
344 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, shl) \
345 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, shr) \
346 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, ushr) \
347 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, rol) \
348 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, ror) \
349 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, bts) \
350 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, btr) \
351 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, btc) \
352 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, equal) \
353 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, not_equal) \
354 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, less) \
355 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, less_equal) \
356 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, greater) \
357 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, greater_equal)\
358 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, uless) \
359 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, uless_equal)\
360 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, ugreater) \
361 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, ugreater_equal)\
362 DEFINE_FIXED_REAL_BINARY_CONST_OPCODE(FIXED_, type, bt) \
363 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, not) \
364 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, neg) \
365 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, bswap) \
366 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, brev) \
367 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, bsf) \
368 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, bsr) \
369 DEFINE_FIXED_REAL_UNARY_OPCODE(FIXED_, type, popcnt) \
370 DEFINE_FIXED_TO_INT(type, utype) \
371 DEFINE_FIXED_UTO_INT(type, utype) \
372 DEFINE_FIXED_FROM_INT(type, utype) \
373 DEFINE_FIXED_UFROM_INT(type, utype) \
374 DEFINE_SCALAR_MOVE_OPCODE(FIXED_, type) \
375 DEFINE_SCALAR_COPY_OPCODE(FIXED_, type) \
376 DEFINE_FIXED_LDC_OPCODE(FIXED, type, utype, ldc, false)
378 for_all_fixed(DEFINE_FIXED_OPCODES)
380 #if TYPE_FIXED_N >= 3
381 DEFINE_FIXED_LDC_OPCODE(FIXED, int32_t, uint32_t, ldc16, true)
382 #if TYPE_FIXED_N >= 4
383 DEFINE_FIXED_LDC_OPCODE(FIXED, int64_t, uint64_t, ldc16, true)
384 #if TYPE_FIXED_N >= 5
385 DEFINE_FIXED_LDC_OPCODE(FIXED, int128_t, uint128_t, ldc16, true)
386 #endif
387 #endif
388 #endif
391 #define DEFINE_REAL_TO_INT(type, op) \
392 DEFINE_OPCODE( \
393 OPCODE_REAL_OP + \
394 cat(OPCODE_REAL_OP_,op) * OPCODE_REAL_OP_MULT + \
395 cat(OPCODE_REAL_TYPE_,type) * OPCODE_REAL_TYPE_MULT, \
396 cat6(REAL_unary_,op,_,type,_,ARG_MODE), \
397 frame_t slot_1 = get_i_param(0); \
398 frame_t slot_r = get_i_param(1); \
399 if (unlikely(frame_test_flag(fp, slot_1))) { \
400 goto cat(escape_convert_thunk_,ARG_MODE); \
401 } else { \
402 bool ret; \
403 barrier_aliasing(); \
404 ret = cat4(REAL_unary_,op,_,type)(frame_slot(fp, slot_1, type), frame_slot(fp, slot_r, int_default_t));\
405 barrier_aliasing(); \
406 if (unlikely(!ret)) \
407 frame_set_pointer(fp, slot_r, convert_real_to_mpint(fp, slot_1, frame_get_type_of_local(fp, slot_1)));\
408 ADVANCE_I_PARAM(3); \
409 } \
410 )
412 #define DEFINE_REAL_FROM_INT(type, op) \
413 DEFINE_OPCODE( \
414 OPCODE_REAL_OP + \
415 cat(OPCODE_REAL_OP_,op) * OPCODE_REAL_OP_MULT + \
416 cat(OPCODE_REAL_TYPE_,type) * OPCODE_REAL_TYPE_MULT, \
417 cat6(REAL_unary_,op,_,type,_,ARG_MODE), \
418 frame_t slot_1 = get_i_param(0); \
419 frame_t slot_r = get_i_param(1); \
420 if (unlikely(frame_test_flag(fp, slot_1))) { \
421 goto cat(escape_convert_thunk_,ARG_MODE); \
422 } else { \
423 barrier_aliasing(); \
424 cat4(REAL_unary_,op,_,type)(frame_slot(fp, slot_1, int_default_t), frame_slot(fp, slot_r, type));\
425 barrier_aliasing(); \
426 ADVANCE_I_PARAM(3); \
427 } \
428 )
431 #define DEFINE_REAL_OPCODES(n, type, ntype, pack, unpack)\
432 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, add) \
433 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, subtract) \
434 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, multiply) \
435 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, divide) \
436 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, modulo) \
437 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, power) \
438 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, ldexp) \
439 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, atan2) \
440 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, equal) \
441 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, not_equal) \
442 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, less) \
443 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, less_equal)\
444 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, greater) \
445 DEFINE_FIXED_REAL_BINARY_OPCODE(REAL_, type, greater_equal)\
446 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, neg) \
447 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, sqrt) \
448 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, cbrt) \
449 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, sin) \
450 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, cos) \
451 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, tan) \
452 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, asin) \
453 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, acos) \
454 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, atan) \
455 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, sinh) \
456 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, cosh) \
457 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, tanh) \
458 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, asinh) \
459 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, acosh) \
460 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, atanh) \
461 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exp2) \
462 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exp) \
463 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exp10) \
464 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, log2) \
465 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, log) \
466 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, log10) \
467 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, round) \
468 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, ceil) \
469 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, floor) \
470 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, trunc) \
471 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, fract) \
472 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, mantissa) \
473 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, exponent) \
474 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, next_number)\
475 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, prev_number)\
476 DEFINE_REAL_TO_INT(type, to_int) \
477 DEFINE_REAL_FROM_INT(type, from_int) \
478 DEFINE_FIXED_REAL_UNARY_OPCODE(REAL_, type, is_exception)\
479 DEFINE_SCALAR_MOVE_OPCODE(REAL_, type) \
480 DEFINE_SCALAR_COPY_OPCODE(REAL_, type) \
481 DEFINE_FIXED_LDC_OPCODE(REAL, type, type, ldc, false)
483 for_all_real(DEFINE_REAL_OPCODES, for_all_empty)
486 DEFINE_LABEL(cat(escape_fixed_binary_thunk_,ARG_MODE),
487 void *ex;
488 ASM_PREVENT_CSE;
489 ex = thunk_fixed_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3));
490 if (ex == POINTER_FOLLOW_THUNK_GO)
491 ADVANCE_I_PARAM(4);
492 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
493 RELOAD_EX_POSITION(ex);
494 )
497 DEFINE_LABEL(cat(escape_fixed_binary_const_thunk_,ARG_MODE),
498 void *ex;
499 ASM_PREVENT_CSE;
500 ex = thunk_fixed_operator(fp, ip, get_i_param(0), frame_t_from_const((int32_t)get_i_param(1)), get_i_param(2), get_i_param(3));
501 if (ex == POINTER_FOLLOW_THUNK_GO)
502 ADVANCE_I_PARAM(4);
503 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
504 RELOAD_EX_POSITION(ex);
505 )
508 DEFINE_LABEL(cat(escape_fixed_unary_thunk_,ARG_MODE),
509 void *ex;
510 ASM_PREVENT_CSE;
511 ex = thunk_fixed_operator(fp, ip, get_i_param(0), NO_FRAME_T, get_i_param(1), get_i_param(2));
512 if (ex == POINTER_FOLLOW_THUNK_GO)
513 ADVANCE_I_PARAM(3);
514 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
515 RELOAD_EX_POSITION(ex);
516 )
519 DEFINE_LABEL(cat(escape_convert_thunk_,ARG_MODE),
520 void *ex;
521 ASM_PREVENT_CSE;
522 ex = thunk_convert(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2));
523 if (ex == POINTER_FOLLOW_THUNK_GO)
524 ADVANCE_I_PARAM(3);
525 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
526 RELOAD_EX_POSITION(ex);
527 )
530 START_BLOCK(
531 bool (attr_fastcall *function_int_binary)(const mpint_t *s1, const mpint_t *s2, mpint_t *r, ajla_error_t *err);
532 bool (attr_fastcall *function_int_binary_logical)(const mpint_t *s1, const mpint_t *s2, ajla_flat_option_t *r, ajla_error_t *err);
533 bool (attr_fastcall *function_int_unary)(const mpint_t *s1, mpint_t *r, ajla_error_t *err);
534 )
536 #define DEFINE_INT_BINARY_OPCODE(type, op) \
537 DEFINE_OPCODE( \
538 OPCODE_INT_OP + \
539 cat(OPCODE_INT_OP_,op) * OPCODE_INT_OP_MULT + \
540 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
541 cat6(int_binary_,op,_,type,_,ARG_MODE), \
542 frame_t slot_1 = get_i_param(0); \
543 frame_t slot_2 = get_i_param(1); \
544 frame_t slot_r = get_i_param(2); \
545 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "int binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
546 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
547 cat6(escape_int_binary_,op,_,type,_,ARG_MODE): \
548 function_int_binary = cat(mpint_,op); \
549 goto cat(escape_int_binary_thunk_,ARG_MODE); \
550 } else { \
551 const void *val1; \
552 const void *val2; \
553 void *result; \
554 val1 = frame_var(fp, slot_1); \
555 val2 = frame_var(fp, slot_2); \
556 result = frame_var(fp, slot_r); \
557 barrier_aliasing(); \
558 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, val2, result))) {\
559 barrier_aliasing(); \
560 goto cat6(escape_int_binary_,op,_,type,_,ARG_MODE);\
561 } \
562 barrier_aliasing(); \
563 ADVANCE_I_PARAM(4); \
564 } \
565 )
567 #define DEFINE_INT_BINARY_LOGICAL_OPCODE(type, op) \
568 DEFINE_OPCODE( \
569 OPCODE_INT_OP + \
570 cat(OPCODE_INT_OP_,op) * OPCODE_INT_OP_MULT + \
571 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
572 cat6(int_binary_,op,_,type,_,ARG_MODE), \
573 frame_t slot_1 = get_i_param(0); \
574 frame_t slot_2 = get_i_param(1); \
575 frame_t slot_r = get_i_param(2); \
576 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "int logical: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
577 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
578 cat6(escape_int_binary_logical_,op,_,type,_,ARG_MODE): \
579 function_int_binary_logical = cat(mpint_,op); \
580 goto cat(escape_int_binary_logical_thunk_,ARG_MODE); \
581 } else { \
582 const void *val1; \
583 const void *val2; \
584 void *result; \
585 val1 = frame_var(fp, slot_1); \
586 val2 = frame_var(fp, slot_2); \
587 result = frame_var(fp, slot_r); \
588 barrier_aliasing(); \
589 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, val2, result))) {\
590 barrier_aliasing(); \
591 goto cat6(escape_int_binary_logical_,op,_,type,_,ARG_MODE);\
592 } \
593 barrier_aliasing(); \
594 ADVANCE_I_PARAM(4); \
595 } \
596 )
598 #define DEFINE_INT_BINARY_CONST_OPCODE(type, op) \
599 DEFINE_OPCODE( \
600 OPCODE_INT_OP + \
601 cat(OPCODE_INT_OP_C_,op) * OPCODE_INT_OP_MULT + \
602 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
603 cat6(int_binary_const_,op,_,type,_,ARG_MODE), \
604 frame_t slot_1 = get_i_param(0); \
605 frame_t slot_2 = get_i_param(1); \
606 frame_t slot_r = get_i_param(2); \
607 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "int binary const: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
608 if (unlikely(frame_test_flag(fp, slot_1))) { \
609 cat6(escape_int_binary_const_,op,_,type,_,ARG_MODE): \
610 function_int_binary = cat(mpint_,op); \
611 goto cat(escape_int_binary_const_thunk_,ARG_MODE); \
612 } else { \
613 const void *val1; \
614 type val2; \
615 void *result; \
616 val1 = frame_var(fp, slot_1); \
617 val2 = (int32_t)slot_2; \
618 result = frame_var(fp, slot_r); \
619 barrier_aliasing(); \
620 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, &val2, result))) {\
621 barrier_aliasing(); \
622 goto cat6(escape_int_binary_const_,op,_,type,_,ARG_MODE);\
623 } \
624 barrier_aliasing(); \
625 ADVANCE_I_PARAM(4); \
626 } \
627 )
629 #define DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(type, op) \
630 DEFINE_OPCODE( \
631 OPCODE_INT_OP + \
632 cat(OPCODE_INT_OP_C_,op) * OPCODE_INT_OP_MULT + \
633 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
634 cat6(int_binary_const_,op,_,type,_,ARG_MODE), \
635 frame_t slot_1 = get_i_param(0); \
636 frame_t slot_2 = get_i_param(1); \
637 frame_t slot_r = get_i_param(2); \
638 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "int logical const: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
639 if (unlikely(frame_test_flag(fp, slot_1))) { \
640 cat6(escape_int_binary_const_logical_,op,_,type,_,ARG_MODE): \
641 function_int_binary_logical = cat(mpint_,op); \
642 goto cat(escape_int_binary_const_logical_thunk_,ARG_MODE);\
643 } else { \
644 const void *val1; \
645 type val2; \
646 void *result; \
647 val1 = frame_var(fp, slot_1); \
648 val2 = (int32_t)slot_2; \
649 result = frame_var(fp, slot_r); \
650 barrier_aliasing(); \
651 if (unlikely(!cat4(INT_binary_,op,_,type)(val1, &val2, result))) {\
652 barrier_aliasing(); \
653 goto cat6(escape_int_binary_const_logical_,op,_,type,_,ARG_MODE);\
654 } \
655 barrier_aliasing(); \
656 ADVANCE_I_PARAM(4); \
657 } \
658 )
660 #define DEFINE_INT_UNARY_OPCODE(type, op) \
661 DEFINE_OPCODE( \
662 OPCODE_INT_OP + \
663 cat(OPCODE_INT_OP_,op) * OPCODE_INT_OP_MULT + \
664 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
665 cat6(int_binary_,op,_,type,_,ARG_MODE), \
666 frame_t slot_1 = get_i_param(0); \
667 frame_t slot_r = get_i_param(1); \
668 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "int unary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
669 if (unlikely(frame_test_flag(fp, slot_1))) { \
670 cat6(escape_int_unary_,op,_,type,_,ARG_MODE): \
671 function_int_unary = cat(mpint_,op); \
672 goto cat(escape_int_unary_thunk_,ARG_MODE); \
673 } else { \
674 const void *val1; \
675 void *result; \
676 val1 = frame_var(fp, slot_1); \
677 result = frame_var(fp, slot_r); \
678 barrier_aliasing(); \
679 if (unlikely(!cat4(INT_unary_,op,_,type)(val1, result))) {\
680 barrier_aliasing(); \
681 goto cat6(escape_int_unary_,op,_,type,_,ARG_MODE);\
682 } \
683 barrier_aliasing(); \
684 ADVANCE_I_PARAM(3); \
685 } \
686 )
688 #define DEFINE_INT_TO_INT(type) \
689 DEFINE_OPCODE( \
690 OPCODE_INT_OP + \
691 OPCODE_INT_OP_to_int * OPCODE_INT_OP_MULT + \
692 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
693 cat4(INT_unary_to_int_,type,_,ARG_MODE), \
694 frame_t slot_1 = get_i_param(0); \
695 frame_t slot_r = get_i_param(1); \
696 if (unlikely(frame_test_flag(fp, slot_1))) { \
697 goto cat(escape_convert_thunk_,ARG_MODE); \
698 } else { \
699 type val1; \
700 int_default_t r; \
701 barrier_aliasing(); \
702 val1 = *frame_slot(fp, slot_1, type); \
703 r = (int_default_t)val1; \
704 if (unlikely(r != val1)) \
705 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, false));\
706 else \
707 *frame_slot(fp, slot_r, int_default_t) = r; \
708 barrier_aliasing(); \
709 ADVANCE_I_PARAM(3); \
710 } \
711 )
713 #define DEFINE_INT_FROM_INT(typeid, type) \
714 DEFINE_OPCODE( \
715 OPCODE_INT_OP + \
716 OPCODE_INT_OP_from_int * OPCODE_INT_OP_MULT + \
717 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
718 cat4(INT_unary_from_int_,type,_,ARG_MODE), \
719 frame_t slot_1 = get_i_param(0); \
720 frame_t slot_r = get_i_param(1); \
721 if (unlikely(frame_test_flag(fp, slot_1))) { \
722 goto cat(escape_convert_thunk_,ARG_MODE); \
723 } else { \
724 int_default_t val1; \
725 type r; \
726 barrier_aliasing(); \
727 val1 = *frame_slot(fp, slot_1, int_default_t); \
728 r = (type)val1; \
729 if (unlikely(r != val1)) \
730 frame_set_pointer(fp, slot_r, convert_fixed_to_mpint(val1, false));\
731 else \
732 *frame_slot(fp, slot_r, type) = r; \
733 barrier_aliasing(); \
734 ADVANCE_I_PARAM(3); \
735 } \
736 )
738 #define DEFINE_INT_LDC_OPCODE(type, opc, shrt) \
739 DEFINE_OPCODE( \
740 OPCODE_INT_OP + \
741 OPCODE_INT_OP_##opc * OPCODE_INT_OP_MULT + \
742 cat(OPCODE_INT_TYPE_,type) * OPCODE_INT_TYPE_MULT, \
743 cat6(int_,opc,_,type,_,ARG_MODE), \
744 frame_t slot; \
745 type *val; \
746 ip_t sz; \
747 slot = get_i_param(0); \
748 ajla_assert(!frame_test_flag(fp, slot), (file_line, "ldc int: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot));\
749 val = frame_slot(fp, slot, type); \
750 barrier_aliasing(); \
751 sz = cat(int_ldc_,type)(val, ip + 1 + param_size(1), shrt); \
752 barrier_aliasing(); \
753 ADVANCE_I_PARAM(1); \
754 ADVANCE_IP(sz); \
755 )
757 #define DEFINE_INT_OPCODES(typeid, s, u, sz, bits) \
758 DEFINE_INT_BINARY_OPCODE(s, add) \
759 DEFINE_INT_BINARY_OPCODE(s, subtract) \
760 DEFINE_INT_BINARY_OPCODE(s, multiply) \
761 DEFINE_INT_BINARY_OPCODE(s, divide) \
762 DEFINE_INT_BINARY_OPCODE(s, modulo) \
763 DEFINE_INT_BINARY_OPCODE(s, power) \
764 DEFINE_INT_BINARY_OPCODE(s, and) \
765 DEFINE_INT_BINARY_OPCODE(s, or) \
766 DEFINE_INT_BINARY_OPCODE(s, xor) \
767 DEFINE_INT_BINARY_OPCODE(s, shl) \
768 DEFINE_INT_BINARY_OPCODE(s, shr) \
769 DEFINE_INT_BINARY_OPCODE(s, bts) \
770 DEFINE_INT_BINARY_OPCODE(s, btr) \
771 DEFINE_INT_BINARY_OPCODE(s, btc) \
772 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, equal) \
773 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, not_equal) \
774 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, less) \
775 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, less_equal) \
776 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, greater) \
777 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, greater_equal) \
778 DEFINE_INT_BINARY_LOGICAL_OPCODE(s, bt) \
779 DEFINE_INT_BINARY_CONST_OPCODE(s, add) \
780 DEFINE_INT_BINARY_CONST_OPCODE(s, subtract) \
781 DEFINE_INT_BINARY_CONST_OPCODE(s, multiply) \
782 DEFINE_INT_BINARY_CONST_OPCODE(s, divide) \
783 DEFINE_INT_BINARY_CONST_OPCODE(s, modulo) \
784 DEFINE_INT_BINARY_CONST_OPCODE(s, power) \
785 DEFINE_INT_BINARY_CONST_OPCODE(s, and) \
786 DEFINE_INT_BINARY_CONST_OPCODE(s, or) \
787 DEFINE_INT_BINARY_CONST_OPCODE(s, xor) \
788 DEFINE_INT_BINARY_CONST_OPCODE(s, shl) \
789 DEFINE_INT_BINARY_CONST_OPCODE(s, shr) \
790 DEFINE_INT_BINARY_CONST_OPCODE(s, bts) \
791 DEFINE_INT_BINARY_CONST_OPCODE(s, btr) \
792 DEFINE_INT_BINARY_CONST_OPCODE(s, btc) \
793 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, equal) \
794 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, not_equal) \
795 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, less) \
796 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, less_equal) \
797 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, greater) \
798 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, greater_equal)\
799 DEFINE_INT_BINARY_CONST_LOGICAL_OPCODE(s, bt) \
800 DEFINE_INT_UNARY_OPCODE(s, not) \
801 DEFINE_INT_UNARY_OPCODE(s, neg) \
802 DEFINE_INT_UNARY_OPCODE(s, bsf) \
803 DEFINE_INT_UNARY_OPCODE(s, bsr) \
804 DEFINE_INT_UNARY_OPCODE(s, popcnt) \
805 DEFINE_INT_TO_INT(s) \
806 DEFINE_INT_FROM_INT(typeid, s) \
807 DEFINE_SCALAR_MOVE_OPCODE(INT_, s) \
808 DEFINE_SCALAR_COPY_OPCODE(INT_, s) \
809 DEFINE_INT_LDC_OPCODE(s, ldc, false) \
810 DEFINE_INT_LDC_OPCODE(s, ldc16, true)
812 for_all_int(DEFINE_INT_OPCODES, for_all_empty)
814 DEFINE_LABEL(cat(escape_int_binary_thunk_,ARG_MODE),
815 void *ex;
816 ASM_PREVENT_CSE;
817 ex = thunk_int_binary_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3), function_int_binary);
818 if (ex == POINTER_FOLLOW_THUNK_GO)
819 ADVANCE_I_PARAM(4);
820 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
821 RELOAD_EX_POSITION(ex);
822 )
824 DEFINE_LABEL(cat(escape_int_binary_logical_thunk_,ARG_MODE),
825 void *ex;
826 ASM_PREVENT_CSE;
827 ex = thunk_int_binary_logical_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3), function_int_binary_logical);
828 if (ex == POINTER_FOLLOW_THUNK_GO)
829 ADVANCE_I_PARAM(4);
830 else if (ex != POINTER_FOLLOW_THUNK_RETRY)
831 RELOAD_EX_POSITION(ex);
832 )
834 DEFINE_LABEL(cat(escape_int_binary_const_thunk_,ARG_MODE),
835 void *ex;
836 ASM_PREVENT_CSE;
837 ex = thunk_int_binary_operator(fp, ip, get_i_param(0), frame_t_from_const((int32_t)get_i_param(1)), get_i_param(2), get_i_param(3), function_int_binary);
838 if (ex == POINTER_FOLLOW_THUNK_GO)
839 ADVANCE_I_PARAM(4);
840 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
841 RELOAD_EX_POSITION(ex);
842 )
844 DEFINE_LABEL(cat(escape_int_binary_const_logical_thunk_,ARG_MODE),
845 void *ex;
846 ASM_PREVENT_CSE;
847 ex = thunk_int_binary_logical_operator(fp, ip, get_i_param(0), frame_t_from_const((int32_t)get_i_param(1)), get_i_param(2), get_i_param(3), function_int_binary_logical);
848 if (ex == POINTER_FOLLOW_THUNK_GO)
849 ADVANCE_I_PARAM(4);
850 else if (ex != POINTER_FOLLOW_THUNK_RETRY)
851 RELOAD_EX_POSITION(ex);
852 )
854 DEFINE_LABEL(cat(escape_int_unary_thunk_,ARG_MODE),
855 void *ex;
856 ASM_PREVENT_CSE;
857 ex = thunk_int_unary_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), function_int_unary);
858 if (ex == POINTER_FOLLOW_THUNK_GO)
859 ADVANCE_I_PARAM(3);
860 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
861 RELOAD_EX_POSITION(ex);
862 )
864 #define file_inc "ipret-a1.inc"
865 #include "for-fix.inc"
867 #define file_inc "ipret-a2.inc"
868 #include "for-int.inc"
870 #define file_inc "ipret-a3.inc"
871 #include "for-real.inc"
873 END_BLOCK()
876 #define DEFINE_BOOL_BINARY_OPCODE(op, operator) \
877 DEFINE_OPCODE( \
878 OPCODE_BOOL_OP + \
879 cat(OPCODE_BOOL_OP_,op) * OPCODE_BOOL_OP_MULT, \
880 cat4(bool_binary_,op,_,ARG_MODE), \
881 frame_t slot_1 = get_i_param(0); \
882 frame_t slot_2 = get_i_param(1); \
883 frame_t slot_r = get_i_param(2); \
884 \
885 if (unlikely(frame_test_2(fp, slot_1, slot_2))) { \
886 ajla_assert(slot_r == slot_1 || slot_r == slot_2 || !frame_test_flag(fp, slot_r), (file_line, "bool binary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
887 goto cat(escape_bool_binary_thunk_,ARG_MODE); \
888 } else { \
889 ajla_flat_option_t val1; \
890 ajla_flat_option_t val2; \
891 ajla_flat_option_t result; \
892 barrier_aliasing(); \
893 val1 = *frame_slot(fp, slot_1, ajla_flat_option_t); \
894 val2 = *frame_slot(fp, slot_2, ajla_flat_option_t); \
895 result = val1 operator val2; \
896 *frame_slot(fp, slot_r, ajla_flat_option_t) = result; \
897 barrier_aliasing(); \
898 ADVANCE_I_PARAM(4); \
899 } \
900 )
902 #define DEFINE_BOOL_UNARY_OPCODE(op, operator) \
903 DEFINE_OPCODE( \
904 OPCODE_BOOL_OP + \
905 cat(OPCODE_BOOL_OP_,op) * OPCODE_BOOL_OP_MULT, \
906 cat4(bool_unary_,op,_,ARG_MODE), \
907 frame_t slot_1 = get_i_param(0); \
908 frame_t slot_r = get_i_param(1); \
909 \
910 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "bool unary: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));\
911 if (unlikely(frame_test_flag(fp, slot_1))) { \
912 goto cat(escape_bool_unary_thunk_,ARG_MODE); \
913 } else { \
914 ajla_flat_option_t val1; \
915 ajla_flat_option_t result; \
916 barrier_aliasing(); \
917 val1 = *frame_slot(fp, slot_1, ajla_flat_option_t); \
918 result = val1 operator; \
919 *frame_slot(fp, slot_r, ajla_flat_option_t) = result; \
920 barrier_aliasing(); \
921 ADVANCE_I_PARAM(3); \
922 } \
923 )
925 DEFINE_BOOL_BINARY_OPCODE(and, &)
926 DEFINE_BOOL_BINARY_OPCODE(or, |)
927 DEFINE_BOOL_BINARY_OPCODE(equal, ==)
928 DEFINE_BOOL_BINARY_OPCODE(not_equal, ^)
929 DEFINE_BOOL_BINARY_OPCODE(less, <)
930 DEFINE_BOOL_BINARY_OPCODE(less_equal, <=)
931 DEFINE_BOOL_BINARY_OPCODE(greater, >)
932 DEFINE_BOOL_BINARY_OPCODE(greater_equal, >=)
933 DEFINE_BOOL_UNARY_OPCODE(not, ^ 1)
935 DEFINE_LABEL(cat(escape_bool_binary_thunk_,ARG_MODE),
936 void *ex;
937 ASM_PREVENT_CSE;
938 ex = thunk_bool_operator(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), get_i_param(3));
939 if (ex == POINTER_FOLLOW_THUNK_GO)
940 ADVANCE_I_PARAM(4);
941 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
942 RELOAD_EX_POSITION(ex);
943 )
945 DEFINE_LABEL(cat(escape_bool_unary_thunk_,ARG_MODE),
946 void *ex;
947 ASM_PREVENT_CSE;
948 ex = thunk_bool_operator(fp, ip, get_i_param(0), NO_FRAME_T, get_i_param(1), get_i_param(2));
949 if (ex == POINTER_FOLLOW_THUNK_GO)
950 ADVANCE_I_PARAM(3);
951 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
952 RELOAD_EX_POSITION(ex);
953 )
956 DEFINE_OPCODE_START_LBL(OPCODE_BOOL_OP + OPCODE_BOOL_OP_move, cat(bool_move_,ARG_MODE))
957 #ifdef EMIT_CODE
958 move_scalar(ajla_flat_option_t);
959 #endif
960 DEFINE_OPCODE_END(OPCODE_BOOL_OP + OPCODE_BOOL_OP_move)
962 DEFINE_OPCODE_START_LBL(OPCODE_BOOL_OP + OPCODE_BOOL_OP_copy, cat(bool_copy_,ARG_MODE))
963 #ifdef EMIT_CODE
964 copy_scalar(ajla_flat_option_t);
965 #endif
966 DEFINE_OPCODE_END(OPCODE_BOOL_OP + OPCODE_BOOL_OP_copy)
969 DEFINE_OPCODE_START(OPCODE_INT_LDC_LONG)
970 #ifdef EMIT_CODE
971 frame_t slot;
972 ip_t sz;
973 slot = get_i_param(0);
974 sz = ipret_int_ldc_long(fp, slot, ip + 1 + param_size(1));
975 ADVANCE_I_PARAM(1);
976 ADVANCE_IP(sz);
977 #endif
978 DEFINE_OPCODE_END(OPCODE_INT_LDC_LONG)
981 DEFINE_OPCODE_START(OPCODE_IS_EXCEPTION)
982 #ifdef EMIT_CODE
983 frame_t slot_1 = get_i_param(0);
984 frame_t slot_r = get_i_param(1);
985 ajla_assert(slot_r == slot_1 || !frame_test_flag(fp, slot_r), (file_line, "is exception: %s: flag already set for destination slot %"PRIuMAX"", da(get_frame(fp)->function,function)->function_name, (uintmax_t)slot_r));
986 if (!frame_variable_is_flat(fp, slot_1)) {
987 void *ex = is_thunk_operator(fp, ip, slot_1, slot_r, get_i_param(2));
988 if (ex == POINTER_FOLLOW_THUNK_GO)
989 ADVANCE_I_PARAM(3);
990 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
991 RELOAD_EX_POSITION(ex);
992 } else {
993 barrier_aliasing();
994 *frame_slot(fp, slot_r, ajla_flat_option_t) = 0;
995 barrier_aliasing();
996 ADVANCE_I_PARAM(3);
997 }
998 #endif
999 DEFINE_OPCODE_END(OPCODE_IS_EXCEPTION)
1001 DEFINE_OPCODE_START(OPCODE_EXCEPTION_CLASS)
1002 #ifdef EMIT_CODE
1003 void *ex;
1004 ex = thunk_get_param(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), 0);
1005 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
1006 ADVANCE_I_PARAM(3);
1007 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
1008 RELOAD_EX_POSITION(ex);
1009 #endif
1010 DEFINE_OPCODE_END(OPCODE_EXCEPTION_CLASS)
1012 DEFINE_OPCODE_START(OPCODE_EXCEPTION_TYPE)
1013 #ifdef EMIT_CODE
1014 void *ex;
1015 ex = thunk_get_param(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), 1);
1016 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
1017 ADVANCE_I_PARAM(3);
1018 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
1019 RELOAD_EX_POSITION(ex);
1020 #endif
1021 DEFINE_OPCODE_END(OPCODE_EXCEPTION_TYPE)
1023 DEFINE_OPCODE_START(OPCODE_EXCEPTION_AUX)
1024 #ifdef EMIT_CODE
1025 void *ex;
1026 ex = thunk_get_param(fp, ip, get_i_param(0), get_i_param(1), get_i_param(2), 2);
1027 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
1028 ADVANCE_I_PARAM(3);
1029 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
1030 RELOAD_EX_POSITION(ex);
1031 #endif
1032 DEFINE_OPCODE_END(OPCODE_EXCEPTION_AUX)
1034 DEFINE_OPCODE_START(OPCODE_SYSTEM_PROPERTY)
1035 #ifdef EMIT_CODE
1036 void *ex;
1037 ex = ipret_get_system_property(fp, ip, get_i_param(0), get_i_param(1));
1038 if (likely(ex == POINTER_FOLLOW_THUNK_GO))
1039 ADVANCE_I_PARAM(3);
1040 else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY))
1041 RELOAD_EX_POSITION(ex);
1042 #endif
1043 DEFINE_OPCODE_END(OPCODE_SYSTEM_PROPERTY)
1046 DEFINE_OPCODE_START(OPCODE_FLAT_MOVE)
1047 #ifdef EMIT_CODE
1048 frame_t slot_1 = get_i_param(0);
1049 frame_t slot_r = get_i_param(1);
1050 if (likely(frame_variable_is_flat(fp, slot_1))) {
1051 memcpy(frame_var(fp, slot_r), frame_var(fp, slot_1), frame_get_type_of_local(fp, slot_1)->size);
1052 } else {
1053 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, true);
1054 frame_set_pointer(fp, slot_r, ptr);
1055 }
1056 ADVANCE_I_PARAM(2);
1057 #endif
1058 DEFINE_OPCODE_END(OPCODE_FLAT_MOVE)
1060 DEFINE_OPCODE_START(OPCODE_FLAT_COPY)
1061 #ifdef EMIT_CODE
1062 frame_t slot_1 = get_i_param(0);
1063 frame_t slot_r = get_i_param(1);
1064 if (likely(frame_variable_is_flat(fp, slot_1))) {
1065 memcpy(frame_var(fp, slot_r), frame_var(fp, slot_1), frame_get_type_of_local(fp, slot_1)->size);
1066 } else {
1067 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, false);
1068 frame_set_pointer(fp, slot_r, ptr);
1069 }
1070 ADVANCE_I_PARAM(2);
1071 #endif
1072 DEFINE_OPCODE_END(OPCODE_FLAT_COPY)
1074 DEFINE_OPCODE_START(OPCODE_REF_MOVE)
1075 #ifdef EMIT_CODE
1076 frame_t slot_1 = get_i_param(0);
1077 frame_t slot_r = get_i_param(1);
1078 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, true);
1079 pointer_poison(frame_pointer(fp, slot_1));
1080 frame_set_pointer(fp, slot_r, ptr);
1081 ADVANCE_I_PARAM(2);
1082 #endif
1083 DEFINE_OPCODE_END(OPCODE_REF_MOVE)
1085 DEFINE_OPCODE_START(OPCODE_REF_MOVE_CLEAR)
1086 #ifdef EMIT_CODE
1087 frame_t slot_1 = get_i_param(0);
1088 frame_t slot_r = get_i_param(1);
1089 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, true);
1090 frame_set_pointer(fp, slot_r, ptr);
1091 ADVANCE_I_PARAM(2);
1092 #endif
1093 DEFINE_OPCODE_END(OPCODE_REF_MOVE_CLEAR)
1095 DEFINE_OPCODE_START(OPCODE_REF_COPY)
1096 #ifdef EMIT_CODE
1097 frame_t slot_1 = get_i_param(0);
1098 frame_t slot_r = get_i_param(1);
1099 pointer_t ptr = frame_get_pointer_reference(fp, slot_1, false);
1100 frame_set_pointer(fp, slot_r, ptr);
1101 ADVANCE_I_PARAM(2);
1102 #endif
1103 DEFINE_OPCODE_END(OPCODE_REF_COPY)
1105 DEFINE_OPCODE_START(OPCODE_BOX_MOVE_CLEAR)
1106 #ifdef EMIT_CODE
1107 frame_t slot_1 = get_i_param(0);
1108 frame_t slot_r = get_i_param(1);
1109 pointer_t ptr = ipret_copy_variable_to_pointer(fp, slot_1, true);
1110 *frame_pointer(fp, slot_1) = pointer_empty();
1111 frame_set_pointer(fp, slot_r, ptr);
1112 ADVANCE_I_PARAM(2);
1113 #endif
1114 DEFINE_OPCODE_END(OPCODE_BOX_MOVE_CLEAR)
1116 DEFINE_OPCODE_START(OPCODE_BOX_COPY)
1117 #ifdef EMIT_CODE
1118 frame_t slot_1 = get_i_param(0);
1119 frame_t slot_r = get_i_param(1);
1120 pointer_t ptr = ipret_copy_variable_to_pointer(fp, slot_1, false);
1121 frame_set_pointer(fp, slot_r, ptr);
1122 ADVANCE_I_PARAM(2);
1123 #endif
1124 DEFINE_OPCODE_END(OPCODE_BOX_COPY)
1126 DEFINE_OPCODE_START(OPCODE_TAKE_BORROWED)
1127 #ifdef EMIT_CODE
1128 frame_t slot = get_i_param(0);
1129 if (!frame_test_and_set_flag(fp, slot)) {
1130 pointer_t ptr = *frame_pointer(fp, slot);
1131 pointer_reference_owned(ptr);
1132 }
1133 ADVANCE_I_PARAM(1);
1134 #endif
1135 DEFINE_OPCODE_END(OPCODE_TAKE_BORROWED)
1138 DEFINE_OPCODE_START(OPCODE_DEREFERENCE)
1139 #ifdef EMIT_CODE
1140 frame_t slot = get_i_param(0);
1141 #if 0
1142 {
1143 struct data *fn = get_frame(fp)->function;
1144 const struct type *type = frame_get_type_of_local(fp, slot);
1145 bool may_be_borrowed = da(fn,function)->local_variables[slot].may_be_borrowed;
1146 if (!frame_test_flag(fp, slot) && !may_be_borrowed && !TYPE_IS_FLAT(type))
1147 internal(file_line, "dereference: %s: the value is unexpectedly borrowed", da(fn,function)->function_name);
1148 }
1149 #endif
1150 frame_free(fp, slot);
1151 pointer_poison(frame_pointer(fp, slot));
1152 ADVANCE_I_PARAM(1);
1153 #endif
1154 DEFINE_OPCODE_END(OPCODE_DEREFERENCE)
1156 DEFINE_OPCODE_START(OPCODE_DEREFERENCE_CLEAR)
1157 #ifdef EMIT_CODE
1158 frame_t slot = get_i_param(0);
1159 #if 0
1160 {
1161 struct data *fn = get_frame(fp)->function;
1162 const struct type *type = frame_get_type_of_local(fp, slot);
1163 bool may_be_borrowed = da(fn,function)->local_variables[slot].may_be_borrowed;
1164 if (!frame_test_flag(fp, slot) && !may_be_borrowed && !TYPE_IS_FLAT(type))
1165 internal(file_line, "dereference clear: %s: the value is unexpectedly borrowed", da(fn,function)->function_name);
1166 }
1167 #endif
1168 frame_free_and_clear(fp, slot);
1169 ADVANCE_I_PARAM(1);
1170 #endif
1171 DEFINE_OPCODE_END(OPCODE_DEREFERENCE_CLEAR)
1173 DEFINE_OPCODE_START(OPCODE_EVAL)
1174 #ifdef EMIT_CODE
1175 frame_t slot = get_i_param(0);
1176 if (frame_test_flag(fp, slot)) {
1177 pointer_t *ptr = frame_pointer(fp, slot);
1178 struct data attr_unused *result;
1179 /*cat(eval_again_,ARG_MODE):*/
1180 pointer_follow(ptr, true, result, PF_WAIT, fp, ip,
1181 RELOAD_EX_POSITION(ex_),
1182 goto cat(eval_skip_,ARG_MODE));
1183 /*
1184 if (unlikely(da_tag(result) == DATA_TAG_array_incomplete)) {
1185 ptr = &da(result,array_incomplete)->next;
1186 goto cat(eval_again_,ARG_MODE);
1187 }
1188 */
1189 cat(eval_skip_,ARG_MODE):;
1190 }
1191 ADVANCE_I_PARAM(1);
1192 #endif
1193 DEFINE_OPCODE_END(OPCODE_EVAL)
1196 DEFINE_OPCODE_START(OPCODE_ESCAPE_NONFLAT)
1197 #ifdef EMIT_CODE
1198 frame_t n = get_i_param(0);
1199 ADVANCE_I_PARAM(1);
1200 ADVANCE_IP(n * ARG_MODE);
1201 #endif
1202 DEFINE_OPCODE_END(OPCODE_ESCAPE_NONFLAT)
1204 DEFINE_OPCODE_START(OPCODE_CHECKPOINT)
1205 #ifdef EMIT_CODE
1206 const code_t attr_unused *orig_ip = ip;
1207 frame_t attr_unused id;
1208 ip_t len;
1209 if (SIZEOF_IP_T == 2) {
1210 id = ip[1];
1211 ADVANCE_IP(2);
1212 } else if (SIZEOF_IP_T == 4) {
1213 id = get_unaligned_32(&ip[1]);
1214 ADVANCE_IP(3);
1215 } else {
1216 not_reached();
1217 }
1218 len = get_param(ip, 0);
1219 ADVANCE_IP((len + 1) * ARG_MODE);
1221 if (unlikely(tick_elapsed(&ts)) && likely(frame_execution_control(fp) != NULL)) {
1222 ipret_checkpoint_forced;
1223 } else {
1224 #ifdef HAVE_CODEGEN
1225 struct data *fn = get_frame(fp)->function;
1226 code_return_t r;
1227 void *new_fp;
1228 ip_t new_ip;
1229 struct data *codegen;
1230 if (unlikely(load_relaxed(&da(fn,function)->codegen_failed)))
1231 goto cat(checkpoint_exit_,ARG_MODE);
1232 #if 1
1233 pointer_follow(&da(fn,function)->codegen, false, codegen, PF_WAIT, fp, orig_ip,
1234 RELOAD_EX_POSITION(ex_),
1235 store_relaxed(&da(fn,function)->codegen_failed, 1);
1236 goto cat(checkpoint_exit_,ARG_MODE);
1237 );
1238 #else
1239 pointer_follow(&da(fn,function)->codegen, false, codegen, PF_SPARK, NULL, 0,
1240 /*debug("sparked: %s %p", da(fn,function)->function_name, ex_);*/
1241 SUBMIT_EX(ex_);
1242 goto cat(checkpoint_exit_,ARG_MODE),
1243 store_relaxed(&da(fn,function)->codegen_failed, 1);
1244 goto cat(checkpoint_exit_,ARG_MODE);
1245 );
1246 #endif
1248 /*for (frame_t l = MIN_USEABLE_SLOT; l < function_n_variables(fn); l++) {
1249 if (da(fn,function)->local_variables_flags[l].must_be_flat) {
1250 if (unlikely(frame_test_flag(fp, l)))
1251 goto cat(checkpoint_exit_,ARG_MODE);
1252 }
1253 }*/
1254 /*for (frame_t l = MIN_USEABLE_SLOT; l < function_n_variables(fn); l++) {
1255 if (da(fn,function)->local_variables_flags[l].must_be_data) {
1256 if (unlikely(pointer_is_thunk(*frame_pointer(fp, l))))
1257 goto cat(checkpoint_exit_,ARG_MODE);
1258 }
1259 }*/
1261 /*debug("running optimized code: %s, %p, %x, %p, %p, %p, %u, %p, %lx", da(fn,function)->function_name, da(codegen,codegen)->unoptimized_code_base, id, da(codegen,codegen)->unoptimized_code[id], fp, &cg_upcall_vector, ts, da(codegen,codegen)->unoptimized_code[id], (unsigned long)((char *)da(codegen,codegen)->unoptimized_code[id] - (char *)da(codegen,codegen)->unoptimized_code_base));*/
1262 /*debug("fp[0] = %lx", *(long *)((char *)fp + 0));
1263 debug("fp[8] = %lx", *(long *)((char *)fp + 8));
1264 debug("fp[16] = %lx", *(long *)((char *)fp + 16));
1265 debug("fp[24] = %lx", *(long *)((char *)fp + 24));
1266 debug("fp[32] = %lx", *(long *)((char *)fp + 32));
1267 debug("fp[40] = %lx", *(long *)((char *)fp + 40));*/
1268 /*{
1269 uintptr_t *stub = (void *)da(codegen,codegen)->unoptimized_code[id];
1270 debug("entry: %p, %lx %lx %lx %lx", stub, stub[0], stub[1], stub[2], stub[3]);
1271 }*/
1272 /*__asm__ volatile("nopr %r6");*/
1273 /*__asm__ volatile("xnop");*/
1274 /*__asm__ volatile("cover");*/
1275 /*debug("calling: %p, %p, %lx, %lx", da(codegen,codegen)->unoptimized_code[id], codegen_entry, ((long *)codegen_entry)[0], ((long *)codegen_entry)[1]);*/
1276 /*debug("calling: %p, %p, %lx, %lx", da(codegen,codegen)->unoptimized_code[id], codegen_entry, ((long *)((char *)codegen_entry - 2))[0], ((long *)((char *)codegen_entry - 2))[1]);*/
1277 r = codegen_entry(fp, &cg_upcall_vector, ts, da(codegen,codegen)->unoptimized_code[id]);
1278 #if defined(ARCH_X86_32) || defined(ARCH_ARM32) || defined(ARCH_MIPS32) || defined(ARCH_POWER32) || defined(ARCH_SPARC32)
1279 #if defined(C_LITTLE_ENDIAN) || defined(ARCH_MIPS_N32)
1280 new_fp = num_to_ptr(r & 0xffffffffU);
1281 new_ip = r >> 32;
1282 #else
1283 new_fp = num_to_ptr(r >> 32);
1284 new_ip = r & 0xffffffffU;
1285 #endif
1286 #else
1287 new_fp = r.fp;
1288 new_ip = r.ip;
1289 #endif
1290 /*debug("ran optimized code: %s, %p, %lx -> %p, %x", da(fn,function)->function_name, fp, (unsigned long)(ip - da(fn,function)->code), new_fp, new_ip);*/
1291 if (unlikely(new_ip == (ip_t)-1)) {
1292 /*debug("reload on optimized code: %p", new_fp);*/
1293 RELOAD_EX_POSITION(new_fp);
1294 } else {
1295 /*if (fp != new_fp) debug("switching function: %s -> %s", da(fn,function)->function_name, da(get_frame(new_fp)->function,function)->function_name);*/
1296 fp = new_fp;
1297 ip = da(get_frame(fp)->function,function)->code + new_ip;
1298 if (unlikely(profiling_escapes)) {
1299 profile_counter_t profiling_counter;
1300 fn = get_frame(fp)->function;
1301 profiling_counter = load_relaxed(&da(fn,function)->escape_data[new_ip].counter);
1302 profiling_counter++;
1303 store_relaxed(&da(fn,function)->escape_data[new_ip].counter, profiling_counter);
1304 }
1305 }
1306 #endif
1307 goto cat(checkpoint_exit_,ARG_MODE);
1308 cat(checkpoint_exit_,ARG_MODE):;
1309 }
1310 #endif
1311 DEFINE_OPCODE_END(OPCODE_CHECKPOINT)
1314 #if ARG_MODE == 0
1315 DEFINE_OPCODE_START(OPCODE_JMP)
1316 #ifdef EMIT_CODE
1317 if (SIZEOF_IP_T == 2) {
1318 int16_t offset = ip[1];
1319 ADVANCE_IP(2);
1320 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1321 } else if (SIZEOF_IP_T == 4) {
1322 int32_t offset = get_unaligned_32(&ip[1]);
1323 ADVANCE_IP(3);
1324 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1325 } else {
1326 not_reached();
1327 }
1328 #endif
1329 DEFINE_OPCODE_END(OPCODE_JMP)
1330 #endif
1332 #if ARG_MODE == 0 && SIZEOF_IP_T > 2
1333 DEFINE_OPCODE_START(OPCODE_JMP_BACK_16)
1334 #ifdef EMIT_CODE
1335 uint16_t offset;
1337 offset = ip[1];
1338 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) - offset);
1339 #endif
1340 DEFINE_OPCODE_END(OPCODE_JMP_BACK_16)
1341 #endif
1343 DEFINE_OPCODE_START(OPCODE_JMP_FALSE)
1344 #ifdef EMIT_CODE
1345 int32_t offset;
1346 frame_t slot;
1347 ajla_flat_option_t val1;
1349 slot = get_i_param(0);
1350 if (unlikely(frame_test_flag(fp, slot))) {
1351 void *ex = thunk_bool_jump(fp, ip, slot);
1352 if (ex != POINTER_FOLLOW_THUNK_RETRY) {
1353 if (likely(ex != POINTER_FOLLOW_THUNK_EXCEPTION)) {
1354 RELOAD_EX_POSITION(ex);
1355 } else {
1356 if (SIZEOF_IP_T == 2) {
1357 offset = ip[1 + param_size(1) + 1];
1358 ADVANCE_I_PARAM(1);
1359 ADVANCE_IP(2);
1360 } else if (SIZEOF_IP_T == 4) {
1361 offset = get_unaligned_32(&ip[1 + param_size(1) + 2]);
1362 ADVANCE_I_PARAM(1);
1363 ADVANCE_IP(4);
1364 } else {
1365 not_reached();
1366 }
1367 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1368 goto cat(jmp_false_exception_,ARG_MODE);
1369 }
1370 }
1371 }
1372 barrier_aliasing();
1373 val1 = *frame_slot(fp, slot, ajla_flat_option_t);
1374 barrier_aliasing();
1375 if (SIZEOF_IP_T == 2) {
1376 offset = ip[1 + param_size(1)];
1377 ADVANCE_I_PARAM(1);
1378 ADVANCE_IP(2);
1379 } else if (SIZEOF_IP_T == 4) {
1380 offset = get_unaligned_32(&ip[1 + param_size(1)]);
1381 ADVANCE_I_PARAM(1);
1382 ADVANCE_IP(4);
1383 } else {
1384 not_reached();
1385 }
1386 if (!val1)
1387 ip = cast_ptr(const code_t *, cast_ptr(const char *, ip) + offset);
1389 cat(jmp_false_exception_,ARG_MODE):;
1390 #endif
1391 DEFINE_OPCODE_END(OPCODE_JMP_FALSE)
1393 #if ARG_MODE == 0
1394 DEFINE_OPCODE_START(OPCODE_LABEL)
1395 #ifdef EMIT_CODE
1396 ADVANCE_IP(1);
1397 #endif
1398 DEFINE_OPCODE_END(OPCODE_LABEL)
1399 #endif
1402 START_BLOCK(
1403 pointer_t *direct_data;
1404 pointer_t indirect_data;
1405 arg_t n_arguments;
1406 frame_t result_slot;
1407 struct thunk *function_error;
1408 )
1410 DEFINE_OPCODE_START(OPCODE_LOAD_FN)
1411 #ifdef EMIT_CODE
1412 struct data *new_reference;
1413 pointer_t result_ptr;
1415 n_arguments = get_i_param(0);
1416 result_slot = get_i_param(1);
1417 direct_data = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1418 indirect_data = pointer_empty(); /* avoid warning */
1420 ADVANCE_I_PARAM(3);
1422 cat(fn_copy_arguments_,ARG_MODE):
1424 new_reference = data_alloc_function_reference_mayfail(n_arguments, &ajla_error pass_file_line);
1425 if (unlikely(!new_reference)) {
1426 function_error = thunk_alloc_exception_error(ajla_error, NULL, fp, ip pass_file_line);
1427 if (!direct_data)
1428 pointer_dereference(indirect_data);
1429 /*cat(fn_set_error_,ARG_MODE):*/
1430 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_arguments);
1431 result_ptr = pointer_thunk(function_error);
1432 } else {
1433 arg_t i;
1434 if (direct_data) {
1435 da(new_reference,function_reference)->is_indirect = false;
1436 da(new_reference,function_reference)->u.direct = direct_data;
1437 } else {
1438 da(new_reference,function_reference)->is_indirect = true;
1439 da(new_reference,function_reference)->u.indirect = indirect_data;
1440 }
1441 for (i = 0; i < n_arguments; i++) {
1442 ipret_fill_function_reference_from_slot(new_reference, i, fp, get_param(ip, 0), (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1443 ADVANCE_IP(param_size(2));
1444 }
1445 result_ptr = pointer_data(new_reference);
1446 }
1448 frame_set_pointer(fp, result_slot, result_ptr);
1449 #endif
1450 DEFINE_OPCODE_END(OPCODE_LOAD_FN)
1452 DEFINE_OPCODE_START(OPCODE_CURRY)
1453 #ifdef EMIT_CODE
1454 frame_t fn_ref_slot;
1456 n_arguments = get_i_param(0);
1457 result_slot = get_i_param(1);
1459 fn_ref_slot = get_i_param(2);
1460 direct_data = NULL;
1461 indirect_data = frame_get_pointer_reference(fp, fn_ref_slot, (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1463 ADVANCE_I_PARAM(4);
1465 goto cat(fn_copy_arguments_,ARG_MODE);
1466 #endif
1467 DEFINE_OPCODE_END(OPCODE_CURRY)
1469 END_BLOCK()
1471 START_BLOCK(
1472 arg_t n_arguments;
1473 arg_t n_return_values;
1474 struct thunk *function_error;
1475 struct data *function;
1476 const code_t *orig_ip;
1477 frame_s *new_fp;
1478 arg_t i;
1479 pointer_t *direct_function;
1480 pointer_t indirect_function;
1481 unsigned char mode;
1482 )
1483 DEFINE_OPCODE_START(OPCODE_CALL)
1484 #ifdef EMIT_CODE
1485 pointer_t *function_ptr;
1487 mode = CALL_MODE_NORMAL;
1488 cat(call_from_mode_,ARG_MODE):
1490 n_arguments = get_i_param(0);
1491 n_return_values = get_i_param(1);
1492 function_ptr = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1494 pointer_follow(function_ptr, false, function, PF_WAIT, fp, ip,
1495 RELOAD_EX_POSITION(ex_),
1496 function_error = thunk_;
1497 thunk_reference(function_error);
1498 ADVANCE_I_PARAM(3);
1499 goto cat(call_set_error_,ARG_MODE);
1500 );
1502 ajla_assert(da(function,function)->n_arguments == n_arguments && da(function,function)->n_return_values == n_return_values,
1503 (file_line, "call %s->%s: the number of arguments does not match: %lu != %lu || %lu != %lu",
1504 da(get_frame(fp)->function,function)->function_name, da(function,function)->function_name,
1505 (unsigned long)da(function,function)->n_arguments, (unsigned long)n_arguments,
1506 (unsigned long)da(function,function)->n_return_values, (unsigned long)n_return_values));
1508 ADVANCE_I_PARAM(3);
1510 new_fp = frame_build(fp, function, &ajla_error);
1511 if (unlikely(!new_fp)) {
1512 cat(call_allocation_error_,ARG_MODE):
1513 function_error = thunk_alloc_exception_error(ajla_error, NULL, fp, ip pass_file_line);
1514 cat(call_set_error_,ARG_MODE):
1515 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_arguments);
1516 pointer_reference_owned_multiple(pointer_thunk(function_error), n_return_values - 1);
1517 i = n_return_values;
1518 while (1) {
1519 frame_t slot_r = get_max_param(ip, 0);
1520 ADVANCE_IP(max_param_size(1) + 1);
1521 frame_set_pointer(fp, slot_r, pointer_thunk(function_error));
1522 if (!--i)
1523 break;
1524 }
1525 } else {
1526 fp = frame_up(new_fp);
1527 frame_init(new_fp, function, get_frame(fp)->timestamp, mode);
1529 i = 0;
1530 cat(call_copy_arguments_,ARG_MODE):
1531 for (; i < da(function,function)->n_arguments; i++) {
1532 frame_t src_slot = get_param(ip, 0);
1533 frame_t dst_slot = da(function,function)->args[i].slot;
1534 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1535 bool may_be_borrowed = da(function,function)->args[i].may_be_borrowed;
1536 if (may_be_borrowed && src_flag & OPCODE_CALL_MAY_LEND && !pointer_is_thunk(*frame_pointer(fp, src_slot))) {
1537 *frame_pointer(new_fp, dst_slot) = *frame_pointer(fp, src_slot);
1538 } else if (may_be_borrowed && src_flag & OPCODE_CALL_MAY_GIVE && !frame_test_flag(fp, src_slot) && !pointer_is_thunk(*frame_pointer(fp, src_slot))) {
1539 *frame_pointer(new_fp, dst_slot) = *frame_pointer(fp, src_slot);
1540 *frame_pointer(fp, src_slot) = pointer_empty();
1541 } else {
1542 ipret_copy_variable(fp, src_slot, new_fp, dst_slot, (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1543 }
1544 ADVANCE_IP(param_size(2));
1545 }
1546 get_frame(new_fp)->previous_ip = frame_ip(fp, ip);
1548 fp = new_fp;
1549 ip = &da(function,function)->code[0];
1551 if (unlikely(mode == CALL_MODE_SPARK)) {
1552 ipret_checkpoint_forced;
1553 }
1554 }
1555 #endif
1556 DEFINE_OPCODE_END(OPCODE_CALL)
1558 DEFINE_OPCODE_START(OPCODE_CALL_STRICT)
1559 #ifdef EMIT_CODE
1560 mode = CALL_MODE_STRICT;
1561 goto cat(call_from_mode_,ARG_MODE);
1562 #endif
1563 DEFINE_OPCODE_END(OPCODE_CALL_STRICT)
1565 DEFINE_OPCODE_START(OPCODE_CALL_SPARK)
1566 #ifdef EMIT_CODE
1567 mode = CALL_MODE_SPARK;
1568 goto cat(call_from_mode_,ARG_MODE);
1569 #endif
1570 DEFINE_OPCODE_END(OPCODE_CALL_SPARK)
1572 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT)
1573 #ifdef EMIT_CODE
1574 frame_t fn_ref_slot;
1575 bool deref;
1576 pointer_t *ptr;
1578 mode = CALL_MODE_NORMAL;
1579 cat(call_indirect_from_mode_,ARG_MODE):
1581 orig_ip = ip;
1582 n_arguments = get_i_param(0);
1583 n_return_values = get_i_param(1);
1584 fn_ref_slot = get_i_param(2);
1585 deref = (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1586 ADVANCE_I_PARAM(4);
1588 ptr = frame_pointer(fp, fn_ref_slot);
1589 while (1) {
1590 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1591 RELOAD_EX_POSITION(ex_),
1592 function_error = thunk_;
1593 thunk_reference(function_error);
1594 if (deref)
1595 frame_free_and_clear(fp, fn_ref_slot);
1596 goto cat(call_set_error_,ARG_MODE)
1597 );
1598 if (!da(function,function_reference)->is_indirect)
1599 break;
1600 ptr = &da(function,function_reference)->u.indirect;
1601 }
1602 ptr = da(function,function_reference)->u.direct;
1603 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1604 RELOAD_EX_POSITION(ex_),
1605 function_error = thunk_;
1606 thunk_reference(function_error);
1607 if (deref)
1608 frame_free_and_clear(fp, fn_ref_slot);
1609 goto cat(call_set_error_,ARG_MODE)
1610 );
1612 new_fp = frame_build(fp, function, &ajla_error);
1613 if (unlikely(!new_fp)) {
1614 if (deref)
1615 frame_free_and_clear(fp, fn_ref_slot);
1616 goto cat(call_allocation_error_,ARG_MODE);
1617 } else {
1618 fp = frame_up(new_fp);
1619 frame_init(new_fp, function, get_frame(fp)->timestamp, mode);
1621 i = da(function,function)->n_arguments - n_arguments;
1623 copy_from_function_reference_to_frame(new_fp, pointer_get_data(*frame_pointer(fp, fn_ref_slot)), i, deref && frame_test_flag(fp, fn_ref_slot));
1624 if (deref)
1625 frame_free_and_clear(fp, fn_ref_slot);
1627 goto cat(call_copy_arguments_,ARG_MODE);
1628 }
1629 #endif
1630 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT)
1632 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_STRICT)
1633 #ifdef EMIT_CODE
1634 mode = CALL_MODE_STRICT;
1635 goto cat(call_indirect_from_mode_,ARG_MODE);
1636 #endif
1637 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_STRICT)
1639 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_SPARK)
1640 #ifdef EMIT_CODE
1641 mode = CALL_MODE_SPARK;
1642 goto cat(call_indirect_from_mode_,ARG_MODE);
1643 #endif
1644 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_SPARK)
1646 DEFINE_OPCODE_START(OPCODE_CALL_LAZY)
1647 #ifdef EMIT_CODE
1648 struct data *function_reference;
1649 struct thunk **results;
1651 n_arguments = get_i_param(0);
1652 n_return_values = get_i_param(1);
1653 direct_function = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1654 indirect_function = pointer_empty(); /* avoid warning */
1656 ADVANCE_I_PARAM(3);
1658 cat(call_lazy_copy_arguments_,ARG_MODE):
1659 results = mem_alloc_array_mayfail(mem_alloc_mayfail, struct thunk **, 0, 0, n_return_values, sizeof(struct thunk *), &ajla_error);
1660 if (unlikely(!results)) {
1661 if (!direct_function)
1662 pointer_dereference(indirect_function);
1663 goto cat(call_allocation_error_,ARG_MODE);
1664 }
1666 if (!(function_reference = data_alloc_function_reference_mayfail(n_arguments, &ajla_error pass_file_line))) {
1667 if (!direct_function)
1668 pointer_dereference(indirect_function);
1669 mem_free(results);
1670 goto cat(call_allocation_error_,ARG_MODE);
1671 }
1672 if (direct_function) {
1673 da(function_reference,function_reference)->is_indirect = false;
1674 da(function_reference,function_reference)->u.direct = direct_function;
1675 } else {
1676 da(function_reference,function_reference)->is_indirect = true;
1677 da(function_reference,function_reference)->u.indirect = indirect_function;
1678 }
1680 if (!thunk_alloc_function_call(pointer_data(function_reference), n_return_values, results, &ajla_error)) {
1681 if (!direct_function)
1682 pointer_dereference(indirect_function);
1683 mem_free(results);
1684 data_free_r1(function_reference);
1685 goto cat(call_allocation_error_,ARG_MODE);
1686 }
1688 for (i = 0; i < n_arguments; i++) {
1689 frame_t src_slot = get_param(ip, 0);
1690 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1691 ipret_fill_function_reference_from_slot(function_reference, i, fp, src_slot, (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1692 ADVANCE_IP(param_size(2));
1693 }
1695 for (i = 0; i < n_return_values; i++) {
1696 frame_t slot_r = get_max_param(ip, 0);
1697 frame_set_pointer(fp, slot_r, pointer_thunk(results[i]));
1698 ADVANCE_IP(max_param_size(1) + 1);
1699 }
1701 mem_free(results);
1702 #endif
1703 DEFINE_OPCODE_END(OPCODE_CALL_LAZY)
1705 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_LAZY)
1706 #ifdef EMIT_CODE
1707 frame_t fn_ref_slot;
1708 bool deref;
1710 n_arguments = get_i_param(0);
1711 n_return_values = get_i_param(1);
1712 fn_ref_slot = get_i_param(2);
1713 deref = (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1714 ADVANCE_I_PARAM(4);
1716 indirect_function = frame_get_pointer_reference(fp, fn_ref_slot, deref);
1717 direct_function = NULL;
1719 goto cat(call_lazy_copy_arguments_,ARG_MODE);
1720 #endif
1721 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_LAZY)
1723 DEFINE_OPCODE_START(OPCODE_CALL_CACHE)
1724 #ifdef EMIT_CODE
1725 struct ipret_call_cache_arg *arguments;
1726 frame_t *return_values;
1727 void *ex;
1729 cat(call_cache_,ARG_MODE):
1730 orig_ip = ip;
1731 n_arguments = get_i_param(0);
1732 n_return_values = get_i_param(1);
1733 direct_function = da(get_frame(fp)->function,function)->local_directory[get_i_param(2)];
1734 ADVANCE_I_PARAM(3);
1736 pointer_follow(direct_function, false, function, PF_WAIT, fp, orig_ip,
1737 RELOAD_EX_POSITION(ex_),
1738 function_error = thunk_;
1739 thunk_reference(function_error);
1740 goto cat(call_set_error_,ARG_MODE);
1741 );
1742 ajla_assert(da(function,function)->n_arguments == n_arguments && da(function,function)->n_return_values == n_return_values,
1743 (file_line, "call_cache: the number of arguments does not match: %lu != %lu || %lu != %lu",
1744 (unsigned long)da(function,function)->n_arguments, (unsigned long)n_arguments,
1745 (unsigned long)da(function,function)->n_return_values, (unsigned long)n_return_values));
1747 arguments = mem_alloc_array_mayfail(mem_alloc_mayfail, struct ipret_call_cache_arg *, 0, 0, n_arguments, sizeof(struct ipret_call_cache_arg), &ajla_error);
1748 if (!arguments) {
1749 goto cat(call_allocation_error_,ARG_MODE);
1750 }
1751 return_values = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n_return_values, sizeof(frame_t), &ajla_error);
1752 if (!return_values) {
1753 mem_free(arguments);
1754 goto cat(call_allocation_error_,ARG_MODE);
1755 }
1757 for (i = 0; i < n_arguments; i++) {
1758 frame_t src_slot = get_param(ip, 0);
1759 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1760 arguments[i].f_arg = NULL;
1761 arguments[i].slot = src_slot;
1762 arguments[i].deref = (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1763 ADVANCE_IP(param_size(2));
1764 }
1766 for (i = 0; i < n_return_values; i++) {
1767 frame_t slot_r = get_max_param(ip, 0);
1768 return_values[i] = slot_r;
1769 ADVANCE_IP(max_param_size(1) + 1);
1770 }
1772 ex = ipret_call_cache(fp, orig_ip, direct_function, arguments, return_values, NO_FRAME_T);
1773 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
1774 RELOAD_EX_POSITION(ex);
1775 }
1776 #endif
1777 DEFINE_OPCODE_END(OPCODE_CALL_CACHE)
1779 DEFINE_OPCODE_START(OPCODE_CALL_SAVE)
1780 #ifdef EMIT_CODE
1781 goto cat(call_cache_,ARG_MODE);
1782 #endif
1783 DEFINE_OPCODE_END(OPCODE_CALL_SAVE)
1785 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_CACHE)
1786 #ifdef EMIT_CODE
1787 struct ipret_call_cache_arg *arguments;
1788 frame_t *return_values;
1789 void *ex;
1791 frame_t fn_ref_slot;
1792 bool deref;
1793 pointer_t *ptr;
1794 arg_t n_curried_arguments;
1796 cat(call_indirect_cache_,ARG_MODE):
1797 orig_ip = ip;
1798 n_arguments = get_i_param(0);
1799 n_return_values = get_i_param(1);
1800 fn_ref_slot = get_i_param(2);
1801 deref = (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1802 ADVANCE_I_PARAM(4);
1804 n_curried_arguments = 0;
1805 ptr = frame_pointer(fp, fn_ref_slot);
1806 while (1) {
1807 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1808 RELOAD_EX_POSITION(ex_),
1809 function_error = thunk_;
1810 thunk_reference(function_error);
1811 if (deref)
1812 frame_free_and_clear(fp, fn_ref_slot);
1813 goto cat(call_set_error_,ARG_MODE)
1814 );
1815 n_curried_arguments += da(function,function_reference)->n_curried_arguments;
1816 if (!da(function,function_reference)->is_indirect)
1817 break;
1818 ptr = &da(function,function_reference)->u.indirect;
1819 }
1820 ptr = da(function,function_reference)->u.direct;
1821 pointer_follow(ptr, false, function, PF_WAIT, fp, orig_ip,
1822 RELOAD_EX_POSITION(ex_),
1823 function_error = thunk_;
1824 thunk_reference(function_error);
1825 if (deref)
1826 frame_free_and_clear(fp, fn_ref_slot);
1827 goto cat(call_set_error_,ARG_MODE)
1828 );
1830 arguments = mem_alloc_array_mayfail(mem_alloc_mayfail, struct ipret_call_cache_arg *, 0, 0, n_curried_arguments + n_arguments, sizeof(struct ipret_call_cache_arg), &ajla_error);
1831 if (!arguments) {
1832 if (deref)
1833 frame_free_and_clear(fp, fn_ref_slot);
1834 goto cat(call_allocation_error_,ARG_MODE);
1835 }
1836 return_values = mem_alloc_array_mayfail(mem_alloc_mayfail, frame_t *, 0, 0, n_return_values, sizeof(frame_t), &ajla_error);
1837 if (!return_values) {
1838 mem_free(arguments);
1839 if (deref)
1840 frame_free_and_clear(fp, fn_ref_slot);
1841 goto cat(call_allocation_error_,ARG_MODE);
1842 }
1844 i = n_curried_arguments;
1845 function = pointer_get_data(*frame_pointer(fp, fn_ref_slot));
1846 while (1) {
1847 arg_t j;
1848 j = da(function,function_reference)->n_curried_arguments;
1849 while (j--) {
1850 i--;
1851 arguments[i].f_arg = &da(function,function_reference)->arguments[j];
1852 arguments[i].deref = false;
1853 }
1854 if (!da(function,function_reference)->is_indirect)
1855 break;
1856 function = pointer_get_data(da(function,function_reference)->u.indirect);
1857 }
1859 for (i = n_curried_arguments; i < n_curried_arguments + n_arguments; i++) {
1860 frame_t src_slot = get_param(ip, 0);
1861 unsigned char src_flag = (unsigned char)get_param(ip, 1);
1862 arguments[i].f_arg = NULL;
1863 arguments[i].slot = src_slot;
1864 arguments[i].deref = (src_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0;
1865 ADVANCE_IP(param_size(2));
1866 }
1868 for (i = 0; i < n_return_values; i++) {
1869 frame_t slot_r = get_max_param(ip, 0);
1870 return_values[i] = slot_r;
1871 ADVANCE_IP(max_param_size(1) + 1);
1872 }
1874 ex = ipret_call_cache(fp, orig_ip, ptr, arguments, return_values, deref ? fn_ref_slot : NO_FRAME_T);
1875 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
1876 RELOAD_EX_POSITION(ex);
1877 }
1878 #endif
1879 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_CACHE)
1881 DEFINE_OPCODE_START(OPCODE_CALL_INDIRECT_SAVE)
1882 #ifdef EMIT_CODE
1883 goto cat(call_indirect_cache_,ARG_MODE);
1884 #endif
1885 DEFINE_OPCODE_END(OPCODE_CALL_INDIRECT_SAVE)
1887 END_BLOCK()
1889 DEFINE_OPCODE_START(OPCODE_RETURN)
1890 #ifdef EMIT_CODE
1891 struct data *function;
1892 frame_s *previous_fp;
1893 const code_t *previous_ip;
1894 arg_t i, n;
1896 /*ADVANCE_I_PARAM(0);*/
1898 function = get_frame(fp)->function;
1899 previous_fp = frame_up(fp);
1900 if (unlikely(frame_is_top(previous_fp))) {
1901 struct execution_control *ex;
1902 void *exx;
1903 struct thunk *t;
1904 struct stack_bottom *sb = frame_stack_bottom(previous_fp);
1905 ex = sb->ex;
1906 if (!ex) {
1907 frame_t src_slot = get_i_param(0);
1908 sb->ret = ipret_copy_variable_to_pointer(fp, src_slot, true);
1909 goto exit_ipret;
1910 }
1911 t = ex->thunk;
1912 n = da(function,function)->n_return_values;
1913 i = 0;
1914 do {
1915 frame_t src_slot = get_i_param(i * 2);
1916 frame_t flags = get_i_param(i * 2 + 1);
1917 t->u.function_call.results[i].ptr = ipret_copy_variable_to_pointer(fp, src_slot, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1918 } while (++i < n);
1919 exx = thunk_terminate(t, n);
1920 #if 0
1921 if (exx != POINTER_FOLLOW_THUNK_EXIT)
1922 task_submit(exx, true);
1923 goto exit_ipret;
1924 #else
1925 RELOAD_EX_POSITION(exx);
1926 #endif
1927 }
1929 get_frame(previous_fp)->timestamp = get_frame(fp)->timestamp;
1930 previous_ip = &da(get_frame(previous_fp)->function,function)->code[get_frame(fp)->previous_ip];
1932 n = da(function,function)->n_return_values;
1933 i = 0;
1934 do {
1935 frame_t src_slot, dst_slot, flags;
1936 src_slot = get_i_param(i * 2);
1937 flags = get_i_param(i * 2 + 1);
1938 dst_slot = (frame_t)get_max_param(previous_ip, 0);
1939 previous_ip += max_param_size(1) + 1;
1940 ipret_copy_variable(fp, src_slot, previous_fp, dst_slot, (flags & OPCODE_FLAG_FREE_ARGUMENT) != 0);
1941 } while (++i < n);
1943 fp = previous_fp;
1944 ip = previous_ip;
1945 #endif
1946 DEFINE_OPCODE_END(OPCODE_RETURN)
1949 DEFINE_OPCODE_START(OPCODE_STRUCTURED)
1950 #ifdef EMIT_CODE
1951 const code_t *orig_ip = ip;
1953 frame_t struc, elem;
1954 pointer_t *struc_ptr;
1955 unsigned char *struc_flat;
1956 const struct type *t;
1957 bool optimize_elem_is_flat;
1959 unsigned char type;
1960 frame_t arg;
1962 pointer_t thunk_ptr;
1964 elem = get_i_param(1);
1965 optimize_elem_is_flat = frame_variable_is_flat(fp, elem);
1967 cat(structured_retry_,ARG_MODE):
1968 struc = get_i_param(0);
1969 ADVANCE_I_PARAM(2);
1971 t = frame_get_type_of_local(fp, struc);
1972 if (frame_variable_is_flat(fp, struc)) {
1973 if (!optimize_elem_is_flat) {
1974 frame_set_pointer(fp, struc, flat_to_data(frame_get_type_of_local(fp, struc), frame_var(fp, struc)));
1975 struc_flat = NULL;
1976 struc_ptr = frame_pointer(fp, struc);
1977 } else {
1978 struc_flat = frame_var(fp, struc);
1979 struc_ptr = NULL; /* avoid warning */
1980 }
1981 } else {
1982 struc_flat = NULL;
1983 struc_ptr = frame_pointer(fp, struc);
1984 }
1986 do {
1987 array_index_t array_index = index_invalid(); /* avoid warning */
1988 struct data *data = NULL; /* avoid warning */
1990 type = (unsigned char)get_param(ip, 0);
1991 arg = get_param(ip, 1);
1992 ADVANCE_IP(param_size(3));
1994 if ((type & OPCODE_STRUCTURED_MASK) == OPCODE_STRUCTURED_ARRAY) {
1995 void *ex = ipret_get_index(fp, orig_ip, fp, arg, NULL, &array_index, &thunk_ptr pass_file_line);
1996 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
1997 if (ex == POINTER_FOLLOW_THUNK_EXCEPTION)
1998 goto cat(structured_error_,ARG_MODE);
1999 RELOAD_EX_POSITION(ex);
2000 }
2001 }
2003 if (struc_flat) {
2004 switch (type & OPCODE_STRUCTURED_MASK) {
2005 case OPCODE_STRUCTURED_RECORD: {
2006 const struct flat_record_definition_entry *e;
2007 ajla_assert(arg < flat_record_n_slots(type_def(t,flat_record)), (file_line, "structured flat record: invalid index: %"PRIuMAX" >= %"PRIuMAX"", (uintmax_t)arg, (uintmax_t)flat_record_n_slots(type_def(t,flat_record))));
2008 e = &type_def(t,flat_record)->entries[arg];
2009 t = e->subtype;
2010 struc_flat += e->flat_offset;
2011 break;
2012 }
2013 case OPCODE_STRUCTURED_OPTION: {
2014 thunk_ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_OPTION_DOESNT_MATCH), fp, ip pass_file_line);
2015 goto cat(structured_error_,ARG_MODE);
2016 }
2017 case OPCODE_STRUCTURED_ARRAY: {
2018 if (unlikely(index_ge_int(array_index, type_def(t,flat_array)->n_elements))) {
2019 index_free(&array_index);
2020 thunk_ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_INDEX_OUT_OF_RANGE), fp, ip pass_file_line);
2021 goto cat(structured_error_,ARG_MODE);
2022 }
2023 t = type_def(t,flat_array)->base;
2024 struc_flat += t->size * index_to_int(array_index);
2025 index_free(&array_index);
2026 break;
2027 }
2028 default:
2029 internal(file_line, "structured flat: invalid structure type %02x", type);
2030 }
2031 } else {
2032 cat(structured_restart_pointer_follow_,ARG_MODE):
2033 pointer_follow(struc_ptr, false, data, PF_WAIT, fp, orig_ip,
2034 if ((type & OPCODE_STRUCTURED_MASK) == OPCODE_STRUCTURED_ARRAY)
2035 index_free(&array_index);
2036 RELOAD_EX_POSITION(ex_),
2037 if ((type & OPCODE_STRUCTURED_MASK) == OPCODE_STRUCTURED_ARRAY)
2038 index_free(&array_index);
2039 goto cat(structured_read_to_the_end_,ARG_MODE);
2040 );
2041 if (unlikely(!data_is_writable(data))) {
2042 struct_clone(struc_ptr);
2043 goto cat(structured_restart_pointer_follow_,ARG_MODE);
2044 }
2045 switch (type & OPCODE_STRUCTURED_MASK) {
2046 case OPCODE_STRUCTURED_RECORD: {
2047 const struct record_definition *def = type_def(da(data,record)->definition,record);
2048 ajla_assert(arg < def->n_slots, (file_line, "structured record: invalid index: %"PRIuMAX" >= %"PRIuMAX"", (uintmax_t)arg, (uintmax_t)def->n_slots));
2049 t = def->types[arg];
2050 struc_ptr = frame_pointer(da_record_frame(data), arg);
2052 if (type & OPCODE_STRUCTURED_FLAG_END) {
2053 if (frame_variable_is_flat(fp, elem)) {
2054 if (TYPE_IS_FLAT(t)) {
2055 frame_free(da_record_frame(data), arg);
2056 struc_flat = frame_var(da_record_frame(data), arg);
2057 goto cat(structured_write_flat_,ARG_MODE);
2058 }
2059 } else {
2060 if (!frame_test_and_set_flag(da_record_frame(data), arg))
2061 goto cat(structured_write_ptr_,ARG_MODE);
2062 }
2063 }
2065 if (!frame_test_flag(da_record_frame(data), arg)) {
2066 struc_flat = frame_var(da_record_frame(data), arg);
2067 if (unlikely(!optimize_elem_is_flat)) {
2068 frame_set_pointer(da_record_frame(data), arg, flat_to_data(t, struc_flat));
2069 struc_flat = NULL;
2070 }
2071 }
2072 break;
2073 }
2074 case OPCODE_STRUCTURED_OPTION: {
2075 if (unlikely(da(data,option)->option != arg)) {
2076 if (likely((type & OPCODE_STRUCTURED_FLAG_END) != 0)) {
2077 da(data,option)->option = arg;
2078 } else {
2079 thunk_ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_OPTION_DOESNT_MATCH), fp, ip pass_file_line);
2080 cat(structured_error_,ARG_MODE):
2081 if (struc_flat) {
2082 pointer_dereference(thunk_ptr);
2083 optimize_elem_is_flat = false;
2084 ip = orig_ip;
2085 goto cat(structured_retry_,ARG_MODE);
2086 } else {
2087 pointer_dereference(*struc_ptr);
2088 *struc_ptr = thunk_ptr;
2089 goto cat(structured_read_to_the_end_,ARG_MODE);
2090 }
2091 }
2092 }
2093 struc_ptr = &da(data,option)->pointer;
2094 break;
2095 }
2096 case OPCODE_STRUCTURED_ARRAY: {
2097 unsigned array_flags;
2098 if (unlikely(da_tag(data) == DATA_TAG_array_incomplete)) {
2099 array_index_t len_first;
2101 if (array_incomplete_collapse(struc_ptr))
2102 goto cat(structured_restart_pointer_follow_,ARG_MODE);
2104 len_first = array_len(pointer_get_data(da(data,array_incomplete)->first));
2105 if (!index_ge_index(array_index, len_first)) {
2106 index_free(&len_first);
2107 struc_ptr = &da(data,array_incomplete)->first;
2108 goto cat(structured_restart_pointer_follow_,ARG_MODE);
2109 } else {
2110 index_sub(&array_index, len_first);
2111 index_free(&len_first);
2112 struc_ptr = &da(data,array_incomplete)->next;
2113 goto cat(structured_restart_pointer_follow_,ARG_MODE);
2114 }
2115 }
2116 if (!optimize_elem_is_flat) {
2117 array_flags = ARRAY_MODIFY_NEED_PTR;
2118 } else if (type & OPCODE_STRUCTURED_FLAG_END) {
2119 array_flags = ARRAY_MODIFY_NEED_FLAT;
2120 t = frame_get_type_of_local(fp, elem);
2121 } else {
2122 array_flags = 0;
2123 }
2124 if (unlikely(!array_modify(struc_ptr, array_index, array_flags, &struc_ptr, &struc_flat, &t, fp, ip))) {
2125 goto cat(structured_read_to_the_end_,ARG_MODE);
2126 }
2127 break;
2128 }
2129 default:
2130 internal(file_line, "structured: invalid structure type %02x", type);
2131 }
2132 }
2133 } while (!(type & OPCODE_STRUCTURED_FLAG_END));
2134 if (struc_flat) {
2135 cat(structured_write_flat_,ARG_MODE):
2136 memcpy_fast(struc_flat, frame_var(fp, elem), frame_get_type_of_local(fp, elem)->size);
2137 } else {
2138 /* The pointer may be empty if we moved from an option without data to an option with data */
2139 if (likely(!pointer_is_empty(*struc_ptr)))
2140 pointer_dereference(*struc_ptr);
2141 if (frame_variable_is_flat(fp, elem)) {
2142 *struc_ptr = flat_to_data(frame_get_type_of_local(fp, elem), frame_var(fp, elem));
2143 } else {
2144 cat(structured_write_ptr_,ARG_MODE):
2145 *struc_ptr = frame_get_pointer_reference(fp, elem, (type & OPCODE_STRUCTURED_FREE_VARIABLE) != 0);
2146 }
2147 }
2148 if (false) {
2149 do {
2150 type = (unsigned char)get_param(ip, 0);
2151 arg = get_param(ip, 1);
2152 ADVANCE_IP(param_size(3));
2153 cat(structured_read_to_the_end_,ARG_MODE):;
2154 } while (!(type & OPCODE_STRUCTURED_FLAG_END));
2155 if (type & OPCODE_STRUCTURED_FREE_VARIABLE)
2156 frame_free(fp, elem);
2157 }
2158 #endif
2159 DEFINE_OPCODE_END(OPCODE_STRUCTURED)
2161 DEFINE_OPCODE_START(OPCODE_RECORD_CREATE)
2162 #ifdef EMIT_CODE
2163 frame_t result_slot;
2164 arg_t n_entries, i, ii;
2165 const struct type *t;
2166 const struct record_definition *def;
2168 result_slot = get_i_param(0);
2169 n_entries = get_i_param(1);
2170 ADVANCE_I_PARAM(2);
2172 t = frame_get_type_of_local(fp, result_slot);
2173 if (t->tag == TYPE_TAG_flat_record) {
2174 const code_t *backup_ip;
2175 def = type_def(type_def(t,flat_record)->base,record);
2176 backup_ip = ip;
2177 for (i = 0, ii = 0; i < n_entries; i++, ii++) {
2178 frame_t var_slot, record_slot;
2179 flat_size_t flat_offset;
2180 const struct type *flat_type;
2182 while (unlikely(record_definition_is_elided(def, ii)))
2183 ii++;
2185 var_slot = get_param(ip, 0);
2187 if (unlikely(!frame_variable_is_flat(fp, var_slot))) {
2188 ip = backup_ip;
2189 goto cat(create_record_no_flat_,ARG_MODE);
2190 }
2191 record_slot = record_definition_slot(def, ii);
2192 flat_offset = type_def(t,flat_record)->entries[record_slot].flat_offset;
2193 flat_type = type_def(t,flat_record)->entries[record_slot].subtype;
2194 ajla_assert(type_is_equal(frame_get_type_of_local(fp, var_slot), flat_type), (file_line, "record create (flat): copying between different types (%u,%u,%u) -> (%u,%u,%u)", frame_get_type_of_local(fp, var_slot)->tag, frame_get_type_of_local(fp, var_slot)->size, frame_get_type_of_local(fp, var_slot)->align, flat_type->tag, flat_type->size, flat_type->align));
2195 memcpy_fast(frame_var(fp, result_slot) + flat_offset, frame_var(fp, var_slot), flat_type->size);
2196 ADVANCE_IP(param_size(2));
2197 }
2198 ajla_assert(!frame_test_flag(fp, result_slot), (file_line, "record create (flat): flag already set for destination slot %"PRIuMAX"", (uintmax_t)result_slot));
2199 } else {
2200 struct data *result;
2201 def = type_def(t,record);
2202 cat(create_record_no_flat_,ARG_MODE):
2203 result = data_alloc_record_mayfail(def, &ajla_error pass_file_line);
2204 if (unlikely(!result)) {
2205 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_entries);
2206 frame_set_pointer(fp, result_slot, pointer_error(ajla_error, fp, ip pass_file_line));
2207 break;
2208 }
2209 memset(da_record_frame(result), 0, bitmap_slots(def->n_slots) * slot_size);
2210 for (i = 0, ii = 0; i < n_entries; i++, ii++) {
2211 frame_t var_slot, record_slot;
2212 const struct type *rec_type, *var_type;
2214 while (unlikely(record_definition_is_elided(def, ii)))
2215 ii++;
2217 var_slot = get_param(ip, 0);
2218 record_slot = record_definition_slot(def, ii);
2219 rec_type = def->types[record_slot];
2220 var_type = frame_get_type_of_local(fp, var_slot);
2222 if (!frame_variable_is_flat(fp, var_slot)) {
2223 pointer_t ptr = frame_get_pointer_reference(fp, var_slot, (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2224 frame_set_pointer(da_record_frame(result), record_slot, ptr);
2225 } else if (TYPE_IS_FLAT(rec_type)) {
2226 ajla_assert(type_is_equal(var_type, rec_type), (file_line, "record create: copying between different types (%u,%u,%u) -> (%u,%u,%u)", var_type->tag, var_type->size, var_type->align, rec_type->tag, rec_type->size, rec_type->align));
2227 memcpy_fast(frame_var(da_record_frame(result), record_slot), frame_var(fp, var_slot), rec_type->size);
2228 } else {
2229 pointer_t ptr = flat_to_data(var_type, frame_var(fp, var_slot));
2230 frame_set_pointer(da_record_frame(result), record_slot, ptr);
2231 }
2233 ADVANCE_IP(param_size(2));
2234 }
2235 frame_set_pointer(fp, result_slot, pointer_data(result));
2236 }
2237 #endif
2238 DEFINE_OPCODE_END(OPCODE_RECORD_CREATE)
2240 DEFINE_OPCODE_START(OPCODE_RECORD_LOAD)
2241 #ifdef EMIT_CODE
2242 frame_t record, record_slot, result;
2243 unsigned flags;
2244 const struct type *t;
2245 record = get_i_param(0);
2246 record_slot = get_i_param(1);
2247 result = get_i_param(2);
2248 flags = get_i_param(3);
2250 t = frame_get_type_of_local(fp, record);
2251 if (t->tag == TYPE_TAG_flat_record && !frame_test_flag(fp, record)) {
2252 const struct flat_record_definition_entry *ft = &type_def(t,flat_record)->entries[record_slot];
2253 if (likely(TYPE_IS_FLAT(frame_get_type_of_local(fp, result)))) {
2254 memcpy_fast(frame_var(fp, result), frame_var(fp, record) + ft->flat_offset, ft->subtype->size);
2255 } else {
2256 pointer_t ptr = flat_to_data(ft->subtype, frame_var(fp, record) + ft->flat_offset);
2257 frame_set_pointer(fp, result, ptr);
2258 }
2259 } else {
2260 const struct type *rtype, *etype;
2261 struct data *data;
2262 frame_s *fr;
2263 pointer_follow(frame_pointer(fp, record), true, data, unlikely(flags & OPCODE_OP_FLAG_STRICT) ? PF_WAIT : PF_NOEVAL, fp, ip,
2264 if (!(flags & OPCODE_OP_FLAG_STRICT)) {
2265 ex_ = ipret_record_load_create_thunk(fp, ip, record, record_slot, result);
2266 if (ex_ == POINTER_FOLLOW_THUNK_GO)
2267 goto cat(record_load_end_,ARG_MODE);
2268 }
2269 RELOAD_EX_POSITION(ex_),
2270 thunk_reference(thunk_);
2271 frame_set_pointer(fp, result, pointer_thunk(thunk_));
2272 goto cat(record_load_end_,ARG_MODE);
2273 );
2274 rtype = da(data,record)->definition;
2275 etype = type_def(rtype,record)->types[record_slot];
2276 fr = da_record_frame(data);
2277 if (!frame_test_flag(fr, record_slot)) {
2278 if (likely(TYPE_IS_FLAT(frame_get_type_of_local(fp, result)))) {
2279 memcpy_fast(frame_var(fp, result), frame_var(fr, record_slot), etype->size);
2280 } else {
2281 pointer_t ptr = flat_to_data(etype, frame_var(fr, record_slot));
2282 frame_set_pointer(fp, result, ptr);
2283 }
2284 } else {
2285 pointer_reference_maybe(fp, result, frame_pointer(fr, record_slot), flags);
2286 }
2287 }
2289 cat(record_load_end_,ARG_MODE):
2290 ADVANCE_I_PARAM(4);
2291 #endif
2292 DEFINE_OPCODE_END(OPCODE_RECORD_CREATE)
2294 DEFINE_OPCODE_START(OPCODE_OPTION_CREATE_EMPTY_FLAT)
2295 #ifdef EMIT_CODE
2296 frame_t slot_r = get_i_param(0);
2297 ajla_flat_option_t opt = (ajla_flat_option_t)get_i_param(1);
2298 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "option create empty flat: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
2299 barrier_aliasing();
2300 *frame_slot(fp, slot_r, ajla_flat_option_t) = opt;
2301 barrier_aliasing();
2302 ADVANCE_I_PARAM(2);
2303 #endif
2304 DEFINE_OPCODE_END(OPCODE_OPTION_CREATE_EMPTY_FLAT)
2306 DEFINE_OPCODE_START(OPCODE_OPTION_CREATE_EMPTY)
2307 #ifdef EMIT_CODE
2308 struct data *d;
2309 frame_t slot_r;
2311 d = data_alloc(option, &ajla_error);
2312 slot_r = get_i_param(0);
2313 if (likely(d != NULL)) {
2314 da(d,option)->option = get_i_param(1);
2315 da(d,option)->pointer = pointer_empty();
2316 frame_set_pointer(fp, slot_r, pointer_data(d));
2317 } else {
2318 frame_set_pointer(fp, slot_r, pointer_error(ajla_error, fp, ip pass_file_line));
2319 }
2320 ADVANCE_I_PARAM(2);
2321 #endif
2322 DEFINE_OPCODE_END(OPCODE_OPTION_CREATE_EMPTY)
2324 DEFINE_OPCODE_START(OPCODE_OPTION_CREATE)
2325 #ifdef EMIT_CODE
2326 struct data *d;
2327 frame_t slot_1, slot_r;
2328 pointer_t ptr;
2330 slot_1 = get_i_param(2);
2331 if (!frame_variable_is_flat(fp, slot_1)) {
2332 ptr = frame_get_pointer_reference(fp, slot_1, (get_i_param(3) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2333 } else {
2334 ptr = flat_to_data(frame_get_type_of_local(fp, slot_1), frame_var(fp, slot_1));
2335 }
2337 d = data_alloc(option, &ajla_error);
2338 slot_r = get_i_param(0);
2339 if (likely(d != NULL)) {
2340 da(d,option)->option = get_i_param(1);
2341 da(d,option)->pointer = ptr;
2342 frame_set_pointer(fp, slot_r, pointer_data(d));
2343 } else {
2344 pointer_dereference(ptr);
2345 frame_set_pointer(fp, slot_r, pointer_error(ajla_error, fp, ip pass_file_line));
2346 }
2347 ADVANCE_I_PARAM(4);
2348 #endif
2349 DEFINE_OPCODE_END(OPCODE_OPTION_CREATE)
2351 DEFINE_OPCODE_START(OPCODE_OPTION_LOAD)
2352 #ifdef EMIT_CODE
2353 frame_t option, option_idx, result;
2354 unsigned flags;
2355 struct data *data;
2356 pointer_t ptr;
2358 option = get_i_param(0);
2359 option_idx = get_i_param(1);
2360 result = get_i_param(2);
2361 flags = get_i_param(3);
2363 if (unlikely(frame_variable_is_flat(fp, option)))
2364 goto cat(option_load_mismatch,ARG_MODE);
2366 pointer_follow(frame_pointer(fp, option), true, data, unlikely(flags & OPCODE_OP_FLAG_STRICT) ? PF_WAIT : PF_NOEVAL, fp, ip,
2367 if (!(flags & OPCODE_OP_FLAG_STRICT)) {
2368 ex_ = ipret_option_load_create_thunk(fp, ip, option, option_idx, result);
2369 if (ex_ == POINTER_FOLLOW_THUNK_GO)
2370 goto cat(option_load_end_,ARG_MODE);
2371 }
2372 RELOAD_EX_POSITION(ex_),
2373 thunk_reference(thunk_);
2374 frame_set_pointer(fp, result, pointer_thunk(thunk_));
2375 goto cat(option_load_end_,ARG_MODE);
2376 );
2378 if (unlikely(da(data,option)->option != option_idx)) {
2379 cat(option_load_mismatch,ARG_MODE):
2380 ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_OPTION_DOESNT_MATCH), fp, ip pass_file_line);
2381 frame_set_pointer(fp, result, ptr);
2382 } else {
2383 pointer_reference_maybe(fp, result, &da(data,option)->pointer, flags);
2384 }
2386 cat(option_load_end_,ARG_MODE):
2387 ADVANCE_I_PARAM(4);
2388 #endif
2389 DEFINE_OPCODE_END(OPCODE_OPTION_LOAD)
2392 START_BLOCK(
2393 frame_t slot_1;
2394 ajla_option_t option;
2395 )
2397 DEFINE_OPCODE_START(OPCODE_OPTION_TEST_FLAT)
2398 #ifdef EMIT_CODE
2399 slot_1 = get_i_param(0);
2401 if (!frame_test_flag(fp, slot_1)) {
2402 barrier_aliasing();
2403 option = *frame_slot(fp, slot_1, ajla_flat_option_t);
2404 barrier_aliasing();
2405 goto cat(option_test_store_result_,ARG_MODE);
2406 }
2407 goto cat(option_test_,ARG_MODE);
2408 #endif
2409 DEFINE_OPCODE_END(OPCODE_OPTION_TEST_FLAT)
2411 DEFINE_OPCODE_START(OPCODE_OPTION_TEST)
2412 #ifdef EMIT_CODE
2413 frame_t slot_r;
2414 pointer_t ptr;
2415 struct data *data;
2417 slot_1 = get_i_param(0);
2418 cat(option_test_,ARG_MODE):
2419 ptr = *frame_pointer(fp, slot_1);
2420 if (unlikely(pointer_is_thunk(ptr))) {
2421 void *ex;
2422 ajla_assert(!frame_test_flag(fp, get_i_param(2)), (file_line, "option test: flag already set for destination slot %"PRIuMAX"", (uintmax_t)get_i_param(2)));
2423 ex = thunk_option_test(fp, ip, slot_1, get_i_param(1), get_i_param(2));
2424 if (ex == POINTER_FOLLOW_THUNK_GO) {
2425 ADVANCE_I_PARAM(3);
2426 break;
2427 } else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY)) {
2428 RELOAD_EX_POSITION(ex);
2429 }
2430 break;
2431 }
2432 data = pointer_get_data(ptr);
2433 option = da(data,option)->option;
2434 cat(option_test_store_result_,ARG_MODE):
2435 slot_r = get_i_param(2);
2436 barrier_aliasing();
2437 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "option test: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
2438 *frame_slot(fp, slot_r, ajla_flat_option_t) = option == (ajla_option_t)get_i_param(1);
2439 barrier_aliasing();
2440 ADVANCE_I_PARAM(3);
2441 #endif
2442 DEFINE_OPCODE_END(OPCODE_OPTION_TEST)
2444 DEFINE_OPCODE_START(OPCODE_OPTION_ORD_FLAT)
2445 #ifdef EMIT_CODE
2446 slot_1 = get_i_param(0);
2448 if (!frame_test_flag(fp, slot_1)) {
2449 barrier_aliasing();
2450 option = *frame_slot(fp, slot_1, ajla_flat_option_t);
2451 barrier_aliasing();
2452 goto cat(option_ord_store_result_,ARG_MODE);
2453 }
2454 goto cat(option_ord_,ARG_MODE);
2455 #endif
2456 DEFINE_OPCODE_END(OPCODE_OPTION_ORD_FLAT)
2458 DEFINE_OPCODE_START(OPCODE_OPTION_ORD)
2459 #ifdef EMIT_CODE
2460 frame_t slot_r;
2461 pointer_t ptr;
2462 struct data *data;
2464 slot_1 = get_i_param(0);
2465 cat(option_ord_,ARG_MODE):
2466 ptr = *frame_pointer(fp, slot_1);
2467 if (unlikely(pointer_is_thunk(ptr))) {
2468 void *ex;
2469 ajla_assert(!frame_test_flag(fp, get_i_param(1)), (file_line, "option ord: flag already set for destination slot %"PRIuMAX"", (uintmax_t)get_i_param(2)));
2470 ex = thunk_option_ord(fp, ip, slot_1, get_i_param(1));
2471 if (ex == POINTER_FOLLOW_THUNK_GO) {
2472 ADVANCE_I_PARAM(2);
2473 break;
2474 } else if (unlikely(ex != POINTER_FOLLOW_THUNK_RETRY)) {
2475 RELOAD_EX_POSITION(ex);
2476 }
2477 break;
2478 }
2479 data = pointer_get_data(ptr);
2480 option = da(data,option)->option;
2481 cat(option_ord_store_result_,ARG_MODE):
2482 slot_r = get_i_param(1);
2483 barrier_aliasing();
2484 ajla_assert(!frame_test_flag(fp, slot_r), (file_line, "option ord: flag already set for destination slot %"PRIuMAX"", (uintmax_t)slot_r));
2485 *frame_slot(fp, slot_r, int_default_t) = option;
2486 barrier_aliasing();
2487 ADVANCE_I_PARAM(2);
2488 #endif
2489 DEFINE_OPCODE_END(OPCODE_OPTION_ORD)
2491 END_BLOCK()
2494 DEFINE_OPCODE_START(OPCODE_ARRAY_CREATE)
2495 #ifdef EMIT_CODE
2496 frame_t result_slot;
2497 size_t n_entries;
2498 struct data *total;
2500 result_slot = get_i_param(0);
2501 n_entries = get_i_param(1);
2502 ADVANCE_I_PARAM(2);
2504 ajla_assert(n_entries != 0, (file_line, "array create: zero entries"));
2506 total = NULL;
2507 while (1) {
2508 size_t n, i;
2509 struct data *a;
2510 const struct type *type = frame_get_type_of_local(fp, get_param(ip, 0));
2511 bool flat = frame_variable_is_flat(fp, get_param(ip, 0));
2512 for (n = 1; n < n_entries; n++) {
2513 if (frame_variable_is_flat(fp, get_param(ip, n * 2)) != flat)
2514 break;
2515 if (unlikely((int_default_t)(n + 1) < zero))
2516 break;
2517 }
2518 do {
2519 if (flat)
2520 a = data_alloc_array_flat_mayfail(type, n, n, false, &ajla_error pass_file_line);
2521 else
2522 a = data_alloc_array_pointers_mayfail(n, n, &ajla_error pass_file_line);
2523 } while (unlikely(!a) && (n >>= 1));
2524 if (unlikely(!a)) {
2525 if (total)
2526 data_dereference(total);
2527 goto cat(array_create_error_,ARG_MODE);
2528 }
2529 n_entries -= n;
2530 if (flat) {
2531 unsigned char *flat_ptr = da_array_flat(a);
2532 for (i = 0; i < n; i++) {
2533 frame_t var_slot = get_param(ip, 0);
2534 memcpy_fast(flat_ptr, frame_var(fp, var_slot), type->size);
2535 flat_ptr += type->size;
2536 ADVANCE_IP(param_size(2));
2537 }
2538 } else {
2539 for (i = 0; i < n; i++) {
2540 frame_t var_slot = get_param(ip, 0);
2541 pointer_t ptr = frame_get_pointer_reference(fp, var_slot, (get_param(ip, 1) & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2542 da(a,array_pointers)->pointer[i] = ptr;
2543 ADVANCE_IP(param_size(2));
2544 }
2545 }
2546 if (likely(!total)) {
2547 total = a;
2548 } else {
2549 total = array_join(total, a, &ajla_error);
2550 if (unlikely(!total)) {
2551 cat(array_create_error_,ARG_MODE):
2552 ip = cat(free_parameters_,ARG_MODE)(fp, ip, n_entries);
2553 frame_set_pointer(fp, result_slot, pointer_error(ajla_error, fp, ip pass_file_line));
2554 break;
2555 }
2556 }
2557 if (likely(!n_entries)) {
2558 frame_set_pointer(fp, result_slot, pointer_data(total));
2559 break;
2560 }
2561 }
2562 #endif
2563 DEFINE_OPCODE_END(OPCODE_ARRAY_CREATE)
2565 DEFINE_OPCODE_START(OPCODE_ARRAY_CREATE_EMPTY_FLAT)
2566 #ifdef EMIT_CODE
2567 frame_t result_slot;
2568 struct data *a;
2569 const struct type *type = da_type(get_frame(fp)->function, get_i_param(1));
2570 TYPE_TAG_VALIDATE(type->tag);
2572 a = data_alloc_array_flat_mayfail(type, ARRAY_PREALLOC_SIZE, 0, false, &ajla_error pass_file_line);
2573 result_slot = get_i_param(0);
2574 frame_set_pointer(fp, result_slot, likely(a != NULL) ? pointer_data(a) : pointer_error(ajla_error, fp, ip pass_file_line));
2575 ADVANCE_I_PARAM(2);
2576 #endif
2577 DEFINE_OPCODE_END(OPCODE_ARRAY_CREATE_EMPTY_FLAT)
2579 DEFINE_OPCODE_START(OPCODE_ARRAY_CREATE_EMPTY)
2580 #ifdef EMIT_CODE
2581 frame_t result_slot;
2582 struct data *a;
2584 a = data_alloc_array_pointers_mayfail(ARRAY_PREALLOC_SIZE, 0, &ajla_error pass_file_line);
2585 result_slot = get_i_param(0);
2586 frame_set_pointer(fp, result_slot, likely(a != NULL) ? pointer_data(a) : pointer_error(ajla_error, fp, ip pass_file_line));
2587 ADVANCE_I_PARAM(1);
2588 #endif
2589 DEFINE_OPCODE_END(OPCODE_ARRAY_CREATE_EMPTY)
2591 DEFINE_OPCODE_START(OPCODE_ARRAY_FILL)
2592 #ifdef EMIT_CODE
2593 frame_t content_slot, length_slot, result_slot;
2594 unsigned char content_flag;
2595 array_index_t length;
2596 pointer_t result_ptr;
2597 void *ex;
2599 content_slot = get_i_param(0);
2600 content_flag = get_i_param(1);
2601 length_slot = get_i_param(2);
2602 result_slot = get_i_param(3);
2604 ex = ipret_get_index(fp, ip, fp, length_slot, NULL, &length, &result_ptr pass_file_line);
2605 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
2606 if (ex == POINTER_FOLLOW_THUNK_EXCEPTION) {
2607 if (content_flag & OPCODE_FLAG_FREE_ARGUMENT)
2608 frame_free_and_clear(fp, content_slot);
2609 goto cat(array_fill_error_,ARG_MODE);
2610 }
2611 RELOAD_EX_POSITION(ex);
2612 }
2614 if (!frame_variable_is_flat(fp, content_slot)) {
2615 pointer_t ptr = frame_get_pointer_reference(fp, content_slot, (content_flag & OPCODE_FLAG_FREE_ARGUMENT) != 0);
2616 if (unlikely((content_flag & OPCODE_ARRAY_FILL_FLAG_SPARSE) != 0)) {
2617 result_ptr = array_create_sparse(length, ptr);
2618 } else {
2619 result_ptr = array_create(length, NULL, NULL, ptr);
2620 }
2621 } else {
2622 const struct type *type = frame_get_type_of_local(fp, content_slot);
2623 barrier_aliasing();
2624 if (unlikely((content_flag & OPCODE_ARRAY_FILL_FLAG_SPARSE) != 0) && likely(index_ge_int(length, 1))) {
2625 pointer_t ptr = flat_to_data(type, frame_var(fp, content_slot));
2626 result_ptr = array_create_sparse(length, ptr);
2627 } else {
2628 if (TYPE_IS_FLAT(frame_get_type_of_local(fp, result_slot))) {
2629 unsigned char *result = frame_var(fp, result_slot);
2630 int_default_t l = index_to_int(length);
2631 index_free(&length);
2632 while (l--) {
2633 result = mempcpy(result, frame_var(fp, content_slot), type->size);
2634 }
2635 goto cat(array_exit_,ARG_MODE);
2636 } else {
2637 result_ptr = array_create(length, type, frame_var(fp, content_slot), pointer_empty());
2638 }
2639 }
2640 barrier_aliasing();
2641 }
2643 cat(array_fill_error_,ARG_MODE):
2644 frame_set_pointer(fp, result_slot, result_ptr);
2646 cat(array_exit_,ARG_MODE):
2647 ADVANCE_I_PARAM(4);
2648 #endif
2649 DEFINE_OPCODE_END(OPCODE_ARRAY_FILL)
2651 DEFINE_OPCODE_START(OPCODE_ARRAY_STRING)
2652 #ifdef EMIT_CODE
2653 frame_t result_slot, length;
2654 pointer_t result_ptr;
2656 result_slot = get_i_param(0);
2657 length = get_i_param(1);
2658 ADVANCE_I_PARAM(2);
2660 if (unlikely((int_default_t)length < (int_default_t)zero) ||
2661 unlikely((frame_t)(int_default_t)length != length)) {
2662 result_ptr = pointer_error(error_ajla(EC_ASYNC, AJLA_ERROR_SIZE_OVERFLOW), fp, ip pass_file_line);
2663 } else {
2664 result_ptr = array_string(length, type_get_fixed(0, true), cast_ptr(unsigned char *, ip));
2665 }
2666 frame_set_pointer(fp, result_slot, result_ptr);
2668 ADVANCE_IP((length + 1) >> 1);
2669 #endif
2670 DEFINE_OPCODE_END(OPCODE_ARRAY_STRING)
2672 DEFINE_OPCODE_START(OPCODE_ARRAY_UNICODE)
2673 #ifdef EMIT_CODE
2674 frame_t result_slot, length;
2675 pointer_t result_ptr;
2677 result_slot = get_i_param(0);
2678 length = get_i_param(1);
2679 ADVANCE_I_PARAM(2);
2681 if (unlikely((int_default_t)length < (int_default_t)zero) ||
2682 unlikely((frame_t)(int_default_t)length != length)) {
2683 result_ptr = pointer_error(error_ajla(EC_ASYNC, AJLA_ERROR_SIZE_OVERFLOW), fp, ip pass_file_line);
2684 } else {
2685 result_ptr = array_string(length, type_get_int(2), cast_ptr(unsigned char *, ip));
2686 }
2687 frame_set_pointer(fp, result_slot, result_ptr);
2689 ADVANCE_IP(length * 2);
2690 #endif
2691 DEFINE_OPCODE_END(OPCODE_ARRAY_STRING)
2693 DEFINE_OPCODE_START(OPCODE_ARRAY_LOAD)
2694 #ifdef EMIT_CODE
2695 frame_t array, idx, result;
2696 unsigned flags;
2697 const struct type *t, *t_elem;
2698 void *ex;
2699 array_index_t array_index;
2700 pointer_t ptr;
2701 unsigned char *flat;
2703 array = get_i_param(0);
2704 idx = get_i_param(1);
2705 result = get_i_param(2);
2706 flags = get_i_param(3);
2708 ex = ipret_get_index(fp, ip, fp, idx, NULL, &array_index, &ptr pass_file_line);
2709 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO)) {
2710 if (ex == POINTER_FOLLOW_THUNK_EXCEPTION)
2711 goto cat(array_load_set_ptr_,ARG_MODE);
2712 RELOAD_EX_POSITION(ex);
2713 }
2715 t = frame_get_type_of_local(fp, array);
2716 if (t->tag == TYPE_TAG_flat_array && !frame_test_flag(fp, array)) {
2717 const struct flat_array_definition *def = type_def(t,flat_array);
2718 if (unlikely(index_ge_int(array_index, def->n_elements))) {
2719 if (unlikely((flags & OPCODE_ARRAY_INDEX_IN_RANGE) != 0))
2720 internal(file_line, "array_load: flat array index out of range");
2721 index_free(&array_index);
2722 ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_INDEX_OUT_OF_RANGE), fp, ip pass_file_line);
2723 goto cat(array_load_set_ptr_,ARG_MODE);
2724 }
2725 t_elem = def->base;
2726 flat = frame_var(fp, array) + t_elem->size * index_to_int(array_index);
2727 index_free(&array_index);
2728 goto cat(array_load_set_flat_,ARG_MODE);
2729 } else {
2730 pointer_t *array_ptr = frame_pointer(fp, array);
2731 pointer_t *pptr;
2732 struct data *data;
2734 cat(array_restart_pointer_follow_,ARG_MODE):
2735 pointer_follow(array_ptr, false, data, unlikely(flags & OPCODE_OP_FLAG_STRICT) ? PF_WAIT : PF_NOEVAL, fp, ip,
2736 index_free(&array_index);
2737 if (!(flags & OPCODE_OP_FLAG_STRICT)) {
2738 ex_ = ipret_array_load_create_thunk(fp, ip, array, idx, result);
2739 if (ex_ == POINTER_FOLLOW_THUNK_GO)
2740 goto cat(array_load_end_,ARG_MODE);
2741 }
2742 RELOAD_EX_POSITION(ex_),
2743 index_free(&array_index);
2744 thunk_reference(thunk_);
2745 ptr = pointer_thunk(thunk_);
2746 goto cat(array_load_set_ptr_,ARG_MODE);
2747 );
2749 if (unlikely(da_tag(data) == DATA_TAG_array_incomplete)) {
2750 array_index_t len_first;
2752 if (array_ptr == frame_pointer(fp, array)) {
2753 if (!frame_test_and_set_flag(fp, array))
2754 data_reference(data);
2755 if (array_incomplete_collapse(array_ptr))
2756 goto cat(array_restart_pointer_follow_,ARG_MODE);
2757 }
2759 len_first = array_len(pointer_get_data(da(data,array_incomplete)->first));
2760 if (!index_ge_index(array_index, len_first)) {
2761 index_free(&len_first);
2762 data = pointer_get_data(da(data,array_incomplete)->first);
2763 } else {
2764 index_sub(&array_index, len_first);
2765 index_free(&len_first);
2766 array_ptr = &da(data,array_incomplete)->next;
2767 goto cat(array_restart_pointer_follow_,ARG_MODE);
2768 }
2769 }
2771 if (unlikely(!array_read(data, array_index, &pptr, &flat, &t_elem, NULL))) {
2772 if (unlikely((flags & OPCODE_ARRAY_INDEX_IN_RANGE) != 0))
2773 internal(file_line, "array_load: %s: array index out of range", da(get_frame(fp)->function,function)->function_name);
2774 ptr = pointer_error(error_ajla(EC_SYNC, AJLA_ERROR_INDEX_OUT_OF_RANGE), fp, ip pass_file_line);
2775 goto cat(array_load_set_ptr_,ARG_MODE);
2776 }
2777 if (pptr) {
2778 pointer_reference_maybe(fp, result, pptr, flags);
2779 goto cat(array_load_end_,ARG_MODE);
2780 } else {
2781 cat(array_load_set_flat_,ARG_MODE):
2782 if (likely(TYPE_IS_FLAT(frame_get_type_of_local(fp, result)))) {
2783 memcpy_fast(frame_var(fp, result), flat, t_elem->size);
2784 goto cat(array_load_end_,ARG_MODE);
2785 } else {
2786 ptr = flat_to_data(t_elem, flat);
2787 }
2788 }
2789 }
2791 cat(array_load_set_ptr_,ARG_MODE):
2792 frame_set_pointer(fp, result, ptr);
2794 cat(array_load_end_,ARG_MODE):
2795 ADVANCE_I_PARAM(4);
2796 #endif
2797 DEFINE_OPCODE_END(OPCODE_ARRAY_LOAD)
2799 DEFINE_OPCODE_START(OPCODE_ARRAY_LEN)
2800 #ifdef EMIT_CODE
2801 frame_t slot_a, slot_r;
2802 unsigned flags;
2803 void *ex;
2805 slot_a = get_i_param(0);
2806 slot_r = get_i_param(1);
2807 flags = get_i_param(2);
2809 ex = ipret_array_len(fp, ip, slot_r, slot_a, flags);
2810 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2811 RELOAD_EX_POSITION(ex);
2813 ADVANCE_I_PARAM(3);
2814 #endif
2815 DEFINE_OPCODE_END(OPCODE_ARRAY_LEN)
2817 DEFINE_OPCODE_START(OPCODE_ARRAY_LEN_GREATER_THAN)
2818 #ifdef EMIT_CODE
2819 frame_t slot_a, slot_l, slot_r;
2820 unsigned flags;
2821 void *ex;
2823 slot_a = get_i_param(0);
2824 slot_l = get_i_param(1);
2825 slot_r = get_i_param(2);
2826 flags = get_i_param(3);
2828 ex = ipret_array_len_greater_than(fp, ip, slot_r, slot_a, slot_l, flags);
2829 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2830 RELOAD_EX_POSITION(ex);
2832 ADVANCE_I_PARAM(4);
2833 #endif
2834 DEFINE_OPCODE_END(OPCODE_ARRAY_LEN_GREATER_THAN)
2836 DEFINE_OPCODE_START(OPCODE_ARRAY_SUB)
2837 #ifdef EMIT_CODE
2838 frame_t slot_a, slot_start, slot_end, slot_r;
2839 unsigned flags;
2840 void *ex;
2842 slot_a = get_i_param(0);
2843 slot_start = get_i_param(1);
2844 slot_end = get_i_param(2);
2845 slot_r = get_i_param(3);
2846 flags = get_i_param(4);
2848 ex = ipret_array_sub(fp, ip, slot_r, slot_a, slot_start, slot_end, flags);
2849 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2850 RELOAD_EX_POSITION(ex);
2852 ADVANCE_I_PARAM(5);
2853 #endif
2854 DEFINE_OPCODE_END(OPCODE_ARRAY_SUB)
2856 DEFINE_OPCODE_START(OPCODE_ARRAY_SKIP)
2857 #ifdef EMIT_CODE
2858 frame_t slot_a, slot_start, slot_r;
2859 unsigned flags;
2860 void *ex;
2862 slot_a = get_i_param(0);
2863 slot_start = get_i_param(1);
2864 slot_r = get_i_param(2);
2865 flags = get_i_param(3);
2867 ex = ipret_array_skip(fp, ip, slot_r, slot_a, slot_start, flags);
2868 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2869 RELOAD_EX_POSITION(ex);
2871 ADVANCE_I_PARAM(4);
2872 #endif
2873 DEFINE_OPCODE_END(OPCODE_ARRAY_SKIP)
2875 DEFINE_OPCODE_START(OPCODE_ARRAY_APPEND)
2876 #ifdef EMIT_CODE
2877 frame_t slot_r, slot_1, slot_2;
2878 unsigned flags;
2879 void *ex;
2881 slot_r = get_i_param(0);
2882 flags = get_i_param(1);
2883 slot_1 = get_i_param(2);
2884 slot_2 = get_i_param(3);
2886 ex = ipret_array_append(fp, ip, slot_r, slot_1, slot_2, flags);
2887 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2888 RELOAD_EX_POSITION(ex);
2890 ADVANCE_I_PARAM(4);
2891 #endif
2892 DEFINE_OPCODE_END(OPCODE_ARRAY_APPEND)
2894 DEFINE_OPCODE_START(OPCODE_ARRAY_APPEND_ONE_FLAT)
2895 #ifdef EMIT_CODE
2896 frame_t slot_r, slot_1, slot_2;
2897 unsigned flags;
2898 void *ex;
2900 slot_r = get_i_param(0);
2901 flags = get_i_param(1);
2902 slot_1 = get_i_param(2);
2903 slot_2 = get_i_param(3);
2905 ex = ipret_array_append_one_flat(fp, ip, slot_r, slot_1, slot_2, flags);
2906 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2907 RELOAD_EX_POSITION(ex);
2909 ADVANCE_I_PARAM(4);
2910 #endif
2911 DEFINE_OPCODE_END(OPCODE_ARRAY_APPEND_ONE_FLAT)
2913 DEFINE_OPCODE_START(OPCODE_ARRAY_APPEND_ONE)
2914 #ifdef EMIT_CODE
2915 frame_t slot_r, slot_1, slot_2;
2916 unsigned flags;
2917 void *ex;
2919 slot_r = get_i_param(0);
2920 flags = get_i_param(1);
2921 slot_1 = get_i_param(2);
2922 slot_2 = get_i_param(3);
2924 ex = ipret_array_append_one(fp, ip, slot_r, slot_1, slot_2, flags);
2925 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2926 RELOAD_EX_POSITION(ex);
2928 ADVANCE_I_PARAM(4);
2929 #endif
2930 DEFINE_OPCODE_END(OPCODE_ARRAY_APPEND_ONE)
2932 DEFINE_OPCODE_START(OPCODE_ARRAY_FLATTEN)
2933 #ifdef EMIT_CODE
2934 frame_t slot_r, slot_1;
2935 unsigned flags;
2936 void *ex;
2938 slot_r = get_i_param(0);
2939 flags = get_i_param(1);
2940 slot_1 = get_i_param(2);
2942 ex = ipret_array_flatten(fp, ip, slot_r, slot_1, flags);
2943 if (unlikely(ex != POINTER_FOLLOW_THUNK_GO))
2944 RELOAD_EX_POSITION(ex);
2946 ADVANCE_I_PARAM(3);
2947 #endif
2948 DEFINE_OPCODE_END(OPCODE_ARRAY_FLATTEN)
2951 #if ARG_MODE == 0
2952 DEFINE_OPCODE_START(OPCODE_IO)
2953 #ifdef EMIT_CODE
2954 unsigned char io_code, n_outputs, n_inputs, n_params;
2955 void *ex;
2957 io_code = get_i_param(0);
2958 n_outputs = get_i_param(1);
2959 n_inputs = get_i_param(2);
2960 n_params = get_i_param(3);
2962 ex = ipret_io(fp, ip, io_code, n_outputs, n_inputs, n_params);
2963 if (ex != POINTER_FOLLOW_THUNK_GO) {
2964 RELOAD_EX_POSITION(ex);
2965 } else {
2966 ADVANCE_IP(3 + 2 * (n_outputs + n_inputs + n_params));
2967 }
2968 #endif
2969 DEFINE_OPCODE_END(OPCODE_IO)
2970 #endif
2972 #if ARG_MODE == 0
2973 DEFINE_OPCODE_START(OPCODE_INTERNAL_FUNCTION)
2974 #ifdef EMIT_CODE
2975 void *ex = function_call_internal(fp, ip);
2976 if (ex != POINTER_FOLLOW_THUNK_RETRY) {
2977 RELOAD_EX_POSITION(ex);
2978 }
2979 #endif
2980 DEFINE_OPCODE_END(OPCODE_INTERNAL_FUNCTION)
2981 #endif
2983 #if ARG_MODE == 0
2984 DEFINE_OPCODE_START(OPCODE_UNREACHABLE)
2985 #ifdef EMIT_CODE
2986 internal(file_line, "unreachable code");
2987 #endif
2988 DEFINE_OPCODE_END(OPCODE_UNREACHABLE)
2989 #endif
2991 #if ARG_MODE == 0
2992 DEFINE_OPCODE_START(OPCODE_EXIT_THREAD)
2993 #ifdef EMIT_CODE
2994 struct execution_control *ex = frame_execution_control(fp);
2995 pointer_t *var_ptr = frame_pointer(fp, get_i_param(0));
2996 struct data attr_unused *data;
2998 pointer_follow(var_ptr, true, data, PF_WAIT, fp, ip,
2999 RELOAD_EX_POSITION(ex_),
3000 ajla_assert_lo(thunk_tag(thunk_) == THUNK_TAG_EXCEPTION, (file_line, "exit_thread: invalid thunk tag %u", thunk_tag(thunk_)));
3001 goto exit_ipret_thunk;
3002 );
3004 exit_ipret_thunk:
3005 ajla_assert_lo(ex->thunk == NULL, (file_line, "exit_thread: non-NULL thunk %p", ex->thunk));
3006 ex->current_frame = fp;
3007 ex->current_ip = frame_ip(fp, ip);
3008 execution_control_terminate(ex, *var_ptr);
3009 goto exit_ipret;
3010 #endif
3011 DEFINE_OPCODE_END(OPCODE_EXIT_THREAD)
3012 #endif
3015 #undef move_scalar
3016 #undef copy_scalar
3018 #undef get_param
3019 #undef param_size
3020 #undef get_i_param
3021 #undef ADVANCE_I_PARAM
3023 #undef ARG_MODE
3024 #undef DEFINE_OPCODE_START