| blob | 2c0df6e3b08505ac67315a671afaa03821596e87 |
1 // z_Linux_asm.S: - microtasking routines specifically
2 // written for Intel platforms running Linux* OS
4 //
5 ////===----------------------------------------------------------------------===//
6 ////
7 //// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 //// See https://llvm.org/LICENSE.txt for license information.
9 //// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 ////
11 ////===----------------------------------------------------------------------===//
12 //
14 // -----------------------------------------------------------------------
15 // macros
16 // -----------------------------------------------------------------------
18 #include "kmp_config.h"
20 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
22 # if KMP_MIC
23 // the 'delay r16/r32/r64' should be used instead of the 'pause'.
24 // The delay operation has the effect of removing the current thread from
25 // the round-robin HT mechanism, and therefore speeds up the issue rate of
26 // the other threads on the same core.
27 //
28 // A value of 0 works fine for <= 2 threads per core, but causes the EPCC
29 // barrier time to increase greatly for 3 or more threads per core.
30 //
31 // A value of 100 works pretty well for up to 4 threads per core, but isn't
32 // quite as fast as 0 for 2 threads per core.
33 //
34 // We need to check what happens for oversubscription / > 4 threads per core.
35 // It is possible that we need to pass the delay value in as a parameter
36 // that the caller determines based on the total # threads / # cores.
37 //
38 //.macro pause_op
39 // mov $100, %rax
40 // delay %rax
41 //.endm
42 # else
43 # define pause_op .byte 0xf3,0x90
44 # endif // KMP_MIC
46 # if KMP_OS_DARWIN
47 # define KMP_PREFIX_UNDERSCORE(x) _##x // extra underscore for OS X* symbols
48 # define KMP_LABEL(x) L_##x // form the name of label
49 .macro KMP_CFI_DEF_OFFSET
50 .endmacro
51 .macro KMP_CFI_OFFSET
52 .endmacro
53 .macro KMP_CFI_REGISTER
54 .endmacro
55 .macro KMP_CFI_DEF
56 .endmacro
57 .macro ALIGN
58 .align $0
59 .endmacro
60 .macro DEBUG_INFO
61 /* Not sure what .size does in icc, not sure if we need to do something
62 similar for OS X*.
63 */
64 .endmacro
65 .macro PROC
66 ALIGN 4
67 .globl KMP_PREFIX_UNDERSCORE($0)
68 KMP_PREFIX_UNDERSCORE($0):
69 .endmacro
70 # else // KMP_OS_DARWIN
71 # define KMP_PREFIX_UNDERSCORE(x) x //no extra underscore for Linux* OS symbols
72 // Format labels so that they don't override function names in gdb's backtraces
73 // MIC assembler doesn't accept .L syntax, the L works fine there (as well as
74 // on OS X*)
75 # if KMP_MIC
76 # define KMP_LABEL(x) L_##x // local label
77 # else
78 # define KMP_LABEL(x) .L_##x // local label hidden from backtraces
79 # endif // KMP_MIC
80 .macro ALIGN size
81 .align 1<<(\size)
82 .endm
83 .macro DEBUG_INFO proc
84 .cfi_endproc
85 // Not sure why we need .type and .size for the functions
86 .align 16
87 .type \proc,@function
88 .size \proc,.-\proc
89 .endm
90 .macro PROC proc
91 ALIGN 4
92 .globl KMP_PREFIX_UNDERSCORE(\proc)
93 KMP_PREFIX_UNDERSCORE(\proc):
94 .cfi_startproc
95 .endm
96 .macro KMP_CFI_DEF_OFFSET sz
97 .cfi_def_cfa_offset \sz
98 .endm
99 .macro KMP_CFI_OFFSET reg, sz
100 .cfi_offset \reg,\sz
101 .endm
102 .macro KMP_CFI_REGISTER reg
103 .cfi_def_cfa_register \reg
104 .endm
105 .macro KMP_CFI_DEF reg, sz
106 .cfi_def_cfa \reg,\sz
107 .endm
108 # endif // KMP_OS_DARWIN
109 #endif // KMP_ARCH_X86 || KMP_ARCH_x86_64
111 #if (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM)
113 # if KMP_OS_DARWIN
114 # define KMP_PREFIX_UNDERSCORE(x) _##x // extra underscore for OS X* symbols
115 # define KMP_LABEL(x) L_##x // form the name of label
117 .macro ALIGN
118 .align $0
119 .endmacro
121 .macro DEBUG_INFO
122 /* Not sure what .size does in icc, not sure if we need to do something
123 similar for OS X*.
124 */
125 .endmacro
127 .macro PROC
128 ALIGN 4
129 .globl KMP_PREFIX_UNDERSCORE($0)
130 KMP_PREFIX_UNDERSCORE($0):
131 .endmacro
132 # elif KMP_OS_WINDOWS
133 # define KMP_PREFIX_UNDERSCORE(x) x // no extra underscore for Windows/ARM64 symbols
134 // Format labels so that they don't override function names in gdb's backtraces
135 # define KMP_LABEL(x) .L_##x // local label hidden from backtraces
137 .macro ALIGN size
138 .align 1<<(\size)
139 .endm
141 .macro DEBUG_INFO proc
142 ALIGN 2
143 .endm
145 .macro PROC proc
146 ALIGN 2
147 .globl KMP_PREFIX_UNDERSCORE(\proc)
148 KMP_PREFIX_UNDERSCORE(\proc):
149 .endm
150 # else // KMP_OS_DARWIN || KMP_OS_WINDOWS
151 # define KMP_PREFIX_UNDERSCORE(x) x // no extra underscore for Linux* OS symbols
152 // Format labels so that they don't override function names in gdb's backtraces
153 # define KMP_LABEL(x) .L_##x // local label hidden from backtraces
155 .macro ALIGN size
156 .align 1<<(\size)
157 .endm
159 .macro DEBUG_INFO proc
160 .cfi_endproc
161 // Not sure why we need .type and .size for the functions
162 ALIGN 2
163 #if KMP_ARCH_ARM
164 .type \proc,%function
165 #else
166 .type \proc,@function
167 #endif
168 .size \proc,.-\proc
169 .endm
171 .macro PROC proc
172 ALIGN 2
173 .globl KMP_PREFIX_UNDERSCORE(\proc)
174 KMP_PREFIX_UNDERSCORE(\proc):
175 .cfi_startproc
176 .endm
177 # endif // KMP_OS_DARWIN
179 #endif // (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM)
181 .macro COMMON name, size, align_power
182 #if KMP_OS_DARWIN
183 .comm \name, \size
184 #elif KMP_OS_WINDOWS
185 .comm \name, \size, \align_power
186 #else // !KMP_OS_DARWIN && !KMP_OS_WINDOWS
187 .comm \name, \size, (1<<(\align_power))
188 #endif
189 .endm
191 // -----------------------------------------------------------------------
192 // data
193 // -----------------------------------------------------------------------
195 #ifdef KMP_GOMP_COMPAT
197 // Support for unnamed common blocks.
198 //
199 // Because the symbol ".gomp_critical_user_" contains a ".", we have to
200 // put this stuff in assembly.
202 # if KMP_ARCH_X86
203 # if KMP_OS_DARWIN
204 .data
205 .comm .gomp_critical_user_,32
206 .data
207 .globl ___kmp_unnamed_critical_addr
208 ___kmp_unnamed_critical_addr:
209 .long .gomp_critical_user_
210 # else /* Linux* OS */
211 .data
212 .comm .gomp_critical_user_,32,8
213 .data
214 ALIGN 4
215 .global __kmp_unnamed_critical_addr
216 __kmp_unnamed_critical_addr:
217 .4byte .gomp_critical_user_
218 .type __kmp_unnamed_critical_addr,@object
219 .size __kmp_unnamed_critical_addr,4
220 # endif /* KMP_OS_DARWIN */
221 # endif /* KMP_ARCH_X86 */
223 # if KMP_ARCH_X86_64
224 # if KMP_OS_DARWIN
225 .data
226 .comm .gomp_critical_user_,32
227 .data
228 .globl ___kmp_unnamed_critical_addr
229 ___kmp_unnamed_critical_addr:
230 .quad .gomp_critical_user_
231 # else /* Linux* OS */
232 .data
233 .comm .gomp_critical_user_,32,8
234 .data
235 ALIGN 8
236 .global __kmp_unnamed_critical_addr
237 __kmp_unnamed_critical_addr:
238 .8byte .gomp_critical_user_
239 .type __kmp_unnamed_critical_addr,@object
240 .size __kmp_unnamed_critical_addr,8
241 # endif /* KMP_OS_DARWIN */
242 # endif /* KMP_ARCH_X86_64 */
244 #endif /* KMP_GOMP_COMPAT */
247 #if KMP_ARCH_X86 && !KMP_ARCH_PPC64
249 // -----------------------------------------------------------------------
250 // microtasking routines specifically written for IA-32 architecture
251 // running Linux* OS
252 // -----------------------------------------------------------------------
254 .ident "Intel Corporation"
255 .data
256 ALIGN 4
257 // void
258 // __kmp_x86_pause( void );
260 .text
261 PROC __kmp_x86_pause
263 pause_op
264 ret
266 DEBUG_INFO __kmp_x86_pause
268 # if !KMP_ASM_INTRINS
270 //------------------------------------------------------------------------
271 // kmp_int32
272 // __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 d );
274 PROC __kmp_test_then_add32
276 movl 4(%esp), %ecx
277 movl 8(%esp), %eax
278 lock
279 xaddl %eax,(%ecx)
280 ret
282 DEBUG_INFO __kmp_test_then_add32
284 //------------------------------------------------------------------------
285 // FUNCTION __kmp_xchg_fixed8
286 //
287 // kmp_int32
288 // __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 d );
289 //
290 // parameters:
291 // p: 4(%esp)
292 // d: 8(%esp)
293 //
294 // return: %al
295 PROC __kmp_xchg_fixed8
297 movl 4(%esp), %ecx // "p"
298 movb 8(%esp), %al // "d"
300 lock
301 xchgb %al,(%ecx)
302 ret
304 DEBUG_INFO __kmp_xchg_fixed8
307 //------------------------------------------------------------------------
308 // FUNCTION __kmp_xchg_fixed16
309 //
310 // kmp_int16
311 // __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 d );
312 //
313 // parameters:
314 // p: 4(%esp)
315 // d: 8(%esp)
316 // return: %ax
317 PROC __kmp_xchg_fixed16
319 movl 4(%esp), %ecx // "p"
320 movw 8(%esp), %ax // "d"
322 lock
323 xchgw %ax,(%ecx)
324 ret
326 DEBUG_INFO __kmp_xchg_fixed16
329 //------------------------------------------------------------------------
330 // FUNCTION __kmp_xchg_fixed32
331 //
332 // kmp_int32
333 // __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 d );
334 //
335 // parameters:
336 // p: 4(%esp)
337 // d: 8(%esp)
338 //
339 // return: %eax
340 PROC __kmp_xchg_fixed32
342 movl 4(%esp), %ecx // "p"
343 movl 8(%esp), %eax // "d"
345 lock
346 xchgl %eax,(%ecx)
347 ret
349 DEBUG_INFO __kmp_xchg_fixed32
352 // kmp_int8
353 // __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
354 PROC __kmp_compare_and_store8
356 movl 4(%esp), %ecx
357 movb 8(%esp), %al
358 movb 12(%esp), %dl
359 lock
360 cmpxchgb %dl,(%ecx)
361 sete %al // if %al == (%ecx) set %al = 1 else set %al = 0
362 and $1, %eax // sign extend previous instruction
363 ret
365 DEBUG_INFO __kmp_compare_and_store8
367 // kmp_int16
368 // __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv);
369 PROC __kmp_compare_and_store16
371 movl 4(%esp), %ecx
372 movw 8(%esp), %ax
373 movw 12(%esp), %dx
374 lock
375 cmpxchgw %dx,(%ecx)
376 sete %al // if %ax == (%ecx) set %al = 1 else set %al = 0
377 and $1, %eax // sign extend previous instruction
378 ret
380 DEBUG_INFO __kmp_compare_and_store16
382 // kmp_int32
383 // __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv);
384 PROC __kmp_compare_and_store32
386 movl 4(%esp), %ecx
387 movl 8(%esp), %eax
388 movl 12(%esp), %edx
389 lock
390 cmpxchgl %edx,(%ecx)
391 sete %al // if %eax == (%ecx) set %al = 1 else set %al = 0
392 and $1, %eax // sign extend previous instruction
393 ret
395 DEBUG_INFO __kmp_compare_and_store32
397 // kmp_int32
398 // __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 s );
399 PROC __kmp_compare_and_store64
401 pushl %ebp
402 movl %esp, %ebp
403 pushl %ebx
404 pushl %edi
405 movl 8(%ebp), %edi
406 movl 12(%ebp), %eax // "cv" low order word
407 movl 16(%ebp), %edx // "cv" high order word
408 movl 20(%ebp), %ebx // "sv" low order word
409 movl 24(%ebp), %ecx // "sv" high order word
410 lock
411 cmpxchg8b (%edi)
412 sete %al // if %edx:eax == (%edi) set %al = 1 else set %al = 0
413 and $1, %eax // sign extend previous instruction
414 popl %edi
415 popl %ebx
416 movl %ebp, %esp
417 popl %ebp
418 ret
420 DEBUG_INFO __kmp_compare_and_store64
422 // kmp_int8
423 // __kmp_compare_and_store_ret8(volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv);
424 PROC __kmp_compare_and_store_ret8
426 movl 4(%esp), %ecx
427 movb 8(%esp), %al
428 movb 12(%esp), %dl
429 lock
430 cmpxchgb %dl,(%ecx)
431 ret
433 DEBUG_INFO __kmp_compare_and_store_ret8
435 // kmp_int16
436 // __kmp_compare_and_store_ret16(volatile kmp_int16 *p, kmp_int16 cv,
437 // kmp_int16 sv);
438 PROC __kmp_compare_and_store_ret16
440 movl 4(%esp), %ecx
441 movw 8(%esp), %ax
442 movw 12(%esp), %dx
443 lock
444 cmpxchgw %dx,(%ecx)
445 ret
447 DEBUG_INFO __kmp_compare_and_store_ret16
449 // kmp_int32
450 // __kmp_compare_and_store_ret32(volatile kmp_int32 *p, kmp_int32 cv,
451 // kmp_int32 sv);
452 PROC __kmp_compare_and_store_ret32
454 movl 4(%esp), %ecx
455 movl 8(%esp), %eax
456 movl 12(%esp), %edx
457 lock
458 cmpxchgl %edx,(%ecx)
459 ret
461 DEBUG_INFO __kmp_compare_and_store_ret32
463 // kmp_int64
464 // __kmp_compare_and_store_ret64(volatile kmp_int64 *p, kmp_int64 cv,
465 // kmp_int64 sv);
466 PROC __kmp_compare_and_store_ret64
468 pushl %ebp
469 movl %esp, %ebp
470 pushl %ebx
471 pushl %edi
472 movl 8(%ebp), %edi
473 movl 12(%ebp), %eax // "cv" low order word
474 movl 16(%ebp), %edx // "cv" high order word
475 movl 20(%ebp), %ebx // "sv" low order word
476 movl 24(%ebp), %ecx // "sv" high order word
477 lock
478 cmpxchg8b (%edi)
479 popl %edi
480 popl %ebx
481 movl %ebp, %esp
482 popl %ebp
483 ret
485 DEBUG_INFO __kmp_compare_and_store_ret64
488 //------------------------------------------------------------------------
489 // FUNCTION __kmp_xchg_real32
490 //
491 // kmp_real32
492 // __kmp_xchg_real32( volatile kmp_real32 *addr, kmp_real32 data );
493 //
494 // parameters:
495 // addr: 4(%esp)
496 // data: 8(%esp)
497 //
498 // return: %eax
499 PROC __kmp_xchg_real32
501 pushl %ebp
502 movl %esp, %ebp
503 subl $4, %esp
504 pushl %esi
506 movl 4(%ebp), %esi
507 flds (%esi)
508 // load <addr>
509 fsts -4(%ebp)
510 // store old value
512 movl 8(%ebp), %eax
514 lock
515 xchgl %eax, (%esi)
517 flds -4(%ebp)
518 // return old value
520 popl %esi
521 movl %ebp, %esp
522 popl %ebp
523 ret
525 DEBUG_INFO __kmp_xchg_real32
527 # endif /* !KMP_ASM_INTRINS */
529 //------------------------------------------------------------------------
530 // int
531 // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
532 // int gtid, int tid,
533 // int argc, void *p_argv[]
534 // #if OMPT_SUPPORT
535 // ,
536 // void **exit_frame_ptr
537 // #endif
538 // ) {
539 // #if OMPT_SUPPORT
540 // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
541 // #endif
542 //
543 // (*pkfn)( & gtid, & tid, argv[0], ... );
544 // return 1;
545 // }
547 // -- Begin __kmp_invoke_microtask
548 // mark_begin;
549 PROC __kmp_invoke_microtask
551 pushl %ebp
552 KMP_CFI_DEF_OFFSET 8
553 KMP_CFI_OFFSET ebp,-8
554 movl %esp,%ebp // establish the base pointer for this routine.
555 KMP_CFI_REGISTER ebp
556 subl $8,%esp // allocate space for two local variables.
557 // These varibales are:
558 // argv: -4(%ebp)
559 // temp: -8(%ebp)
560 //
561 pushl %ebx // save %ebx to use during this routine
562 //
563 #if OMPT_SUPPORT
564 movl 28(%ebp),%ebx // get exit_frame address
565 movl %ebp,(%ebx) // save exit_frame
566 #endif
568 movl 20(%ebp),%ebx // Stack alignment - # args
569 addl $2,%ebx // #args +2 Always pass at least 2 args (gtid and tid)
570 shll $2,%ebx // Number of bytes used on stack: (#args+2)*4
571 movl %esp,%eax //
572 subl %ebx,%eax // %esp-((#args+2)*4) -> %eax -- without mods, stack ptr would be this
573 movl %eax,%ebx // Save to %ebx
574 andl $0xFFFFFF80,%eax // mask off 7 bits
575 subl %eax,%ebx // Amount to subtract from %esp
576 subl %ebx,%esp // Prepare the stack ptr --
577 // now it will be aligned on 128-byte boundary at the call
579 movl 24(%ebp),%eax // copy from p_argv[]
580 movl %eax,-4(%ebp) // into the local variable *argv.
582 movl 20(%ebp),%ebx // argc is 20(%ebp)
583 shll $2,%ebx
585 KMP_LABEL(invoke_2):
586 cmpl $0,%ebx
587 jg KMP_LABEL(invoke_4)
588 jmp KMP_LABEL(invoke_3)
589 ALIGN 2
590 KMP_LABEL(invoke_4):
591 movl -4(%ebp),%eax
592 subl $4,%ebx // decrement argc.
593 addl %ebx,%eax // index into argv.
594 movl (%eax),%edx
595 pushl %edx
597 jmp KMP_LABEL(invoke_2)
598 ALIGN 2
599 KMP_LABEL(invoke_3):
600 leal 16(%ebp),%eax // push & tid
601 pushl %eax
603 leal 12(%ebp),%eax // push & gtid
604 pushl %eax
606 movl 8(%ebp),%ebx
607 call *%ebx // call (*pkfn)();
609 movl $1,%eax // return 1;
611 movl -12(%ebp),%ebx // restore %ebx
612 leave
613 KMP_CFI_DEF esp,4
614 ret
616 DEBUG_INFO __kmp_invoke_microtask
617 // -- End __kmp_invoke_microtask
620 // kmp_uint64
621 // __kmp_hardware_timestamp(void)
622 PROC __kmp_hardware_timestamp
623 rdtsc
624 ret
626 DEBUG_INFO __kmp_hardware_timestamp
627 // -- End __kmp_hardware_timestamp
629 #endif /* KMP_ARCH_X86 */
632 #if KMP_ARCH_X86_64
634 // -----------------------------------------------------------------------
635 // microtasking routines specifically written for IA-32 architecture and
636 // Intel(R) 64 running Linux* OS
637 // -----------------------------------------------------------------------
639 // -- Machine type P
640 // mark_description "Intel Corporation";
641 .ident "Intel Corporation"
642 // -- .file "z_Linux_asm.S"
643 .data
644 ALIGN 4
646 // To prevent getting our code into .data section .text added to every routine
647 // definition for x86_64.
648 //------------------------------------------------------------------------
649 # if !KMP_ASM_INTRINS
651 //------------------------------------------------------------------------
652 // FUNCTION __kmp_test_then_add32
653 //
654 // kmp_int32
655 // __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 d );
656 //
657 // parameters:
658 // p: %rdi
659 // d: %esi
660 //
661 // return: %eax
662 .text
663 PROC __kmp_test_then_add32
665 movl %esi, %eax // "d"
666 lock
667 xaddl %eax,(%rdi)
668 ret
670 DEBUG_INFO __kmp_test_then_add32
673 //------------------------------------------------------------------------
674 // FUNCTION __kmp_test_then_add64
675 //
676 // kmp_int64
677 // __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d );
678 //
679 // parameters:
680 // p: %rdi
681 // d: %rsi
682 // return: %rax
683 .text
684 PROC __kmp_test_then_add64
686 movq %rsi, %rax // "d"
687 lock
688 xaddq %rax,(%rdi)
689 ret
691 DEBUG_INFO __kmp_test_then_add64
694 //------------------------------------------------------------------------
695 // FUNCTION __kmp_xchg_fixed8
696 //
697 // kmp_int32
698 // __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 d );
699 //
700 // parameters:
701 // p: %rdi
702 // d: %sil
703 //
704 // return: %al
705 .text
706 PROC __kmp_xchg_fixed8
708 movb %sil, %al // "d"
710 lock
711 xchgb %al,(%rdi)
712 ret
714 DEBUG_INFO __kmp_xchg_fixed8
717 //------------------------------------------------------------------------
718 // FUNCTION __kmp_xchg_fixed16
719 //
720 // kmp_int16
721 // __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 d );
722 //
723 // parameters:
724 // p: %rdi
725 // d: %si
726 // return: %ax
727 .text
728 PROC __kmp_xchg_fixed16
730 movw %si, %ax // "d"
732 lock
733 xchgw %ax,(%rdi)
734 ret
736 DEBUG_INFO __kmp_xchg_fixed16
739 //------------------------------------------------------------------------
740 // FUNCTION __kmp_xchg_fixed32
741 //
742 // kmp_int32
743 // __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 d );
744 //
745 // parameters:
746 // p: %rdi
747 // d: %esi
748 //
749 // return: %eax
750 .text
751 PROC __kmp_xchg_fixed32
753 movl %esi, %eax // "d"
755 lock
756 xchgl %eax,(%rdi)
757 ret
759 DEBUG_INFO __kmp_xchg_fixed32
762 //------------------------------------------------------------------------
763 // FUNCTION __kmp_xchg_fixed64
764 //
765 // kmp_int64
766 // __kmp_xchg_fixed64( volatile kmp_int64 *p, kmp_int64 d );
767 //
768 // parameters:
769 // p: %rdi
770 // d: %rsi
771 // return: %rax
772 .text
773 PROC __kmp_xchg_fixed64
775 movq %rsi, %rax // "d"
777 lock
778 xchgq %rax,(%rdi)
779 ret
781 DEBUG_INFO __kmp_xchg_fixed64
784 //------------------------------------------------------------------------
785 // FUNCTION __kmp_compare_and_store8
786 //
787 // kmp_int8
788 // __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
789 //
790 // parameters:
791 // p: %rdi
792 // cv: %esi
793 // sv: %edx
794 //
795 // return: %eax
796 .text
797 PROC __kmp_compare_and_store8
799 movb %sil, %al // "cv"
800 lock
801 cmpxchgb %dl,(%rdi)
802 sete %al // if %al == (%rdi) set %al = 1 else set %al = 0
803 andq $1, %rax // sign extend previous instruction for return value
804 ret
806 DEBUG_INFO __kmp_compare_and_store8
809 //------------------------------------------------------------------------
810 // FUNCTION __kmp_compare_and_store16
811 //
812 // kmp_int16
813 // __kmp_compare_and_store16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
814 //
815 // parameters:
816 // p: %rdi
817 // cv: %si
818 // sv: %dx
819 //
820 // return: %eax
821 .text
822 PROC __kmp_compare_and_store16
824 movw %si, %ax // "cv"
825 lock
826 cmpxchgw %dx,(%rdi)
827 sete %al // if %ax == (%rdi) set %al = 1 else set %al = 0
828 andq $1, %rax // sign extend previous instruction for return value
829 ret
831 DEBUG_INFO __kmp_compare_and_store16
834 //------------------------------------------------------------------------
835 // FUNCTION __kmp_compare_and_store32
836 //
837 // kmp_int32
838 // __kmp_compare_and_store32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
839 //
840 // parameters:
841 // p: %rdi
842 // cv: %esi
843 // sv: %edx
844 //
845 // return: %eax
846 .text
847 PROC __kmp_compare_and_store32
849 movl %esi, %eax // "cv"
850 lock
851 cmpxchgl %edx,(%rdi)
852 sete %al // if %eax == (%rdi) set %al = 1 else set %al = 0
853 andq $1, %rax // sign extend previous instruction for return value
854 ret
856 DEBUG_INFO __kmp_compare_and_store32
859 //------------------------------------------------------------------------
860 // FUNCTION __kmp_compare_and_store64
861 //
862 // kmp_int32
863 // __kmp_compare_and_store64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
864 //
865 // parameters:
866 // p: %rdi
867 // cv: %rsi
868 // sv: %rdx
869 // return: %eax
870 .text
871 PROC __kmp_compare_and_store64
873 movq %rsi, %rax // "cv"
874 lock
875 cmpxchgq %rdx,(%rdi)
876 sete %al // if %rax == (%rdi) set %al = 1 else set %al = 0
877 andq $1, %rax // sign extend previous instruction for return value
878 ret
880 DEBUG_INFO __kmp_compare_and_store64
882 //------------------------------------------------------------------------
883 // FUNCTION __kmp_compare_and_store_ret8
884 //
885 // kmp_int8
886 // __kmp_compare_and_store_ret8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
887 //
888 // parameters:
889 // p: %rdi
890 // cv: %esi
891 // sv: %edx
892 //
893 // return: %eax
894 .text
895 PROC __kmp_compare_and_store_ret8
897 movb %sil, %al // "cv"
898 lock
899 cmpxchgb %dl,(%rdi)
900 ret
902 DEBUG_INFO __kmp_compare_and_store_ret8
905 //------------------------------------------------------------------------
906 // FUNCTION __kmp_compare_and_store_ret16
907 //
908 // kmp_int16
909 // __kmp_compare_and_store16_ret( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
910 //
911 // parameters:
912 // p: %rdi
913 // cv: %si
914 // sv: %dx
915 //
916 // return: %eax
917 .text
918 PROC __kmp_compare_and_store_ret16
920 movw %si, %ax // "cv"
921 lock
922 cmpxchgw %dx,(%rdi)
923 ret
925 DEBUG_INFO __kmp_compare_and_store_ret16
928 //------------------------------------------------------------------------
929 // FUNCTION __kmp_compare_and_store_ret32
930 //
931 // kmp_int32
932 // __kmp_compare_and_store_ret32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
933 //
934 // parameters:
935 // p: %rdi
936 // cv: %esi
937 // sv: %edx
938 //
939 // return: %eax
940 .text
941 PROC __kmp_compare_and_store_ret32
943 movl %esi, %eax // "cv"
944 lock
945 cmpxchgl %edx,(%rdi)
946 ret
948 DEBUG_INFO __kmp_compare_and_store_ret32
951 //------------------------------------------------------------------------
952 // FUNCTION __kmp_compare_and_store_ret64
953 //
954 // kmp_int64
955 // __kmp_compare_and_store_ret64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
956 //
957 // parameters:
958 // p: %rdi
959 // cv: %rsi
960 // sv: %rdx
961 // return: %eax
962 .text
963 PROC __kmp_compare_and_store_ret64
965 movq %rsi, %rax // "cv"
966 lock
967 cmpxchgq %rdx,(%rdi)
968 ret
970 DEBUG_INFO __kmp_compare_and_store_ret64
972 # endif /* !KMP_ASM_INTRINS */
975 # if !KMP_MIC
977 # if !KMP_ASM_INTRINS
979 //------------------------------------------------------------------------
980 // FUNCTION __kmp_xchg_real32
981 //
982 // kmp_real32
983 // __kmp_xchg_real32( volatile kmp_real32 *addr, kmp_real32 data );
984 //
985 // parameters:
986 // addr: %rdi
987 // data: %xmm0 (lower 4 bytes)
988 //
989 // return: %xmm0 (lower 4 bytes)
990 .text
991 PROC __kmp_xchg_real32
993 movd %xmm0, %eax // load "data" to eax
995 lock
996 xchgl %eax, (%rdi)
998 movd %eax, %xmm0 // load old value into return register
1000 ret
1002 DEBUG_INFO __kmp_xchg_real32
1005 //------------------------------------------------------------------------
1006 // FUNCTION __kmp_xchg_real64
1007 //
1008 // kmp_real64
1009 // __kmp_xchg_real64( volatile kmp_real64 *addr, kmp_real64 data );
1010 //
1011 // parameters:
1012 // addr: %rdi
1013 // data: %xmm0 (lower 8 bytes)
1014 // return: %xmm0 (lower 8 bytes)
1015 .text
1016 PROC __kmp_xchg_real64
1018 movd %xmm0, %rax // load "data" to rax
1020 lock
1021 xchgq %rax, (%rdi)
1023 movd %rax, %xmm0 // load old value into return register
1024 ret
1026 DEBUG_INFO __kmp_xchg_real64
1029 # endif /* !KMP_MIC */
1031 # endif /* !KMP_ASM_INTRINS */
1033 //------------------------------------------------------------------------
1034 // int
1035 // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
1036 // int gtid, int tid,
1037 // int argc, void *p_argv[]
1038 // #if OMPT_SUPPORT
1039 // ,
1040 // void **exit_frame_ptr
1041 // #endif
1042 // ) {
1043 // #if OMPT_SUPPORT
1044 // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1045 // #endif
1046 //
1047 // (*pkfn)( & gtid, & tid, argv[0], ... );
1048 // return 1;
1049 // }
1050 //
1051 // note: at call to pkfn must have %rsp 128-byte aligned for compiler
1052 //
1053 // parameters:
1054 // %rdi: pkfn
1055 // %esi: gtid
1056 // %edx: tid
1057 // %ecx: argc
1058 // %r8: p_argv
1059 // %r9: &exit_frame
1060 //
1061 // locals:
1062 // __gtid: gtid parm pushed on stack so can pass >id to pkfn
1063 // __tid: tid parm pushed on stack so can pass &tid to pkfn
1064 //
1065 // reg temps:
1066 // %rax: used all over the place
1067 // %rdx: used in stack pointer alignment calculation
1068 // %r11: used to traverse p_argv array
1069 // %rsi: used as temporary for stack parameters
1070 // used as temporary for number of pkfn parms to push
1071 // %rbx: used to hold pkfn address, and zero constant, callee-save
1072 //
1073 // return: %eax (always 1/TRUE)
1074 __gtid = -16
1075 __tid = -24
1077 // -- Begin __kmp_invoke_microtask
1078 // mark_begin;
1079 .text
1080 PROC __kmp_invoke_microtask
1082 pushq %rbp // save base pointer
1083 KMP_CFI_DEF_OFFSET 16
1084 KMP_CFI_OFFSET rbp,-16
1085 movq %rsp,%rbp // establish the base pointer for this routine.
1086 KMP_CFI_REGISTER rbp
1088 #if OMPT_SUPPORT
1089 movq %rbp, (%r9) // save exit_frame
1090 #endif
1092 pushq %rbx // %rbx is callee-saved register
1093 pushq %rsi // Put gtid on stack so can pass &tgid to pkfn
1094 pushq %rdx // Put tid on stack so can pass &tid to pkfn
1096 movq %rcx, %rax // Stack alignment calculation begins; argc -> %rax
1097 movq $0, %rbx // constant for cmovs later
1098 subq $4, %rax // subtract four args passed in registers to pkfn
1099 #if KMP_MIC
1100 js KMP_LABEL(kmp_0) // jump to movq
1101 jmp KMP_LABEL(kmp_0_exit) // jump ahead
1102 KMP_LABEL(kmp_0):
1103 movq %rbx, %rax // zero negative value in %rax <- max(0, argc-4)
1104 KMP_LABEL(kmp_0_exit):
1105 #else
1106 cmovsq %rbx, %rax // zero negative value in %rax <- max(0, argc-4)
1107 #endif // KMP_MIC
1109 movq %rax, %rsi // save max(0, argc-4) -> %rsi for later
1110 shlq $3, %rax // Number of bytes used on stack: max(0, argc-4)*8
1112 movq %rsp, %rdx //
1113 subq %rax, %rdx // %rsp-(max(0,argc-4)*8) -> %rdx --
1114 // without align, stack ptr would be this
1115 movq %rdx, %rax // Save to %rax
1117 andq $0xFFFFFFFFFFFFFF80, %rax // mask off lower 7 bits (128 bytes align)
1118 subq %rax, %rdx // Amount to subtract from %rsp
1119 subq %rdx, %rsp // Prepare the stack ptr --
1120 // now %rsp will align to 128-byte boundary at call site
1122 // setup pkfn parameter reg and stack
1123 movq %rcx, %rax // argc -> %rax
1124 cmpq $0, %rsi
1125 je KMP_LABEL(kmp_invoke_pass_parms) // jump ahead if no parms to push
1126 shlq $3, %rcx // argc*8 -> %rcx
1127 movq %r8, %rdx // p_argv -> %rdx
1128 addq %rcx, %rdx // &p_argv[argc] -> %rdx
1130 movq %rsi, %rcx // max (0, argc-4) -> %rcx
1132 KMP_LABEL(kmp_invoke_push_parms):
1133 // push nth - 7th parms to pkfn on stack
1134 subq $8, %rdx // decrement p_argv pointer to previous parm
1135 movq (%rdx), %rsi // p_argv[%rcx-1] -> %rsi
1136 pushq %rsi // push p_argv[%rcx-1] onto stack (reverse order)
1137 subl $1, %ecx
1139 // C69570: "X86_64_RELOC_BRANCH not supported" error at linking on mac_32e
1140 // if the name of the label that is an operand of this jecxz starts with a dot (".");
1141 // Apple's linker does not support 1-byte length relocation;
1142 // Resolution: replace all .labelX entries with L_labelX.
1144 jecxz KMP_LABEL(kmp_invoke_pass_parms) // stop when four p_argv[] parms left
1145 jmp KMP_LABEL(kmp_invoke_push_parms)
1146 ALIGN 3
1147 KMP_LABEL(kmp_invoke_pass_parms): // put 1st - 6th parms to pkfn in registers.
1148 // order here is important to avoid trashing
1149 // registers used for both input and output parms!
1150 movq %rdi, %rbx // pkfn -> %rbx
1151 leaq __gtid(%rbp), %rdi // >id -> %rdi (store 1st parm to pkfn)
1152 leaq __tid(%rbp), %rsi // &tid -> %rsi (store 2nd parm to pkfn)
1154 movq %r8, %r11 // p_argv -> %r11
1156 #if KMP_MIC
1157 cmpq $4, %rax // argc >= 4?
1158 jns KMP_LABEL(kmp_4) // jump to movq
1159 jmp KMP_LABEL(kmp_4_exit) // jump ahead
1160 KMP_LABEL(kmp_4):
1161 movq 24(%r11), %r9 // p_argv[3] -> %r9 (store 6th parm to pkfn)
1162 KMP_LABEL(kmp_4_exit):
1164 cmpq $3, %rax // argc >= 3?
1165 jns KMP_LABEL(kmp_3) // jump to movq
1166 jmp KMP_LABEL(kmp_3_exit) // jump ahead
1167 KMP_LABEL(kmp_3):
1168 movq 16(%r11), %r8 // p_argv[2] -> %r8 (store 5th parm to pkfn)
1169 KMP_LABEL(kmp_3_exit):
1171 cmpq $2, %rax // argc >= 2?
1172 jns KMP_LABEL(kmp_2) // jump to movq
1173 jmp KMP_LABEL(kmp_2_exit) // jump ahead
1174 KMP_LABEL(kmp_2):
1175 movq 8(%r11), %rcx // p_argv[1] -> %rcx (store 4th parm to pkfn)
1176 KMP_LABEL(kmp_2_exit):
1178 cmpq $1, %rax // argc >= 1?
1179 jns KMP_LABEL(kmp_1) // jump to movq
1180 jmp KMP_LABEL(kmp_1_exit) // jump ahead
1181 KMP_LABEL(kmp_1):
1182 movq (%r11), %rdx // p_argv[0] -> %rdx (store 3rd parm to pkfn)
1183 KMP_LABEL(kmp_1_exit):
1184 #else
1185 cmpq $4, %rax // argc >= 4?
1186 cmovnsq 24(%r11), %r9 // p_argv[3] -> %r9 (store 6th parm to pkfn)
1188 cmpq $3, %rax // argc >= 3?
1189 cmovnsq 16(%r11), %r8 // p_argv[2] -> %r8 (store 5th parm to pkfn)
1191 cmpq $2, %rax // argc >= 2?
1192 cmovnsq 8(%r11), %rcx // p_argv[1] -> %rcx (store 4th parm to pkfn)
1194 cmpq $1, %rax // argc >= 1?
1195 cmovnsq (%r11), %rdx // p_argv[0] -> %rdx (store 3rd parm to pkfn)
1196 #endif // KMP_MIC
1198 call *%rbx // call (*pkfn)();
1199 movq $1, %rax // move 1 into return register;
1201 movq -8(%rbp), %rbx // restore %rbx using %rbp since %rsp was modified
1202 movq %rbp, %rsp // restore stack pointer
1203 popq %rbp // restore frame pointer
1204 KMP_CFI_DEF rsp,8
1205 ret
1207 DEBUG_INFO __kmp_invoke_microtask
1208 // -- End __kmp_invoke_microtask
1210 // kmp_uint64
1211 // __kmp_hardware_timestamp(void)
1212 .text
1213 PROC __kmp_hardware_timestamp
1214 rdtsc
1215 shlq $32, %rdx
1216 orq %rdx, %rax
1217 ret
1219 DEBUG_INFO __kmp_hardware_timestamp
1220 // -- End __kmp_hardware_timestamp
1222 //------------------------------------------------------------------------
1223 // FUNCTION __kmp_bsr32
1224 //
1225 // int
1226 // __kmp_bsr32( int );
1227 .text
1228 PROC __kmp_bsr32
1230 bsr %edi,%eax
1231 ret
1233 DEBUG_INFO __kmp_bsr32
1235 // -----------------------------------------------------------------------
1236 #endif /* KMP_ARCH_X86_64 */
1238 // '
1239 #if (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && KMP_ARCH_AARCH64
1241 //------------------------------------------------------------------------
1242 // int
1243 // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
1244 // int gtid, int tid,
1245 // int argc, void *p_argv[]
1246 // #if OMPT_SUPPORT
1247 // ,
1248 // void **exit_frame_ptr
1249 // #endif
1250 // ) {
1251 // #if OMPT_SUPPORT
1252 // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1253 // #endif
1254 //
1255 // (*pkfn)( & gtid, & tid, argv[0], ... );
1256 //
1257 // // FIXME: This is done at call-site and can be removed here.
1258 // #if OMPT_SUPPORT
1259 // *exit_frame_ptr = 0;
1260 // #endif
1261 //
1262 // return 1;
1263 // }
1264 //
1265 // parameters:
1266 // x0: pkfn
1267 // w1: gtid
1268 // w2: tid
1269 // w3: argc
1270 // x4: p_argv
1271 // x5: &exit_frame
1272 //
1273 // locals:
1274 // __gtid: gtid parm pushed on stack so can pass >id to pkfn
1275 // __tid: tid parm pushed on stack so can pass &tid to pkfn
1276 //
1277 // reg temps:
1278 // x8: used to hold pkfn address
1279 // w9: used as temporary for number of pkfn parms
1280 // x10: used to traverse p_argv array
1281 // x11: used as temporary for stack placement calculation
1282 // x12: used as temporary for stack parameters
1283 // x19: used to preserve exit_frame_ptr, callee-save
1284 //
1285 // return: w0 (always 1/TRUE)
1286 //
1288 __gtid = 4
1289 __tid = 8
1291 // -- Begin __kmp_invoke_microtask
1292 // mark_begin;
1293 .text
1294 PROC __kmp_invoke_microtask
1296 stp x29, x30, [sp, #-16]!
1297 # if OMPT_SUPPORT
1298 stp x19, x20, [sp, #-16]!
1299 # endif
1300 mov x29, sp
1302 orr w9, wzr, #1
1303 add w9, w9, w3, lsr #1
1304 sub sp, sp, w9, uxtw #4
1305 mov x11, sp
1307 mov x8, x0
1308 str w1, [x29, #-__gtid]
1309 str w2, [x29, #-__tid]
1310 mov w9, w3
1311 mov x10, x4
1312 # if OMPT_SUPPORT
1313 mov x19, x5
1314 str x29, [x19]
1315 # endif
1317 sub x0, x29, #__gtid
1318 sub x1, x29, #__tid
1320 cbz w9, KMP_LABEL(kmp_1)
1321 ldr x2, [x10]
1323 sub w9, w9, #1
1324 cbz w9, KMP_LABEL(kmp_1)
1325 ldr x3, [x10, #8]!
1327 sub w9, w9, #1
1328 cbz w9, KMP_LABEL(kmp_1)
1329 ldr x4, [x10, #8]!
1331 sub w9, w9, #1
1332 cbz w9, KMP_LABEL(kmp_1)
1333 ldr x5, [x10, #8]!
1335 sub w9, w9, #1
1336 cbz w9, KMP_LABEL(kmp_1)
1337 ldr x6, [x10, #8]!
1339 sub w9, w9, #1
1340 cbz w9, KMP_LABEL(kmp_1)
1341 ldr x7, [x10, #8]!
1343 KMP_LABEL(kmp_0):
1344 sub w9, w9, #1
1345 cbz w9, KMP_LABEL(kmp_1)
1346 ldr x12, [x10, #8]!
1347 str x12, [x11], #8
1348 b KMP_LABEL(kmp_0)
1349 KMP_LABEL(kmp_1):
1350 blr x8
1351 orr w0, wzr, #1
1352 mov sp, x29
1353 # if OMPT_SUPPORT
1354 str xzr, [x19]
1355 ldp x19, x20, [sp], #16
1356 # endif
1357 ldp x29, x30, [sp], #16
1358 ret
1360 DEBUG_INFO __kmp_invoke_microtask
1361 // -- End __kmp_invoke_microtask
1363 #endif /* (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && KMP_ARCH_AARCH64 */
1365 #if (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && KMP_ARCH_ARM
1367 //------------------------------------------------------------------------
1368 // int
1369 // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
1370 // int gtid, int tid,
1371 // int argc, void *p_argv[]
1372 // #if OMPT_SUPPORT
1373 // ,
1374 // void **exit_frame_ptr
1375 // #endif
1376 // ) {
1377 // #if OMPT_SUPPORT
1378 // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1379 // #endif
1380 //
1381 // (*pkfn)( & gtid, & tid, argv[0], ... );
1382 //
1383 // // FIXME: This is done at call-site and can be removed here.
1384 // #if OMPT_SUPPORT
1385 // *exit_frame_ptr = 0;
1386 // #endif
1387 //
1388 // return 1;
1389 // }
1390 //
1391 // parameters:
1392 // r0: pkfn
1393 // r1: gtid
1394 // r2: tid
1395 // r3: argc
1396 // r4(stack): p_argv
1397 // r5(stack): &exit_frame
1398 //
1399 // locals:
1400 // __gtid: gtid parm pushed on stack so can pass >id to pkfn
1401 // __tid: tid parm pushed on stack so can pass &tid to pkfn
1402 //
1403 // reg temps:
1404 // r4: used to hold pkfn address
1405 // r5: used as temporary for number of pkfn parms
1406 // r6: used to traverse p_argv array
1407 // r7: frame pointer (in some configurations)
1408 // r8: used as temporary for stack placement calculation
1409 // and as pointer to base of callee saved area
1410 // r9: used as temporary for stack parameters
1411 // r10: used to preserve exit_frame_ptr, callee-save
1412 // r11: frame pointer (in some configurations)
1413 //
1414 // return: r0 (always 1/TRUE)
1415 //
1417 __gtid = 4
1418 __tid = 8
1420 // -- Begin __kmp_invoke_microtask
1421 // mark_begin;
1422 .text
1423 PROC __kmp_invoke_microtask
1425 // Pushing one extra register (r3) to keep the stack aligned
1426 // for when we call pkfn below
1427 push {r3-r11,lr}
1428 // Load p_argv and &exit_frame
1429 ldr r4, [sp, #10*4]
1430 # if OMPT_SUPPORT
1431 ldr r5, [sp, #11*4]
1432 # endif
1434 # if KMP_OS_DARWIN || (defined(__thumb__) && !KMP_OS_WINDOWS)
1435 # define FP r7
1436 # define FPOFF 4*4
1437 #else
1438 # define FP r11
1439 # define FPOFF 8*4
1440 #endif
1441 add FP, sp, #FPOFF
1442 # if OMPT_SUPPORT
1443 mov r10, r5
1444 str FP, [r10]
1445 # endif
1446 mov r8, sp
1448 // Calculate how much stack to allocate, in increments of 8 bytes.
1449 // We strictly need 4*(argc-2) bytes (2 arguments are passed in
1450 // registers) but allocate 4*argc for simplicity (to avoid needing
1451 // to handle the argc<2 cases). We align the number of bytes
1452 // allocated to 8 bytes, to keep the stack aligned. (Since we
1453 // already allocate more than enough, it's ok to round down
1454 // instead of up for the alignment.) We allocate another extra
1455 // 8 bytes for gtid and tid.
1456 mov r5, #1
1457 add r5, r5, r3, lsr #1
1458 sub sp, sp, r5, lsl #3
1460 str r1, [r8, #-__gtid]
1461 str r2, [r8, #-__tid]
1462 mov r5, r3
1463 mov r6, r4
1464 mov r4, r0
1466 // Prepare the first 2 parameters to pkfn - pointers to gtid and tid
1467 // in our stack frame.
1468 sub r0, r8, #__gtid
1469 sub r1, r8, #__tid
1471 mov r8, sp
1473 // Load p_argv[0] and p_argv[1] into r2 and r3, if argc >= 1/2
1474 cmp r5, #0
1475 beq KMP_LABEL(kmp_1)
1476 ldr r2, [r6]
1478 subs r5, r5, #1
1479 beq KMP_LABEL(kmp_1)
1480 ldr r3, [r6, #4]!
1482 // Loop, loading the rest of p_argv and writing the elements on the
1483 // stack.
1484 KMP_LABEL(kmp_0):
1485 subs r5, r5, #1
1486 beq KMP_LABEL(kmp_1)
1487 ldr r12, [r6, #4]!
1488 str r12, [r8], #4
1489 b KMP_LABEL(kmp_0)
1490 KMP_LABEL(kmp_1):
1491 blx r4
1492 mov r0, #1
1494 sub r4, FP, #FPOFF
1495 mov sp, r4
1496 # undef FP
1497 # undef FPOFF
1499 # if OMPT_SUPPORT
1500 mov r1, #0
1501 str r1, [r10]
1502 # endif
1503 pop {r3-r11,pc}
1505 DEBUG_INFO __kmp_invoke_microtask
1506 // -- End __kmp_invoke_microtask
1508 #endif /* (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && KMP_ARCH_AARCH64 */
1510 #if KMP_ARCH_PPC64
1512 //------------------------------------------------------------------------
1513 // int
1514 // __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
1515 // int gtid, int tid,
1516 // int argc, void *p_argv[]
1517 // #if OMPT_SUPPORT
1518 // ,
1519 // void **exit_frame_ptr
1520 // #endif
1521 // ) {
1522 // #if OMPT_SUPPORT
1523 // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1524 // #endif
1525 //
1526 // (*pkfn)( & gtid, & tid, argv[0], ... );
1527 //
1528 // // FIXME: This is done at call-site and can be removed here.
1529 // #if OMPT_SUPPORT
1530 // *exit_frame_ptr = 0;
1531 // #endif
1532 //
1533 // return 1;
1534 // }
1535 //
1536 // parameters:
1537 // r3: pkfn
1538 // r4: gtid
1539 // r5: tid
1540 // r6: argc
1541 // r7: p_argv
1542 // r8: &exit_frame
1543 //
1544 // return: r3 (always 1/TRUE)
1545 //
1546 .text
1547 # if KMP_ARCH_PPC64_ELFv2
1548 .abiversion 2
1549 # endif
1550 .globl __kmp_invoke_microtask
1552 # if KMP_ARCH_PPC64_ELFv2
1553 .p2align 4
1554 # else
1555 .p2align 2
1556 # endif
1558 .type __kmp_invoke_microtask,@function
1560 # if KMP_ARCH_PPC64_ELFv2
1561 __kmp_invoke_microtask:
1562 .Lfunc_begin0:
1563 .Lfunc_gep0:
1564 addis 2, 12, .TOC.-.Lfunc_gep0@ha
1565 addi 2, 2, .TOC.-.Lfunc_gep0@l
1566 .Lfunc_lep0:
1567 .localentry __kmp_invoke_microtask, .Lfunc_lep0-.Lfunc_gep0
1568 # else
1569 .section .opd,"aw",@progbits
1570 __kmp_invoke_microtask:
1571 .p2align 3
1572 .quad .Lfunc_begin0
1573 .quad .TOC.@tocbase
1574 .quad 0
1575 .text
1576 .Lfunc_begin0:
1577 # endif
1579 // -- Begin __kmp_invoke_microtask
1580 // mark_begin;
1582 // We need to allocate a stack frame large enough to hold all of the parameters
1583 // on the stack for the microtask plus what this function needs. That's 48
1584 // bytes under the ELFv1 ABI (32 bytes under ELFv2), plus 8*(2 + argc) for the
1585 // parameters to the microtask, plus 8 bytes to store the values of r4 and r5,
1586 // and 8 bytes to store r31. With OMP-T support, we need an additional 8 bytes
1587 // to save r30 to hold a copy of r8.
1589 .cfi_startproc
1590 mflr 0
1591 std 31, -8(1)
1592 std 0, 16(1)
1594 // This is unusual because normally we'd set r31 equal to r1 after the stack
1595 // frame is established. In this case, however, we need to dynamically compute
1596 // the stack frame size, and so we keep a direct copy of r1 to access our
1597 // register save areas and restore the r1 value before returning.
1598 mr 31, 1
1599 .cfi_def_cfa_register r31
1600 .cfi_offset r31, -8
1601 .cfi_offset lr, 16
1603 // Compute the size necessary for the local stack frame.
1604 # if KMP_ARCH_PPC64_ELFv2
1605 li 12, 72
1606 # else
1607 li 12, 88
1608 # endif
1609 sldi 0, 6, 3
1610 add 12, 0, 12
1611 neg 12, 12
1613 // We need to make sure that the stack frame stays aligned (to 16 bytes).
1614 li 0, -16
1615 and 12, 0, 12
1617 // Establish the local stack frame.
1618 stdux 1, 1, 12
1620 # if OMPT_SUPPORT
1621 .cfi_offset r30, -16
1622 std 30, -16(31)
1623 std 1, 0(8)
1624 mr 30, 8
1625 # endif
1627 // Store gtid and tid to the stack because they're passed by reference to the microtask.
1628 stw 4, -20(31)
1629 stw 5, -24(31)
1631 mr 12, 6
1632 mr 4, 7
1634 cmpwi 0, 12, 1
1635 blt 0, .Lcall
1637 ld 5, 0(4)
1639 cmpwi 0, 12, 2
1640 blt 0, .Lcall
1642 ld 6, 8(4)
1644 cmpwi 0, 12, 3
1645 blt 0, .Lcall
1647 ld 7, 16(4)
1649 cmpwi 0, 12, 4
1650 blt 0, .Lcall
1652 ld 8, 24(4)
1654 cmpwi 0, 12, 5
1655 blt 0, .Lcall
1657 ld 9, 32(4)
1659 cmpwi 0, 12, 6
1660 blt 0, .Lcall
1662 ld 10, 40(4)
1664 cmpwi 0, 12, 7
1665 blt 0, .Lcall
1667 // There are more than 6 microtask parameters, so we need to store the
1668 // remainder to the stack.
1669 addi 12, 12, -6
1670 mtctr 12
1672 // These are set to 8 bytes before the first desired store address (we're using
1673 // pre-increment loads and stores in the loop below). The parameter save area
1674 // for the microtask begins 48 + 8*8 == 112 bytes above r1 for ELFv1 and
1675 // 32 + 8*8 == 96 bytes above r1 for ELFv2.
1676 addi 4, 4, 40
1677 # if KMP_ARCH_PPC64_ELFv2
1678 addi 12, 1, 88
1679 # else
1680 addi 12, 1, 104
1681 # endif
1683 .Lnext:
1684 ldu 0, 8(4)
1685 stdu 0, 8(12)
1686 bdnz .Lnext
1688 .Lcall:
1689 # if KMP_ARCH_PPC64_ELFv2
1690 std 2, 24(1)
1691 mr 12, 3
1692 #else
1693 std 2, 40(1)
1694 // For ELFv1, we need to load the actual function address from the function descriptor.
1695 ld 12, 0(3)
1696 ld 2, 8(3)
1697 ld 11, 16(3)
1698 #endif
1700 addi 3, 31, -20
1701 addi 4, 31, -24
1703 mtctr 12
1704 bctrl
1705 # if KMP_ARCH_PPC64_ELFv2
1706 ld 2, 24(1)
1707 # else
1708 ld 2, 40(1)
1709 # endif
1711 # if OMPT_SUPPORT
1712 li 3, 0
1713 std 3, 0(30)
1714 # endif
1716 li 3, 1
1718 # if OMPT_SUPPORT
1719 ld 30, -16(31)
1720 # endif
1722 mr 1, 31
1723 ld 0, 16(1)
1724 ld 31, -8(1)
1725 mtlr 0
1726 blr
1728 .long 0
1729 .quad 0
1730 .Lfunc_end0:
1731 .size __kmp_invoke_microtask, .Lfunc_end0-.Lfunc_begin0
1732 .cfi_endproc
1734 // -- End __kmp_invoke_microtask
1736 #endif /* KMP_ARCH_PPC64 */
1738 #if KMP_ARCH_RISCV64
1740 //------------------------------------------------------------------------
1741 //
1742 // typedef void (*microtask_t)(int *gtid, int *tid, ...);
1743 //
1744 // int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
1745 // void *p_argv[]
1746 // #if OMPT_SUPPORT
1747 // ,
1748 // void **exit_frame_ptr
1749 // #endif
1750 // ) {
1751 // #if OMPT_SUPPORT
1752 // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1753 // #endif
1754 //
1755 // (*pkfn)(>id, &tid, argv[0], ...);
1756 //
1757 // return 1;
1758 // }
1759 //
1760 // Parameters:
1761 // a0: pkfn
1762 // a1: gtid
1763 // a2: tid
1764 // a3: argc
1765 // a4: p_argv
1766 // a5: exit_frame_ptr
1767 //
1768 // Locals:
1769 // __gtid: gtid param pushed on stack so can pass >id to pkfn
1770 // __tid: tid param pushed on stack so can pass &tid to pkfn
1771 //
1772 // Temp. registers:
1773 //
1774 // t0: used to calculate the dynamic stack size / used to hold pkfn address
1775 // t1: used as temporary for stack placement calculation
1776 // t2: used as temporary for stack arguments
1777 // t3: used as temporary for number of remaining pkfn parms
1778 // t4: used to traverse p_argv array
1779 //
1780 // return: a0 (always 1/TRUE)
1781 //
1783 __gtid = -20
1784 __tid = -24
1786 // -- Begin __kmp_invoke_microtask
1787 // mark_begin;
1788 .text
1789 .globl __kmp_invoke_microtask
1790 .p2align 1
1791 .type __kmp_invoke_microtask,@function
1792 __kmp_invoke_microtask:
1793 .cfi_startproc
1795 // First, save ra and fp
1796 addi sp, sp, -16
1797 sd ra, 8(sp)
1798 sd fp, 0(sp)
1799 addi fp, sp, 16
1800 .cfi_def_cfa fp, 0
1801 .cfi_offset ra, -8
1802 .cfi_offset fp, -16
1804 // Compute the dynamic stack size:
1805 //
1806 // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by
1807 // reference
1808 // - We need 8 bytes for each argument that cannot be passed to the 'pkfn'
1809 // function by register. Given that we have 8 of such registers (a[0-7])
1810 // and two + 'argc' arguments (consider >id and &tid), we need to
1811 // reserve max(0, argc - 6)*8 extra bytes
1812 //
1813 // The total number of bytes is then max(0, argc - 6)*8 + 8
1815 // Compute max(0, argc - 6) using the following bithack:
1816 // max(0, x) = x - (x & (x >> 31)), where x := argc - 6
1817 // Source: http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
1818 addi t0, a3, -6
1819 srai t1, t0, 31
1820 and t1, t0, t1
1821 sub t0, t0, t1
1823 addi t0, t0, 1
1825 slli t0, t0, 3
1826 sub sp, sp, t0
1828 // Align the stack to 16 bytes
1829 andi sp, sp, -16
1831 mv t0, a0
1832 mv t3, a3
1833 mv t4, a4
1835 #if OMPT_SUPPORT
1836 // Save frame pointer into exit_frame
1837 sd fp, 0(a5)
1838 #endif
1840 // Prepare arguments for the pkfn function (first 8 using a0-a7 registers)
1842 sw a1, __gtid(fp)
1843 sw a2, __tid(fp)
1845 addi a0, fp, __gtid
1846 addi a1, fp, __tid
1848 beqz t3, .L_kmp_3
1849 ld a2, 0(t4)
1851 addi t3, t3, -1
1852 beqz t3, .L_kmp_3
1853 ld a3, 8(t4)
1855 addi t3, t3, -1
1856 beqz t3, .L_kmp_3
1857 ld a4, 16(t4)
1859 addi t3, t3, -1
1860 beqz t3, .L_kmp_3
1861 ld a5, 24(t4)
1863 addi t3, t3, -1
1864 beqz t3, .L_kmp_3
1865 ld a6, 32(t4)
1867 addi t3, t3, -1
1868 beqz t3, .L_kmp_3
1869 ld a7, 40(t4)
1871 // Prepare any additional argument passed through the stack
1872 addi t4, t4, 48
1873 mv t1, sp
1874 j .L_kmp_2
1875 .L_kmp_1:
1876 ld t2, 0(t4)
1877 sd t2, 0(t1)
1878 addi t4, t4, 8
1879 addi t1, t1, 8
1880 .L_kmp_2:
1881 addi t3, t3, -1
1882 bnez t3, .L_kmp_1
1884 .L_kmp_3:
1885 // Call pkfn function
1886 jalr t0
1888 // Restore stack and return
1890 addi a0, zero, 1
1892 addi sp, fp, -16
1893 ld fp, 0(sp)
1894 ld ra, 8(sp)
1895 addi sp, sp, 16
1896 ret
1897 .Lfunc_end0:
1898 .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask
1899 .cfi_endproc
1901 // -- End __kmp_invoke_microtask
1903 #endif /* KMP_ARCH_RISCV64 */
1905 #if KMP_ARCH_LOONGARCH64
1907 //------------------------------------------------------------------------
1908 //
1909 // typedef void (*microtask_t)(int *gtid, int *tid, ...);
1910 //
1911 // int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
1912 // void *p_argv[]
1913 // #if OMPT_SUPPORT
1914 // ,
1915 // void **exit_frame_ptr
1916 // #endif
1917 // ) {
1918 // #if OMPT_SUPPORT
1919 // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1920 // #endif
1921 //
1922 // (*pkfn)(>id, &tid, argv[0], ...);
1923 //
1924 // return 1;
1925 // }
1926 //
1927 // Parameters:
1928 // a0: pkfn
1929 // a1: gtid
1930 // a2: tid
1931 // a3: argc
1932 // a4: p_argv
1933 // a5: exit_frame_ptr
1934 //
1935 // Locals:
1936 // __gtid: gtid param pushed on stack so can pass >id to pkfn
1937 // __tid: tid param pushed on stack so can pass &tid to pkfn
1938 //
1939 // Temp registers:
1940 //
1941 // t0: used to calculate the dynamic stack size / used to hold pkfn address
1942 // t1: used as temporary for stack placement calculation
1943 // t2: used as temporary for stack arguments
1944 // t3: used as temporary for number of remaining pkfn parms
1945 // t4: used to traverse p_argv array
1946 //
1947 // return: a0 (always 1/TRUE)
1948 //
1950 // -- Begin __kmp_invoke_microtask
1951 // mark_begin;
1952 .text
1953 .globl __kmp_invoke_microtask
1954 .p2align 2
1955 .type __kmp_invoke_microtask,@function
1956 __kmp_invoke_microtask:
1957 .cfi_startproc
1959 // First, save ra and fp
1960 addi.d $sp, $sp, -16
1961 st.d $ra, $sp, 8
1962 st.d $fp, $sp, 0
1963 addi.d $fp, $sp, 16
1964 .cfi_def_cfa 22, 0
1965 .cfi_offset 1, -8
1966 .cfi_offset 22, -16
1968 // Compute the dynamic stack size:
1969 //
1970 // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by
1971 // reference
1972 // - We need 8 bytes for each argument that cannot be passed to the 'pkfn'
1973 // function by register. Given that we have 8 of such registers (a[0-7])
1974 // and two + 'argc' arguments (consider >id and &tid), we need to
1975 // reserve max(0, argc - 6)*8 extra bytes
1976 //
1977 // The total number of bytes is then max(0, argc - 6)*8 + 8
1979 addi.d $t0, $a3, -6
1980 slt $t1, $t0, $zero
1981 masknez $t0, $t0, $t1
1982 addi.d $t0, $t0, 1
1983 slli.d $t0, $t0, 3
1984 sub.d $sp, $sp, $t0
1986 // Align the stack to 16 bytes
1987 bstrins.d $sp, $zero, 3, 0
1989 move $t0, $a0
1990 move $t3, $a3
1991 move $t4, $a4
1993 #if OMPT_SUPPORT
1994 // Save frame pointer into exit_frame
1995 st.d $fp, $a5, 0
1996 #endif
1998 // Prepare arguments for the pkfn function (first 8 using a0-a7 registers)
2000 st.w $a1, $fp, -20
2001 st.w $a2, $fp, -24
2003 addi.d $a0, $fp, -20
2004 addi.d $a1, $fp, -24
2006 beqz $t3, .L_kmp_3
2007 ld.d $a2, $t4, 0
2009 addi.d $t3, $t3, -1
2010 beqz $t3, .L_kmp_3
2011 ld.d $a3, $t4, 8
2013 addi.d $t3, $t3, -1
2014 beqz $t3, .L_kmp_3
2015 ld.d $a4, $t4, 16
2017 addi.d $t3, $t3, -1
2018 beqz $t3, .L_kmp_3
2019 ld.d $a5, $t4, 24
2021 addi.d $t3, $t3, -1
2022 beqz $t3, .L_kmp_3
2023 ld.d $a6, $t4, 32
2025 addi.d $t3, $t3, -1
2026 beqz $t3, .L_kmp_3
2027 ld.d $a7, $t4, 40
2029 // Prepare any additional argument passed through the stack
2030 addi.d $t4, $t4, 48
2031 move $t1, $sp
2032 b .L_kmp_2
2033 .L_kmp_1:
2034 ld.d $t2, $t4, 0
2035 st.d $t2, $t1, 0
2036 addi.d $t4, $t4, 8
2037 addi.d $t1, $t1, 8
2038 .L_kmp_2:
2039 addi.d $t3, $t3, -1
2040 bnez $t3, .L_kmp_1
2042 .L_kmp_3:
2043 // Call pkfn function
2044 jirl $ra, $t0, 0
2046 // Restore stack and return
2048 addi.d $a0, $zero, 1
2050 addi.d $sp, $fp, -16
2051 ld.d $fp, $sp, 0
2052 ld.d $ra, $sp, 8
2053 addi.d $sp, $sp, 16
2054 jr $ra
2055 .Lfunc_end0:
2056 .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask
2057 .cfi_endproc
2059 // -- End __kmp_invoke_microtask
2061 #endif /* KMP_ARCH_LOONGARCH64 */
2063 #if KMP_ARCH_VE
2065 //------------------------------------------------------------------------
2066 //
2067 // typedef void (*microtask_t)(int *gtid, int *tid, ...);
2068 //
2069 // int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2070 // void *p_argv[]
2071 // #if OMPT_SUPPORT
2072 // ,
2073 // void **exit_frame_ptr
2074 // #endif
2075 // ) {
2076 // #if OMPT_SUPPORT
2077 // *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2078 // #endif
2079 //
2080 // (*pkfn)(>id, &tid, argv[0], ...);
2081 //
2082 // return 1;
2083 // }
2084 //
2085 // Parameters:
2086 // s0: pkfn
2087 // s1: gtid
2088 // s2: tid
2089 // s3: argc
2090 // s4: p_argv
2091 // s5: exit_frame_ptr
2092 //
2093 // Locals:
2094 // __gtid: gtid param pushed on stack so can pass >id to pkfn
2095 // __tid: tid param pushed on stack so can pass &tid to pkfn
2096 //
2097 // Temp. registers:
2098 //
2099 // s34: used to calculate the dynamic stack size
2100 // s35: used as temporary for stack placement calculation
2101 // s36: used as temporary for stack arguments
2102 // s37: used as temporary for number of remaining pkfn parms
2103 // s38: used to traverse p_argv array
2104 //
2105 // return: s0 (always 1/TRUE)
2106 //
2108 __gtid = -4
2109 __tid = -8
2111 // -- Begin __kmp_invoke_microtask
2112 // mark_begin;
2113 .text
2114 .globl __kmp_invoke_microtask
2115 // A function requires 8 bytes align.
2116 .p2align 3
2117 .type __kmp_invoke_microtask,@function
2118 __kmp_invoke_microtask:
2119 .cfi_startproc
2121 // First, save fp and lr. VE stores them at caller stack frame.
2122 st %fp, 0(, %sp)
2123 st %lr, 8(, %sp)
2124 or %fp, 0, %sp
2125 .cfi_def_cfa %fp, 0
2126 .cfi_offset %lr, 8
2127 .cfi_offset %fp, 0
2129 // Compute the dynamic stack size:
2130 //
2131 // - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them
2132 // by reference
2133 // - We need 8 bytes for whole arguments. We have two + 'argc'
2134 // arguments (condider >id and &tid). We need to reserve
2135 // (argc + 2) * 8 bytes.
2136 // - We need 176 bytes for RSA and others
2137 //
2138 // The total number of bytes is then (argc + 2) * 8 + 8 + 176.
2139 //
2140 // |------------------------------|
2141 // | return address of callee | 8(%fp)
2142 // |------------------------------|
2143 // | frame pointer of callee | 0(%fp)
2144 // |------------------------------| <------------------ %fp
2145 // | __tid / __gtid | -8(%fp) / -4(%fp)
2146 // |------------------------------|
2147 // | argc+2 for arguments | 176(%sp)
2148 // |------------------------------|
2149 // | RSA |
2150 // |------------------------------|
2151 // | return address |
2152 // |------------------------------|
2153 // | frame pointer |
2154 // |------------------------------| <------------------ %sp
2156 adds.w.sx %s34, 2, %s3
2157 sll %s34, %s34, 3
2158 lea %s34, 184(, %s34)
2159 subs.l %sp, %sp, %s34
2161 // Align the stack to 16 bytes.
2162 and %sp, -16, %sp
2164 // Save pkfn.
2165 or %s12, 0, %s0
2167 // Call host to allocate stack if it is necessary.
2168 brge.l %sp, %sl, .L_kmp_pass
2169 ld %s61, 24(, %tp)
2170 lea %s63, 0x13b
2171 shm.l %s63, 0(%s61)
2172 shm.l %sl, 8(%s61)
2173 shm.l %sp, 16(%s61)
2174 monc
2176 .L_kmp_pass:
2177 lea %s35, 176(, %sp)
2178 adds.w.sx %s37, 0, %s3
2179 or %s38, 0, %s4
2181 #if OMPT_SUPPORT
2182 // Save frame pointer into exit_frame.
2183 st %fp, 0(%s5)
2184 #endif
2186 // Prepare arguments for the pkfn function (first 8 using s0-s7
2187 // registers, but need to store stack also because of varargs).
2189 stl %s1, __gtid(%fp)
2190 stl %s2, __tid(%fp)
2192 adds.l %s0, __gtid, %fp
2193 st %s0, 0(, %s35)
2194 adds.l %s1, __tid, %fp
2195 st %s1, 8(, %s35)
2197 breq.l 0, %s37, .L_kmp_call
2198 ld %s2, 0(, %s38)
2199 st %s2, 16(, %s35)
2201 breq.l 1, %s37, .L_kmp_call
2202 ld %s3, 8(, %s38)
2203 st %s3, 24(, %s35)
2205 breq.l 2, %s37, .L_kmp_call
2206 ld %s4, 16(, %s38)
2207 st %s4, 32(, %s35)
2209 breq.l 3, %s37, .L_kmp_call
2210 ld %s5, 24(, %s38)
2211 st %s5, 40(, %s35)
2213 breq.l 4, %s37, .L_kmp_call
2214 ld %s6, 32(, %s38)
2215 st %s6, 48(, %s35)
2217 breq.l 5, %s37, .L_kmp_call
2218 ld %s7, 40(, %s38)
2219 st %s7, 56(, %s35)
2221 breq.l 6, %s37, .L_kmp_call
2223 // Prepare any additional argument passed through the stack.
2224 adds.l %s37, -6, %s37
2225 lea %s38, 48(, %s38)
2226 lea %s35, 64(, %s35)
2227 .L_kmp_loop:
2228 ld %s36, 0(, %s38)
2229 st %s36, 0(, %s35)
2230 adds.l %s37, -1, %s37
2231 adds.l %s38, 8, %s38
2232 adds.l %s35, 8, %s35
2233 brne.l 0, %s37, .L_kmp_loop
2235 .L_kmp_call:
2236 // Call pkfn function.
2237 bsic %lr, (, %s12)
2239 // Return value.
2240 lea %s0, 1
2242 // Restore stack and return.
2243 or %sp, 0, %fp
2244 ld %lr, 8(, %sp)
2245 ld %fp, 0(, %sp)
2246 b.l.t (, %lr)
2247 .Lfunc_end0:
2248 .size __kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask
2249 .cfi_endproc
2251 // -- End __kmp_invoke_microtask
2253 #endif /* KMP_ARCH_VE */
2255 #if KMP_ARCH_ARM || KMP_ARCH_MIPS
2256 .data
2257 COMMON .gomp_critical_user_, 32, 3
2258 .data
2259 .align 4
2260 .global __kmp_unnamed_critical_addr
2261 __kmp_unnamed_critical_addr:
2262 .4byte .gomp_critical_user_
2263 #ifdef __ELF__
2264 .size __kmp_unnamed_critical_addr,4
2265 #endif
2266 #endif /* KMP_ARCH_ARM */
2268 #if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 || \
2269 KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || KMP_ARCH_VE
2270 #ifndef KMP_PREFIX_UNDERSCORE
2271 # define KMP_PREFIX_UNDERSCORE(x) x
2272 #endif
2273 .data
2274 COMMON .gomp_critical_user_, 32, 3
2275 .data
2276 .align 8
2277 .global KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr)
2278 KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr):
2279 .8byte .gomp_critical_user_
2280 #ifdef __ELF__
2281 .size KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),8
2282 #endif
2283 #endif /* KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 ||
2284 KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || KMP_ARCH_VE */
2286 #if KMP_OS_LINUX
2287 # if KMP_ARCH_ARM || KMP_ARCH_AARCH64
2288 .section .note.GNU-stack,"",%progbits
2289 # else
2290 .section .note.GNU-stack,"",@progbits
2291 # endif
2292 #endif