1 #ifndef _ASM_IA64_GCC_INTRIN_H
2 #define _ASM_IA64_GCC_INTRIN_H
5 * Copyright (C) 2002,2003 Jun Nakajima <jun.nakajima@intel.com>
6 * Copyright (C) 2002,2003 Suresh Siddha <suresh.b.siddha@intel.com>
9 #include <linux/compiler.h>
11 /* define this macro to get some asm stmts included in 'c' files */
14 /* Optimization barrier */
15 /* The "volatile" is due to gcc bugs */
16 #define ia64_barrier() asm volatile ("":::"memory")
18 #define ia64_stop() asm volatile (";;"::)
20 #define ia64_invala_gr(regnum) asm volatile ("invala.e r%0" :: "i"(regnum))
22 #define ia64_invala_fr(regnum) asm volatile ("invala.e f%0" :: "i"(regnum))
24 extern void ia64_bad_param_for_setreg (void);
25 extern void ia64_bad_param_for_getreg (void);
28 register unsigned long ia64_r13
asm ("r13") __used
;
31 #define ia64_setreg(regnum, val) \
34 case _IA64_REG_PSR_L: \
35 asm volatile ("mov psr.l=%0" :: "r"(val) : "memory"); \
37 case _IA64_REG_AR_KR0 ... _IA64_REG_AR_EC: \
38 asm volatile ("mov ar%0=%1" :: \
39 "i" (regnum - _IA64_REG_AR_KR0), \
40 "r"(val): "memory"); \
42 case _IA64_REG_CR_DCR ... _IA64_REG_CR_LRR1: \
43 asm volatile ("mov cr%0=%1" :: \
44 "i" (regnum - _IA64_REG_CR_DCR), \
45 "r"(val): "memory" ); \
48 asm volatile ("mov r12=%0" :: \
49 "r"(val): "memory"); \
52 asm volatile ("mov gp=%0" :: "r"(val) : "memory"); \
55 ia64_bad_param_for_setreg(); \
60 #define ia64_getreg(regnum) \
62 __u64 ia64_intri_res; \
66 asm volatile ("mov %0=gp" : "=r"(ia64_intri_res)); \
69 asm volatile ("mov %0=ip" : "=r"(ia64_intri_res)); \
72 asm volatile ("mov %0=psr" : "=r"(ia64_intri_res)); \
74 case _IA64_REG_TP: /* for current() */ \
75 ia64_intri_res = ia64_r13; \
77 case _IA64_REG_AR_KR0 ... _IA64_REG_AR_EC: \
78 asm volatile ("mov %0=ar%1" : "=r" (ia64_intri_res) \
79 : "i"(regnum - _IA64_REG_AR_KR0)); \
81 case _IA64_REG_CR_DCR ... _IA64_REG_CR_LRR1: \
82 asm volatile ("mov %0=cr%1" : "=r" (ia64_intri_res) \
83 : "i" (regnum - _IA64_REG_CR_DCR)); \
86 asm volatile ("mov %0=sp" : "=r" (ia64_intri_res)); \
89 ia64_bad_param_for_getreg(); \
95 #define ia64_hint_pause 0
97 #define ia64_hint(mode) \
100 case ia64_hint_pause: \
101 asm volatile ("hint @pause" ::: "memory"); \
107 /* Integer values for mux1 instruction */
108 #define ia64_mux1_brcst 0
109 #define ia64_mux1_mix 8
110 #define ia64_mux1_shuf 9
111 #define ia64_mux1_alt 10
112 #define ia64_mux1_rev 11
114 #define ia64_mux1(x, mode) \
116 __u64 ia64_intri_res; \
119 case ia64_mux1_brcst: \
120 asm ("mux1 %0=%1,@brcst" : "=r" (ia64_intri_res) : "r" (x)); \
122 case ia64_mux1_mix: \
123 asm ("mux1 %0=%1,@mix" : "=r" (ia64_intri_res) : "r" (x)); \
125 case ia64_mux1_shuf: \
126 asm ("mux1 %0=%1,@shuf" : "=r" (ia64_intri_res) : "r" (x)); \
128 case ia64_mux1_alt: \
129 asm ("mux1 %0=%1,@alt" : "=r" (ia64_intri_res) : "r" (x)); \
131 case ia64_mux1_rev: \
132 asm ("mux1 %0=%1,@rev" : "=r" (ia64_intri_res) : "r" (x)); \
138 #if __GNUC__ >= 4 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)
139 # define ia64_popcnt(x) __builtin_popcountl(x)
141 # define ia64_popcnt(x) \
143 __u64 ia64_intri_res; \
144 asm ("popcnt %0=%1" : "=r" (ia64_intri_res) : "r" (x)); \
150 #define ia64_getf_exp(x) \
152 long ia64_intri_res; \
154 asm ("getf.exp %0=%1" : "=r"(ia64_intri_res) : "f"(x)); \
159 #define ia64_shrp(a, b, count) \
161 __u64 ia64_intri_res; \
162 asm ("shrp %0=%1,%2,%3" : "=r"(ia64_intri_res) : "r"(a), "r"(b), "i"(count)); \
166 #define ia64_ldfs(regnum, x) \
168 register double __f__ asm ("f"#regnum); \
169 asm volatile ("ldfs %0=[%1]" :"=f"(__f__): "r"(x)); \
172 #define ia64_ldfd(regnum, x) \
174 register double __f__ asm ("f"#regnum); \
175 asm volatile ("ldfd %0=[%1]" :"=f"(__f__): "r"(x)); \
178 #define ia64_ldfe(regnum, x) \
180 register double __f__ asm ("f"#regnum); \
181 asm volatile ("ldfe %0=[%1]" :"=f"(__f__): "r"(x)); \
184 #define ia64_ldf8(regnum, x) \
186 register double __f__ asm ("f"#regnum); \
187 asm volatile ("ldf8 %0=[%1]" :"=f"(__f__): "r"(x)); \
190 #define ia64_ldf_fill(regnum, x) \
192 register double __f__ asm ("f"#regnum); \
193 asm volatile ("ldf.fill %0=[%1]" :"=f"(__f__): "r"(x)); \
196 #define ia64_st4_rel_nta(m, val) \
198 asm volatile ("st4.rel.nta [%0] = %1\n\t" :: "r"(m), "r"(val)); \
201 #define ia64_stfs(x, regnum) \
203 register double __f__ asm ("f"#regnum); \
204 asm volatile ("stfs [%0]=%1" :: "r"(x), "f"(__f__) : "memory"); \
207 #define ia64_stfd(x, regnum) \
209 register double __f__ asm ("f"#regnum); \
210 asm volatile ("stfd [%0]=%1" :: "r"(x), "f"(__f__) : "memory"); \
213 #define ia64_stfe(x, regnum) \
215 register double __f__ asm ("f"#regnum); \
216 asm volatile ("stfe [%0]=%1" :: "r"(x), "f"(__f__) : "memory"); \
219 #define ia64_stf8(x, regnum) \
221 register double __f__ asm ("f"#regnum); \
222 asm volatile ("stf8 [%0]=%1" :: "r"(x), "f"(__f__) : "memory"); \
225 #define ia64_stf_spill(x, regnum) \
227 register double __f__ asm ("f"#regnum); \
228 asm volatile ("stf.spill [%0]=%1" :: "r"(x), "f"(__f__) : "memory"); \
231 #define ia64_fetchadd4_acq(p, inc) \
234 __u64 ia64_intri_res; \
235 asm volatile ("fetchadd4.acq %0=[%1],%2" \
236 : "=r"(ia64_intri_res) : "r"(p), "i" (inc) \
242 #define ia64_fetchadd4_rel(p, inc) \
244 __u64 ia64_intri_res; \
245 asm volatile ("fetchadd4.rel %0=[%1],%2" \
246 : "=r"(ia64_intri_res) : "r"(p), "i" (inc) \
252 #define ia64_fetchadd8_acq(p, inc) \
255 __u64 ia64_intri_res; \
256 asm volatile ("fetchadd8.acq %0=[%1],%2" \
257 : "=r"(ia64_intri_res) : "r"(p), "i" (inc) \
263 #define ia64_fetchadd8_rel(p, inc) \
265 __u64 ia64_intri_res; \
266 asm volatile ("fetchadd8.rel %0=[%1],%2" \
267 : "=r"(ia64_intri_res) : "r"(p), "i" (inc) \
273 #define ia64_xchg1(ptr,x) \
275 __u64 ia64_intri_res; \
276 asm volatile ("xchg1 %0=[%1],%2" \
277 : "=r" (ia64_intri_res) : "r" (ptr), "r" (x) : "memory"); \
281 #define ia64_xchg2(ptr,x) \
283 __u64 ia64_intri_res; \
284 asm volatile ("xchg2 %0=[%1],%2" : "=r" (ia64_intri_res) \
285 : "r" (ptr), "r" (x) : "memory"); \
289 #define ia64_xchg4(ptr,x) \
291 __u64 ia64_intri_res; \
292 asm volatile ("xchg4 %0=[%1],%2" : "=r" (ia64_intri_res) \
293 : "r" (ptr), "r" (x) : "memory"); \
297 #define ia64_xchg8(ptr,x) \
299 __u64 ia64_intri_res; \
300 asm volatile ("xchg8 %0=[%1],%2" : "=r" (ia64_intri_res) \
301 : "r" (ptr), "r" (x) : "memory"); \
305 #define ia64_cmpxchg1_acq(ptr, new, old) \
307 __u64 ia64_intri_res; \
308 asm volatile ("mov ar.ccv=%0;;" :: "rO"(old)); \
309 asm volatile ("cmpxchg1.acq %0=[%1],%2,ar.ccv": \
310 "=r"(ia64_intri_res) : "r"(ptr), "r"(new) : "memory"); \
314 #define ia64_cmpxchg1_rel(ptr, new, old) \
316 __u64 ia64_intri_res; \
317 asm volatile ("mov ar.ccv=%0;;" :: "rO"(old)); \
318 asm volatile ("cmpxchg1.rel %0=[%1],%2,ar.ccv": \
319 "=r"(ia64_intri_res) : "r"(ptr), "r"(new) : "memory"); \
323 #define ia64_cmpxchg2_acq(ptr, new, old) \
325 __u64 ia64_intri_res; \
326 asm volatile ("mov ar.ccv=%0;;" :: "rO"(old)); \
327 asm volatile ("cmpxchg2.acq %0=[%1],%2,ar.ccv": \
328 "=r"(ia64_intri_res) : "r"(ptr), "r"(new) : "memory"); \
332 #define ia64_cmpxchg2_rel(ptr, new, old) \
334 __u64 ia64_intri_res; \
335 asm volatile ("mov ar.ccv=%0;;" :: "rO"(old)); \
337 asm volatile ("cmpxchg2.rel %0=[%1],%2,ar.ccv": \
338 "=r"(ia64_intri_res) : "r"(ptr), "r"(new) : "memory"); \
342 #define ia64_cmpxchg4_acq(ptr, new, old) \
344 __u64 ia64_intri_res; \
345 asm volatile ("mov ar.ccv=%0;;" :: "rO"(old)); \
346 asm volatile ("cmpxchg4.acq %0=[%1],%2,ar.ccv": \
347 "=r"(ia64_intri_res) : "r"(ptr), "r"(new) : "memory"); \
351 #define ia64_cmpxchg4_rel(ptr, new, old) \
353 __u64 ia64_intri_res; \
354 asm volatile ("mov ar.ccv=%0;;" :: "rO"(old)); \
355 asm volatile ("cmpxchg4.rel %0=[%1],%2,ar.ccv": \
356 "=r"(ia64_intri_res) : "r"(ptr), "r"(new) : "memory"); \
360 #define ia64_cmpxchg8_acq(ptr, new, old) \
362 __u64 ia64_intri_res; \
363 asm volatile ("mov ar.ccv=%0;;" :: "rO"(old)); \
364 asm volatile ("cmpxchg8.acq %0=[%1],%2,ar.ccv": \
365 "=r"(ia64_intri_res) : "r"(ptr), "r"(new) : "memory"); \
369 #define ia64_cmpxchg8_rel(ptr, new, old) \
371 __u64 ia64_intri_res; \
372 asm volatile ("mov ar.ccv=%0;;" :: "rO"(old)); \
374 asm volatile ("cmpxchg8.rel %0=[%1],%2,ar.ccv": \
375 "=r"(ia64_intri_res) : "r"(ptr), "r"(new) : "memory"); \
379 #define ia64_mf() asm volatile ("mf" ::: "memory")
380 #define ia64_mfa() asm volatile ("mf.a" ::: "memory")
382 #define ia64_invala() asm volatile ("invala" ::: "memory")
384 #define ia64_thash(addr) \
386 __u64 ia64_intri_res; \
387 asm volatile ("thash %0=%1" : "=r"(ia64_intri_res) : "r" (addr)); \
391 #define ia64_srlz_i() asm volatile (";; srlz.i ;;" ::: "memory")
392 #define ia64_srlz_d() asm volatile (";; srlz.d" ::: "memory");
394 #ifdef HAVE_SERIALIZE_DIRECTIVE
395 # define ia64_dv_serialize_data() asm volatile (".serialize.data");
396 # define ia64_dv_serialize_instruction() asm volatile (".serialize.instruction");
398 # define ia64_dv_serialize_data()
399 # define ia64_dv_serialize_instruction()
402 #define ia64_nop(x) asm volatile ("nop %0"::"i"(x));
404 #define ia64_itci(addr) asm volatile ("itc.i %0;;" :: "r"(addr) : "memory")
406 #define ia64_itcd(addr) asm volatile ("itc.d %0;;" :: "r"(addr) : "memory")
409 #define ia64_itri(trnum, addr) asm volatile ("itr.i itr[%0]=%1" \
410 :: "r"(trnum), "r"(addr) : "memory")
412 #define ia64_itrd(trnum, addr) asm volatile ("itr.d dtr[%0]=%1" \
413 :: "r"(trnum), "r"(addr) : "memory")
415 #define ia64_tpa(addr) \
418 asm volatile ("tpa %0 = %1" : "=r"(ia64_pa) : "r"(addr) : "memory"); \
422 #define __ia64_set_dbr(index, val) \
423 asm volatile ("mov dbr[%0]=%1" :: "r"(index), "r"(val) : "memory")
425 #define ia64_set_ibr(index, val) \
426 asm volatile ("mov ibr[%0]=%1" :: "r"(index), "r"(val) : "memory")
428 #define ia64_set_pkr(index, val) \
429 asm volatile ("mov pkr[%0]=%1" :: "r"(index), "r"(val) : "memory")
431 #define ia64_set_pmc(index, val) \
432 asm volatile ("mov pmc[%0]=%1" :: "r"(index), "r"(val) : "memory")
434 #define ia64_set_pmd(index, val) \
435 asm volatile ("mov pmd[%0]=%1" :: "r"(index), "r"(val) : "memory")
437 #define ia64_set_rr(index, val) \
438 asm volatile ("mov rr[%0]=%1" :: "r"(index), "r"(val) : "memory");
440 #define ia64_get_cpuid(index) \
442 __u64 ia64_intri_res; \
443 asm volatile ("mov %0=cpuid[%r1]" : "=r"(ia64_intri_res) : "rO"(index)); \
447 #define __ia64_get_dbr(index) \
449 __u64 ia64_intri_res; \
450 asm volatile ("mov %0=dbr[%1]" : "=r"(ia64_intri_res) : "r"(index)); \
454 #define ia64_get_ibr(index) \
456 __u64 ia64_intri_res; \
457 asm volatile ("mov %0=ibr[%1]" : "=r"(ia64_intri_res) : "r"(index)); \
461 #define ia64_get_pkr(index) \
463 __u64 ia64_intri_res; \
464 asm volatile ("mov %0=pkr[%1]" : "=r"(ia64_intri_res) : "r"(index)); \
468 #define ia64_get_pmc(index) \
470 __u64 ia64_intri_res; \
471 asm volatile ("mov %0=pmc[%1]" : "=r"(ia64_intri_res) : "r"(index)); \
476 #define ia64_get_pmd(index) \
478 __u64 ia64_intri_res; \
479 asm volatile ("mov %0=pmd[%1]" : "=r"(ia64_intri_res) : "r"(index)); \
483 #define ia64_get_rr(index) \
485 __u64 ia64_intri_res; \
486 asm volatile ("mov %0=rr[%1]" : "=r"(ia64_intri_res) : "r" (index)); \
490 #define ia64_fc(addr) asm volatile ("fc %0" :: "r"(addr) : "memory")
493 #define ia64_sync_i() asm volatile (";; sync.i" ::: "memory")
495 #define ia64_ssm(mask) asm volatile ("ssm %0":: "i"((mask)) : "memory")
496 #define ia64_rsm(mask) asm volatile ("rsm %0":: "i"((mask)) : "memory")
497 #define ia64_sum(mask) asm volatile ("sum %0":: "i"((mask)) : "memory")
498 #define ia64_rum(mask) asm volatile ("rum %0":: "i"((mask)) : "memory")
500 #define ia64_ptce(addr) asm volatile ("ptc.e %0" :: "r"(addr))
502 #define ia64_ptcga(addr, size) \
504 asm volatile ("ptc.ga %0,%1" :: "r"(addr), "r"(size) : "memory"); \
505 ia64_dv_serialize_data(); \
508 #define ia64_ptcl(addr, size) \
510 asm volatile ("ptc.l %0,%1" :: "r"(addr), "r"(size) : "memory"); \
511 ia64_dv_serialize_data(); \
514 #define ia64_ptri(addr, size) \
515 asm volatile ("ptr.i %0,%1" :: "r"(addr), "r"(size) : "memory")
517 #define ia64_ptrd(addr, size) \
518 asm volatile ("ptr.d %0,%1" :: "r"(addr), "r"(size) : "memory")
520 /* Values for lfhint in ia64_lfetch and ia64_lfetch_fault */
522 #define ia64_lfhint_none 0
523 #define ia64_lfhint_nt1 1
524 #define ia64_lfhint_nt2 2
525 #define ia64_lfhint_nta 3
527 #define ia64_lfetch(lfhint, y) \
530 case ia64_lfhint_none: \
531 asm volatile ("lfetch [%0]" : : "r"(y)); \
533 case ia64_lfhint_nt1: \
534 asm volatile ("lfetch.nt1 [%0]" : : "r"(y)); \
536 case ia64_lfhint_nt2: \
537 asm volatile ("lfetch.nt2 [%0]" : : "r"(y)); \
539 case ia64_lfhint_nta: \
540 asm volatile ("lfetch.nta [%0]" : : "r"(y)); \
545 #define ia64_lfetch_excl(lfhint, y) \
548 case ia64_lfhint_none: \
549 asm volatile ("lfetch.excl [%0]" :: "r"(y)); \
551 case ia64_lfhint_nt1: \
552 asm volatile ("lfetch.excl.nt1 [%0]" :: "r"(y)); \
554 case ia64_lfhint_nt2: \
555 asm volatile ("lfetch.excl.nt2 [%0]" :: "r"(y)); \
557 case ia64_lfhint_nta: \
558 asm volatile ("lfetch.excl.nta [%0]" :: "r"(y)); \
563 #define ia64_lfetch_fault(lfhint, y) \
566 case ia64_lfhint_none: \
567 asm volatile ("lfetch.fault [%0]" : : "r"(y)); \
569 case ia64_lfhint_nt1: \
570 asm volatile ("lfetch.fault.nt1 [%0]" : : "r"(y)); \
572 case ia64_lfhint_nt2: \
573 asm volatile ("lfetch.fault.nt2 [%0]" : : "r"(y)); \
575 case ia64_lfhint_nta: \
576 asm volatile ("lfetch.fault.nta [%0]" : : "r"(y)); \
581 #define ia64_lfetch_fault_excl(lfhint, y) \
584 case ia64_lfhint_none: \
585 asm volatile ("lfetch.fault.excl [%0]" :: "r"(y)); \
587 case ia64_lfhint_nt1: \
588 asm volatile ("lfetch.fault.excl.nt1 [%0]" :: "r"(y)); \
590 case ia64_lfhint_nt2: \
591 asm volatile ("lfetch.fault.excl.nt2 [%0]" :: "r"(y)); \
593 case ia64_lfhint_nta: \
594 asm volatile ("lfetch.fault.excl.nta [%0]" :: "r"(y)); \
599 #define ia64_intrin_local_irq_restore(x) \
601 asm volatile (";; cmp.ne p6,p7=%0,r0;;" \
605 :: "r"((x)) : "p6", "p7", "memory"); \
608 #endif /* _ASM_IA64_GCC_INTRIN_H */