tools/virtio/ringtest/main.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2016 Red Hat, Inc.
   4  * Author: Michael S. Tsirkin <mst@redhat.com>
   5  *
   6  * Common macros and functions for ring benchmarking.
   7  */
   8 #ifndef MAIN_H
   9 #define MAIN_H
  10
  11 #include <assert.h>
  12 #include <stdbool.h>
  13
  14 extern int param;
  15
  16 extern bool do_exit;
  17
  18 #if defined(__x86_64__) || defined(__i386__)
  19 #include "x86intrin.h"
  20
  21 static inline void wait_cycles(unsigned long long cycles)
  22 {
  23         unsigned long long t;
  24
  25         t = __rdtsc();
  26         while (__rdtsc() - t < cycles) {}
  27 }
  28
  29 #define VMEXIT_CYCLES 500
  30 #define VMENTRY_CYCLES 500
  31
  32 #elif defined(__s390x__)
  33 static inline void wait_cycles(unsigned long long cycles)
  34 {
  35         asm volatile("0: brctg %0,0b" : : "d" (cycles));
  36 }
  37
  38 /* tweak me */
  39 #define VMEXIT_CYCLES 200
  40 #define VMENTRY_CYCLES 200
  41
  42 #else
  43 static inline void wait_cycles(unsigned long long cycles)
  44 {
  45         _Exit(5);
  46 }
  47 #define VMEXIT_CYCLES 0
  48 #define VMENTRY_CYCLES 0
  49 #endif
  50
  51 static inline void vmexit(void)
  52 {
  53         if (!do_exit)
  54                 return;
  55
  56         wait_cycles(VMEXIT_CYCLES);
  57 }
  58 static inline void vmentry(void)
  59 {
  60         if (!do_exit)
  61                 return;
  62
  63         wait_cycles(VMENTRY_CYCLES);
  64 }
  65
  66 /* implemented by ring */
  67 void alloc_ring(void);
  68 /* guest side */
  69 int add_inbuf(unsigned, void *, void *);
  70 void *get_buf(unsigned *, void **);
  71 void disable_call();
  72 bool used_empty();
  73 bool enable_call();
  74 void kick_available();
  75 /* host side */
  76 void disable_kick();
  77 bool avail_empty();
  78 bool enable_kick();
  79 bool use_buf(unsigned *, void **);
  80 void call_used();
  81
  82 /* implemented by main */
  83 extern bool do_sleep;
  84 void kick(void);
  85 void wait_for_kick(void);
  86 void call(void);
  87 void wait_for_call(void);
  88
  89 extern unsigned ring_size;
  90
  91 /* Compiler barrier - similar to what Linux uses */
  92 #define barrier() asm volatile("" ::: "memory")
  93
  94 /* Is there a portable way to do this? */
  95 #if defined(__x86_64__) || defined(__i386__)
  96 #define cpu_relax() asm ("rep; nop" ::: "memory")
  97 #elif defined(__s390x__)
  98 #define cpu_relax() barrier()
  99 #elif defined(__aarch64__)
 100 #define cpu_relax() asm ("yield" ::: "memory")
 101 #else
 102 #define cpu_relax() assert(0)
 103 #endif
 104
 105 extern bool do_relax;
 106
 107 static inline void busy_wait(void)
 108 {
 109         if (do_relax)
 110                 cpu_relax();
 111         else
 112                 /* prevent compiler from removing busy loops */
 113                 barrier();
 114 }
 115
 116 #if defined(__x86_64__) || defined(__i386__)
 117 #define smp_mb()     asm volatile("lock; addl $0,-132(%%rsp)" ::: "memory", "cc")
 118 #elif defined(__aarch64__)
 119 #define smp_mb()     asm volatile("dmb ish" ::: "memory")
 120 #else
 121 /*
 122  * Not using __ATOMIC_SEQ_CST since gcc docs say they are only synchronized
 123  * with other __ATOMIC_SEQ_CST calls.
 124  */
 125 #define smp_mb() __sync_synchronize()
 126 #endif
 127
 128 /*
 129  * This abuses the atomic builtins for thread fences, and
 130  * adds a compiler barrier.
 131  */
 132 #define smp_release() do { \
 133     barrier(); \
 134     __atomic_thread_fence(__ATOMIC_RELEASE); \
 135 } while (0)
 136
 137 #define smp_acquire() do { \
 138     __atomic_thread_fence(__ATOMIC_ACQUIRE); \
 139     barrier(); \
 140 } while (0)
 141
 142 #if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
 143 #define smp_wmb() barrier()
 144 #elif defined(__aarch64__)
 145 #define smp_wmb() asm volatile("dmb ishst" ::: "memory")
 146 #else
 147 #define smp_wmb() smp_release()
 148 #endif
 149
 150 #ifndef __always_inline
 151 #define __always_inline inline __attribute__((always_inline))
 152 #endif
 153
 154 static __always_inline
 155 void __read_once_size(const volatile void *p, void *res, int size)
 156 {
 157         switch (size) {
 158         case 1: *(unsigned char *)res = *(volatile unsigned char *)p; break;
 159         case 2: *(unsigned short *)res = *(volatile unsigned short *)p; break;
 160         case 4: *(unsigned int *)res = *(volatile unsigned int *)p; break;
 161         case 8: *(unsigned long long *)res = *(volatile unsigned long long *)p; break;
 162         default:
 163                 barrier();
 164                 __builtin_memcpy((void *)res, (const void *)p, size);
 165                 barrier();
 166         }
 167 }
 168
 169 static __always_inline void __write_once_size(volatile void *p, void *res, int size)
 170 {
 171         switch (size) {
 172         case 1: *(volatile unsigned char *)p = *(unsigned char *)res; break;
 173         case 2: *(volatile unsigned short *)p = *(unsigned short *)res; break;
 174         case 4: *(volatile unsigned int *)p = *(unsigned int *)res; break;
 175         case 8: *(volatile unsigned long long *)p = *(unsigned long long *)res; break;
 176         default:
 177                 barrier();
 178                 __builtin_memcpy((void *)p, (const void *)res, size);
 179                 barrier();
 180         }
 181 }
 182
 183 #ifdef __alpha__
 184 #define READ_ONCE(x) \
 185 ({                                                                      \
 186         union { typeof(x) __val; char __c[1]; } __u;                    \
 187         __read_once_size(&(x), __u.__c, sizeof(x));             \
 188         smp_mb(); /* Enforce dependency ordering from x */              \
 189         __u.__val;                                                      \
 190 })
 191 #else
 192 #define READ_ONCE(x)                                                    \
 193 ({                                                                      \
 194         union { typeof(x) __val; char __c[1]; } __u;                    \
 195         __read_once_size(&(x), __u.__c, sizeof(x));                     \
 196         __u.__val;                                                      \
 197 })
 198 #endif
 199
 200 #define WRITE_ONCE(x, val) \
 201 ({                                                      \
 202         union { typeof(x) __val; char __c[1]; } __u =   \
 203                 { .__val = (typeof(x)) (val) }; \
 204         __write_once_size(&(x), __u.__c, sizeof(x));    \
 205         __u.__val;                                      \
 206 })
 207
 208 #endif