tools/virtio/ringtest/virtio_ring_0_9.c

   1 /*
   2  * Copyright (C) 2016 Red Hat, Inc.
   3  * Author: Michael S. Tsirkin <mst@redhat.com>
   4  * This work is licensed under the terms of the GNU GPL, version 2.
   5  *
   6  * Partial implementation of virtio 0.9. event index is used for signalling,
   7  * unconditionally. Design roughly follows linux kernel implementation in order
   8  * to be able to judge its performance.
   9  */
  10 #define _GNU_SOURCE
  11 #include "main.h"
  12 #include <stdlib.h>
  13 #include <stdio.h>
  14 #include <assert.h>
  15 #include <string.h>
  16 #include <linux/virtio_ring.h>
  17
  18 struct data {
  19         void *data;
  20 } *data;
  21
  22 struct vring ring;
  23
  24 /* enabling the below activates experimental ring polling code
  25  * (which skips index reads on consumer in favor of looking at
  26  * high bits of ring id ^ 0x8000).
  27  */
  28 /* #ifdef RING_POLL */
  29 /* enabling the below activates experimental in-order code
  30  * (which skips ring updates and reads and writes len in descriptor).
  31  */
  32 /* #ifdef INORDER */
  33
  34 #if defined(RING_POLL) && defined(INORDER)
  35 #error "RING_POLL and INORDER are mutually exclusive"
  36 #endif
  37
  38 /* how much padding is needed to avoid false cache sharing */
  39 #define HOST_GUEST_PADDING 0x80
  40
  41 struct guest {
  42         unsigned short avail_idx;
  43         unsigned short last_used_idx;
  44         unsigned short num_free;
  45         unsigned short kicked_avail_idx;
  46 #ifndef INORDER
  47         unsigned short free_head;
  48 #else
  49         unsigned short reserved_free_head;
  50 #endif
  51         unsigned char reserved[HOST_GUEST_PADDING - 10];
  52 } guest;
  53
  54 struct host {
  55         /* we do not need to track last avail index
  56          * unless we have more than one in flight.
  57          */
  58         unsigned short used_idx;
  59         unsigned short called_used_idx;
  60         unsigned char reserved[HOST_GUEST_PADDING - 4];
  61 } host;
  62
  63 /* implemented by ring */
  64 void alloc_ring(void)
  65 {
  66         int ret;
  67         int i;
  68         void *p;
  69
  70         ret = posix_memalign(&p, 0x1000, vring_size(ring_size, 0x1000));
  71         if (ret) {
  72                 perror("Unable to allocate ring buffer.\n");
  73                 exit(3);
  74         }
  75         memset(p, 0, vring_size(ring_size, 0x1000));
  76         vring_init(&ring, ring_size, p, 0x1000);
  77
  78         guest.avail_idx = 0;
  79         guest.kicked_avail_idx = -1;
  80         guest.last_used_idx = 0;
  81 #ifndef INORDER
  82         /* Put everything in free lists. */
  83         guest.free_head = 0;
  84 #endif
  85         for (i = 0; i < ring_size - 1; i++)
  86                 ring.desc[i].next = i + 1;
  87         host.used_idx = 0;
  88         host.called_used_idx = -1;
  89         guest.num_free = ring_size;
  90         data = malloc(ring_size * sizeof *data);
  91         if (!data) {
  92                 perror("Unable to allocate data buffer.\n");
  93                 exit(3);
  94         }
  95         memset(data, 0, ring_size * sizeof *data);
  96 }
  97
  98 /* guest side */
  99 int add_inbuf(unsigned len, void *buf, void *datap)
 100 {
 101         unsigned head;
 102 #ifndef INORDER
 103         unsigned avail;
 104 #endif
 105         struct vring_desc *desc;
 106
 107         if (!guest.num_free)
 108                 return -1;
 109
 110 #ifdef INORDER
 111         head = (ring_size - 1) & (guest.avail_idx++);
 112 #else
 113         head = guest.free_head;
 114 #endif
 115         guest.num_free--;
 116
 117         desc = ring.desc;
 118         desc[head].flags = VRING_DESC_F_NEXT;
 119         desc[head].addr = (unsigned long)(void *)buf;
 120         desc[head].len = len;
 121         /* We do it like this to simulate the way
 122          * we'd have to flip it if we had multiple
 123          * descriptors.
 124          */
 125         desc[head].flags &= ~VRING_DESC_F_NEXT;
 126 #ifndef INORDER
 127         guest.free_head = desc[head].next;
 128 #endif
 129
 130         data[head].data = datap;
 131
 132 #ifdef RING_POLL
 133         /* Barrier A (for pairing) */
 134         smp_release();
 135         avail = guest.avail_idx++;
 136         ring.avail->ring[avail & (ring_size - 1)] =
 137                 (head | (avail & ~(ring_size - 1))) ^ 0x8000;
 138 #else
 139 #ifndef INORDER
 140         /* Barrier A (for pairing) */
 141         smp_release();
 142         avail = (ring_size - 1) & (guest.avail_idx++);
 143         ring.avail->ring[avail] = head;
 144 #endif
 145         /* Barrier A (for pairing) */
 146         smp_release();
 147 #endif
 148         ring.avail->idx = guest.avail_idx;
 149         return 0;
 150 }
 151
 152 void *get_buf(unsigned *lenp, void **bufp)
 153 {
 154         unsigned head;
 155         unsigned index;
 156         void *datap;
 157
 158 #ifdef RING_POLL
 159         head = (ring_size - 1) & guest.last_used_idx;
 160         index = ring.used->ring[head].id;
 161         if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1))
 162                 return NULL;
 163         /* Barrier B (for pairing) */
 164         smp_acquire();
 165         index &= ring_size - 1;
 166 #else
 167         if (ring.used->idx == guest.last_used_idx)
 168                 return NULL;
 169         /* Barrier B (for pairing) */
 170         smp_acquire();
 171 #ifdef INORDER
 172         head = (ring_size - 1) & guest.last_used_idx;
 173         index = head;
 174 #else
 175         head = (ring_size - 1) & guest.last_used_idx;
 176         index = ring.used->ring[head].id;
 177 #endif
 178
 179 #endif
 180 #ifdef INORDER
 181         *lenp = ring.desc[index].len;
 182 #else
 183         *lenp = ring.used->ring[head].len;
 184 #endif
 185         datap = data[index].data;
 186         *bufp = (void*)(unsigned long)ring.desc[index].addr;
 187         data[index].data = NULL;
 188 #ifndef INORDER
 189         ring.desc[index].next = guest.free_head;
 190         guest.free_head = index;
 191 #endif
 192         guest.num_free++;
 193         guest.last_used_idx++;
 194         return datap;
 195 }
 196
 197 bool used_empty()
 198 {
 199         unsigned short last_used_idx = guest.last_used_idx;
 200 #ifdef RING_POLL
 201         unsigned short head = last_used_idx & (ring_size - 1);
 202         unsigned index = ring.used->ring[head].id;
 203
 204         return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1);
 205 #else
 206         return ring.used->idx == last_used_idx;
 207 #endif
 208 }
 209
 210 void disable_call()
 211 {
 212         /* Doing nothing to disable calls might cause
 213          * extra interrupts, but reduces the number of cache misses.
 214          */
 215 }
 216
 217 bool enable_call()
 218 {
 219         vring_used_event(&ring) = guest.last_used_idx;
 220         /* Flush call index write */
 221         /* Barrier D (for pairing) */
 222         smp_mb();
 223         return used_empty();
 224 }
 225
 226 void kick_available(void)
 227 {
 228         /* Flush in previous flags write */
 229         /* Barrier C (for pairing) */
 230         smp_mb();
 231         if (!vring_need_event(vring_avail_event(&ring),
 232                               guest.avail_idx,
 233                               guest.kicked_avail_idx))
 234                 return;
 235
 236         guest.kicked_avail_idx = guest.avail_idx;
 237         kick();
 238 }
 239
 240 /* host side */
 241 void disable_kick()
 242 {
 243         /* Doing nothing to disable kicks might cause
 244          * extra interrupts, but reduces the number of cache misses.
 245          */
 246 }
 247
 248 bool enable_kick()
 249 {
 250         vring_avail_event(&ring) = host.used_idx;
 251         /* Barrier C (for pairing) */
 252         smp_mb();
 253         return avail_empty();
 254 }
 255
 256 bool avail_empty()
 257 {
 258         unsigned head = host.used_idx;
 259 #ifdef RING_POLL
 260         unsigned index = ring.avail->ring[head & (ring_size - 1)];
 261
 262         return ((index ^ head ^ 0x8000) & ~(ring_size - 1));
 263 #else
 264         return head == ring.avail->idx;
 265 #endif
 266 }
 267
 268 bool use_buf(unsigned *lenp, void **bufp)
 269 {
 270         unsigned used_idx = host.used_idx;
 271         struct vring_desc *desc;
 272         unsigned head;
 273
 274 #ifdef RING_POLL
 275         head = ring.avail->ring[used_idx & (ring_size - 1)];
 276         if ((used_idx ^ head ^ 0x8000) & ~(ring_size - 1))
 277                 return false;
 278         /* Barrier A (for pairing) */
 279         smp_acquire();
 280
 281         used_idx &= ring_size - 1;
 282         desc = &ring.desc[head & (ring_size - 1)];
 283 #else
 284         if (used_idx == ring.avail->idx)
 285                 return false;
 286
 287         /* Barrier A (for pairing) */
 288         smp_acquire();
 289
 290         used_idx &= ring_size - 1;
 291 #ifdef INORDER
 292         head = used_idx;
 293 #else
 294         head = ring.avail->ring[used_idx];
 295 #endif
 296         desc = &ring.desc[head];
 297 #endif
 298
 299         *lenp = desc->len;
 300         *bufp = (void *)(unsigned long)desc->addr;
 301
 302 #ifdef INORDER
 303         desc->len = desc->len - 1;
 304 #else
 305         /* now update used ring */
 306         ring.used->ring[used_idx].id = head;
 307         ring.used->ring[used_idx].len = desc->len - 1;
 308 #endif
 309         /* Barrier B (for pairing) */
 310         smp_release();
 311         host.used_idx++;
 312         ring.used->idx = host.used_idx;
 313
 314         return true;
 315 }
 316
 317 void call_used(void)
 318 {
 319         /* Flush in previous flags write */
 320         /* Barrier D (for pairing) */
 321         smp_mb();
 322         if (!vring_need_event(vring_used_event(&ring),
 323                               host.used_idx,
 324                               host.called_used_idx))
 325                 return;
 326
 327         host.called_used_idx = host.used_idx;
 328         call();
 329 }