drivers/infiniband/hw/hfi1/iowait.h

   1 #ifndef _HFI1_IOWAIT_H
   2 #define _HFI1_IOWAIT_H
   3 /*
   4  * Copyright(c) 2015, 2016 Intel Corporation.
   5  *
   6  * This file is provided under a dual BSD/GPLv2 license.  When using or
   7  * redistributing this file, you may do so under either license.
   8  *
   9  * GPL LICENSE SUMMARY
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of version 2 of the GNU General Public License as
  13  * published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful, but
  16  * WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * General Public License for more details.
  19  *
  20  * BSD LICENSE
  21  *
  22  * Redistribution and use in source and binary forms, with or without
  23  * modification, are permitted provided that the following conditions
  24  * are met:
  25  *
  26  *  - Redistributions of source code must retain the above copyright
  27  *    notice, this list of conditions and the following disclaimer.
  28  *  - Redistributions in binary form must reproduce the above copyright
  29  *    notice, this list of conditions and the following disclaimer in
  30  *    the documentation and/or other materials provided with the
  31  *    distribution.
  32  *  - Neither the name of Intel Corporation nor the names of its
  33  *    contributors may be used to endorse or promote products derived
  34  *    from this software without specific prior written permission.
  35  *
  36  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  37  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  38  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  39  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  40  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  41  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  42  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  43  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  44  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  45  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  46  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47  *
  48  */
  49
  50 #include <linux/list.h>
  51 #include <linux/workqueue.h>
  52 #include <linux/sched.h>
  53
  54 #include "sdma_txreq.h"
  55
  56 /*
  57  * typedef (*restart_t)() - restart callback
  58  * @work: pointer to work structure
  59  */
  60 typedef void (*restart_t)(struct work_struct *work);
  61
  62 struct sdma_txreq;
  63 struct sdma_engine;
  64 /**
  65  * struct iowait - linkage for delayed progress/waiting
  66  * @list: used to add/insert into QP/PQ wait lists
  67  * @lock: uses to record the list head lock
  68  * @tx_head: overflow list of sdma_txreq's
  69  * @sleep: no space callback
  70  * @wakeup: space callback wakeup
  71  * @sdma_drained: sdma count drained
  72  * @iowork: workqueue overhead
  73  * @wait_dma: wait for sdma_busy == 0
  74  * @wait_pio: wait for pio_busy == 0
  75  * @sdma_busy: # of packets in flight
  76  * @count: total number of descriptors in tx_head'ed list
  77  * @tx_limit: limit for overflow queuing
  78  * @tx_count: number of tx entry's in tx_head'ed list
  79  *
  80  * This is to be embedded in user's state structure
  81  * (QP or PQ).
  82  *
  83  * The sleep and wakeup members are a
  84  * bit misnamed.   They do not strictly
  85  * speaking sleep or wake up, but they
  86  * are callbacks for the ULP to implement
  87  * what ever queuing/dequeuing of
  88  * the embedded iowait and its containing struct
  89  * when a resource shortage like SDMA ring space is seen.
  90  *
  91  * Both potentially have locks help
  92  * so sleeping is not allowed.
  93  *
  94  * The wait_dma member along with the iow
  95  *
  96  * The lock field is used by waiters to record
  97  * the seqlock_t that guards the list head.
  98  * Waiters explicity know that, but the destroy
  99  * code that unwaits QPs does not.
 100  */
 101
 102 struct iowait {
 103         struct list_head list;
 104         struct list_head tx_head;
 105         int (*sleep)(
 106                 struct sdma_engine *sde,
 107                 struct iowait *wait,
 108                 struct sdma_txreq *tx,
 109                 uint seq,
 110                 bool pkts_sent
 111                 );
 112         void (*wakeup)(struct iowait *wait, int reason);
 113         void (*sdma_drained)(struct iowait *wait);
 114         seqlock_t *lock;
 115         struct work_struct iowork;
 116         wait_queue_head_t wait_dma;
 117         wait_queue_head_t wait_pio;
 118         atomic_t sdma_busy;
 119         atomic_t pio_busy;
 120         u32 count;
 121         u32 tx_limit;
 122         u32 tx_count;
 123         u8 starved_cnt;
 124 };
 125
 126 #define SDMA_AVAIL_REASON 0
 127
 128 /**
 129  * iowait_init() - initialize wait structure
 130  * @wait: wait struct to initialize
 131  * @tx_limit: limit for overflow queuing
 132  * @func: restart function for workqueue
 133  * @sleep: sleep function for no space
 134  * @resume: wakeup function for no space
 135  *
 136  * This function initializes the iowait
 137  * structure embedded in the QP or PQ.
 138  *
 139  */
 140
 141 static inline void iowait_init(
 142         struct iowait *wait,
 143         u32 tx_limit,
 144         void (*func)(struct work_struct *work),
 145         int (*sleep)(
 146                 struct sdma_engine *sde,
 147                 struct iowait *wait,
 148                 struct sdma_txreq *tx,
 149                 uint seq,
 150                 bool pkts_sent),
 151         void (*wakeup)(struct iowait *wait, int reason),
 152         void (*sdma_drained)(struct iowait *wait))
 153 {
 154         wait->count = 0;
 155         wait->lock = NULL;
 156         INIT_LIST_HEAD(&wait->list);
 157         INIT_LIST_HEAD(&wait->tx_head);
 158         INIT_WORK(&wait->iowork, func);
 159         init_waitqueue_head(&wait->wait_dma);
 160         init_waitqueue_head(&wait->wait_pio);
 161         atomic_set(&wait->sdma_busy, 0);
 162         atomic_set(&wait->pio_busy, 0);
 163         wait->tx_limit = tx_limit;
 164         wait->sleep = sleep;
 165         wait->wakeup = wakeup;
 166         wait->sdma_drained = sdma_drained;
 167 }
 168
 169 /**
 170  * iowait_schedule() - initialize wait structure
 171  * @wait: wait struct to schedule
 172  * @wq: workqueue for schedule
 173  * @cpu: cpu
 174  */
 175 static inline void iowait_schedule(
 176         struct iowait *wait,
 177         struct workqueue_struct *wq,
 178         int cpu)
 179 {
 180         queue_work_on(cpu, wq, &wait->iowork);
 181 }
 182
 183 /**
 184  * iowait_sdma_drain() - wait for DMAs to drain
 185  *
 186  * @wait: iowait structure
 187  *
 188  * This will delay until the iowait sdmas have
 189  * completed.
 190  */
 191 static inline void iowait_sdma_drain(struct iowait *wait)
 192 {
 193         wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
 194 }
 195
 196 /**
 197  * iowait_sdma_pending() - return sdma pending count
 198  *
 199  * @wait: iowait structure
 200  *
 201  */
 202 static inline int iowait_sdma_pending(struct iowait *wait)
 203 {
 204         return atomic_read(&wait->sdma_busy);
 205 }
 206
 207 /**
 208  * iowait_sdma_inc - note sdma io pending
 209  * @wait: iowait structure
 210  */
 211 static inline void iowait_sdma_inc(struct iowait *wait)
 212 {
 213         atomic_inc(&wait->sdma_busy);
 214 }
 215
 216 /**
 217  * iowait_sdma_add - add count to pending
 218  * @wait: iowait structure
 219  */
 220 static inline void iowait_sdma_add(struct iowait *wait, int count)
 221 {
 222         atomic_add(count, &wait->sdma_busy);
 223 }
 224
 225 /**
 226  * iowait_sdma_dec - note sdma complete
 227  * @wait: iowait structure
 228  */
 229 static inline int iowait_sdma_dec(struct iowait *wait)
 230 {
 231         return atomic_dec_and_test(&wait->sdma_busy);
 232 }
 233
 234 /**
 235  * iowait_pio_drain() - wait for pios to drain
 236  *
 237  * @wait: iowait structure
 238  *
 239  * This will delay until the iowait pios have
 240  * completed.
 241  */
 242 static inline void iowait_pio_drain(struct iowait *wait)
 243 {
 244         wait_event_timeout(wait->wait_pio,
 245                            !atomic_read(&wait->pio_busy),
 246                            HZ);
 247 }
 248
 249 /**
 250  * iowait_pio_pending() - return pio pending count
 251  *
 252  * @wait: iowait structure
 253  *
 254  */
 255 static inline int iowait_pio_pending(struct iowait *wait)
 256 {
 257         return atomic_read(&wait->pio_busy);
 258 }
 259
 260 /**
 261  * iowait_pio_inc - note pio pending
 262  * @wait: iowait structure
 263  */
 264 static inline void iowait_pio_inc(struct iowait *wait)
 265 {
 266         atomic_inc(&wait->pio_busy);
 267 }
 268
 269 /**
 270  * iowait_sdma_dec - note pio complete
 271  * @wait: iowait structure
 272  */
 273 static inline int iowait_pio_dec(struct iowait *wait)
 274 {
 275         return atomic_dec_and_test(&wait->pio_busy);
 276 }
 277
 278 /**
 279  * iowait_drain_wakeup() - trigger iowait_drain() waiter
 280  *
 281  * @wait: iowait structure
 282  *
 283  * This will trigger any waiters.
 284  */
 285 static inline void iowait_drain_wakeup(struct iowait *wait)
 286 {
 287         wake_up(&wait->wait_dma);
 288         wake_up(&wait->wait_pio);
 289         if (wait->sdma_drained)
 290                 wait->sdma_drained(wait);
 291 }
 292
 293 /**
 294  * iowait_get_txhead() - get packet off of iowait list
 295  *
 296  * @wait wait struture
 297  */
 298 static inline struct sdma_txreq *iowait_get_txhead(struct iowait *wait)
 299 {
 300         struct sdma_txreq *tx = NULL;
 301
 302         if (!list_empty(&wait->tx_head)) {
 303                 tx = list_first_entry(
 304                         &wait->tx_head,
 305                         struct sdma_txreq,
 306                         list);
 307                 list_del_init(&tx->list);
 308         }
 309         return tx;
 310 }
 311
 312 /**
 313  * iowait_queue - Put the iowait on a wait queue
 314  * @pkts_sent: have some packets been sent before queuing?
 315  * @w: the iowait struct
 316  * @wait_head: the wait queue
 317  *
 318  * This function is called to insert an iowait struct into a
 319  * wait queue after a resource (eg, sdma decriptor or pio
 320  * buffer) is run out.
 321  */
 322 static inline void iowait_queue(bool pkts_sent, struct iowait *w,
 323                                 struct list_head *wait_head)
 324 {
 325         /*
 326          * To play fair, insert the iowait at the tail of the wait queue if it
 327          * has already sent some packets; Otherwise, put it at the head.
 328          */
 329         if (pkts_sent) {
 330                 list_add_tail(&w->list, wait_head);
 331                 w->starved_cnt = 0;
 332         } else {
 333                 list_add(&w->list, wait_head);
 334                 w->starved_cnt++;
 335         }
 336 }
 337
 338 /**
 339  * iowait_starve_clear - clear the wait queue's starve count
 340  * @pkts_sent: have some packets been sent?
 341  * @w: the iowait struct
 342  *
 343  * This function is called to clear the starve count. If no
 344  * packets have been sent, the starve count will not be cleared.
 345  */
 346 static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
 347 {
 348         if (pkts_sent)
 349                 w->starved_cnt = 0;
 350 }
 351
 352 /**
 353  * iowait_starve_find_max - Find the maximum of the starve count
 354  * @w: the iowait struct
 355  * @max: a variable containing the max starve count
 356  * @idx: the index of the current iowait in an array
 357  * @max_idx: a variable containing the array index for the
 358  *         iowait entry that has the max starve count
 359  *
 360  * This function is called to compare the starve count of a
 361  * given iowait with the given max starve count. The max starve
 362  * count and the index will be updated if the iowait's start
 363  * count is larger.
 364  */
 365 static inline void iowait_starve_find_max(struct iowait *w, u8 *max,
 366                                           uint idx, uint *max_idx)
 367 {
 368         if (w->starved_cnt > *max) {
 369                 *max = w->starved_cnt;
 370                 *max_idx = idx;
 371         }
 372 }
 373
 374 /**
 375  * iowait_packet_queued() - determine if a packet is already built
 376  * @wait: the wait structure
 377  */
 378 static inline bool iowait_packet_queued(struct iowait *wait)
 379 {
 380         return !list_empty(&wait->tx_head);
 381 }
 382
 383 #endif