drivers/infiniband/hw/hfi1/iowait.h

   1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
   2 /*
   3  * Copyright(c) 2015 - 2018 Intel Corporation.
   4  */
   5
   6 #ifndef _HFI1_IOWAIT_H
   7 #define _HFI1_IOWAIT_H
   8
   9 #include <linux/list.h>
  10 #include <linux/workqueue.h>
  11 #include <linux/wait.h>
  12 #include <linux/sched.h>
  13
  14 #include "sdma_txreq.h"
  15
  16 /*
  17  * typedef (*restart_t)() - restart callback
  18  * @work: pointer to work structure
  19  */
  20 typedef void (*restart_t)(struct work_struct *work);
  21
  22 #define IOWAIT_PENDING_IB  0x0
  23 #define IOWAIT_PENDING_TID 0x1
  24
  25 /*
  26  * A QP can have multiple Send Engines (SEs).
  27  *
  28  * The current use case is for supporting a TID RDMA
  29  * packet build/xmit mechanism independent from verbs.
  30  */
  31 #define IOWAIT_SES 2
  32 #define IOWAIT_IB_SE 0
  33 #define IOWAIT_TID_SE 1
  34
  35 struct sdma_txreq;
  36 struct sdma_engine;
  37 /**
  38  * @iowork: the work struct
  39  * @tx_head: list of prebuilt packets
  40  * @iow: the parent iowait structure
  41  *
  42  * This structure is the work item (process) specific
  43  * details associated with the each of the two SEs of the
  44  * QP.
  45  *
  46  * The workstruct and the queued TXs are unique to each
  47  * SE.
  48  */
  49 struct iowait;
  50 struct iowait_work {
  51         struct work_struct iowork;
  52         struct list_head tx_head;
  53         struct iowait *iow;
  54 };
  55
  56 /**
  57  * @list: used to add/insert into QP/PQ wait lists
  58  * @tx_head: overflow list of sdma_txreq's
  59  * @sleep: no space callback
  60  * @wakeup: space callback wakeup
  61  * @sdma_drained: sdma count drained
  62  * @init_priority: callback to manipulate priority
  63  * @lock: lock protected head of wait queue
  64  * @iowork: workqueue overhead
  65  * @wait_dma: wait for sdma_busy == 0
  66  * @wait_pio: wait for pio_busy == 0
  67  * @sdma_busy: # of packets in flight
  68  * @count: total number of descriptors in tx_head'ed list
  69  * @tx_limit: limit for overflow queuing
  70  * @tx_count: number of tx entry's in tx_head'ed list
  71  * @flags: wait flags (one per QP)
  72  * @wait: SE array for multiple legs
  73  *
  74  * This is to be embedded in user's state structure
  75  * (QP or PQ).
  76  *
  77  * The sleep and wakeup members are a
  78  * bit misnamed.   They do not strictly
  79  * speaking sleep or wake up, but they
  80  * are callbacks for the ULP to implement
  81  * what ever queuing/dequeuing of
  82  * the embedded iowait and its containing struct
  83  * when a resource shortage like SDMA ring space
  84  * or PIO credit space is seen.
  85  *
  86  * Both potentially have locks help
  87  * so sleeping is not allowed and it is not
  88  * supported to submit txreqs from the wakeup
  89  * call directly because of lock conflicts.
  90  *
  91  * The wait_dma member along with the iow
  92  *
  93  * The lock field is used by waiters to record
  94  * the seqlock_t that guards the list head.
  95  * Waiters explicity know that, but the destroy
  96  * code that unwaits QPs does not.
  97  */
  98 struct iowait {
  99         struct list_head list;
 100         int (*sleep)(
 101                 struct sdma_engine *sde,
 102                 struct iowait_work *wait,
 103                 struct sdma_txreq *tx,
 104                 uint seq,
 105                 bool pkts_sent
 106                 );
 107         void (*wakeup)(struct iowait *wait, int reason);
 108         void (*sdma_drained)(struct iowait *wait);
 109         void (*init_priority)(struct iowait *wait);
 110         seqlock_t *lock;
 111         wait_queue_head_t wait_dma;
 112         wait_queue_head_t wait_pio;
 113         atomic_t sdma_busy;
 114         atomic_t pio_busy;
 115         u32 count;
 116         u32 tx_limit;
 117         u32 tx_count;
 118         u8 starved_cnt;
 119         u8 priority;
 120         unsigned long flags;
 121         struct iowait_work wait[IOWAIT_SES];
 122 };
 123
 124 #define SDMA_AVAIL_REASON 0
 125
 126 void iowait_set_flag(struct iowait *wait, u32 flag);
 127 bool iowait_flag_set(struct iowait *wait, u32 flag);
 128 void iowait_clear_flag(struct iowait *wait, u32 flag);
 129
 130 void iowait_init(struct iowait *wait, u32 tx_limit,
 131                  void (*func)(struct work_struct *work),
 132                  void (*tidfunc)(struct work_struct *work),
 133                  int (*sleep)(struct sdma_engine *sde,
 134                               struct iowait_work *wait,
 135                               struct sdma_txreq *tx,
 136                               uint seq,
 137                               bool pkts_sent),
 138                  void (*wakeup)(struct iowait *wait, int reason),
 139                  void (*sdma_drained)(struct iowait *wait),
 140                  void (*init_priority)(struct iowait *wait));
 141
 142 /**
 143  * iowait_schedule() - schedule the default send engine work
 144  * @wait: wait struct to schedule
 145  * @wq: workqueue for schedule
 146  * @cpu: cpu
 147  */
 148 static inline bool iowait_schedule(struct iowait *wait,
 149                                    struct workqueue_struct *wq, int cpu)
 150 {
 151         return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
 152 }
 153
 154 /**
 155  * iowait_tid_schedule - schedule the tid SE
 156  * @wait: the iowait structure
 157  * @wq: the work queue
 158  * @cpu: the cpu
 159  */
 160 static inline bool iowait_tid_schedule(struct iowait *wait,
 161                                        struct workqueue_struct *wq, int cpu)
 162 {
 163         return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
 164 }
 165
 166 /**
 167  * iowait_sdma_drain() - wait for DMAs to drain
 168  *
 169  * @wait: iowait structure
 170  *
 171  * This will delay until the iowait sdmas have
 172  * completed.
 173  */
 174 static inline void iowait_sdma_drain(struct iowait *wait)
 175 {
 176         wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
 177 }
 178
 179 /**
 180  * iowait_sdma_pending() - return sdma pending count
 181  *
 182  * @wait: iowait structure
 183  *
 184  */
 185 static inline int iowait_sdma_pending(struct iowait *wait)
 186 {
 187         return atomic_read(&wait->sdma_busy);
 188 }
 189
 190 /**
 191  * iowait_sdma_inc - note sdma io pending
 192  * @wait: iowait structure
 193  */
 194 static inline void iowait_sdma_inc(struct iowait *wait)
 195 {
 196         atomic_inc(&wait->sdma_busy);
 197 }
 198
 199 /**
 200  * iowait_sdma_add - add count to pending
 201  * @wait: iowait structure
 202  */
 203 static inline void iowait_sdma_add(struct iowait *wait, int count)
 204 {
 205         atomic_add(count, &wait->sdma_busy);
 206 }
 207
 208 /**
 209  * iowait_sdma_dec - note sdma complete
 210  * @wait: iowait structure
 211  */
 212 static inline int iowait_sdma_dec(struct iowait *wait)
 213 {
 214         if (!wait)
 215                 return 0;
 216         return atomic_dec_and_test(&wait->sdma_busy);
 217 }
 218
 219 /**
 220  * iowait_pio_drain() - wait for pios to drain
 221  *
 222  * @wait: iowait structure
 223  *
 224  * This will delay until the iowait pios have
 225  * completed.
 226  */
 227 static inline void iowait_pio_drain(struct iowait *wait)
 228 {
 229         wait_event_timeout(wait->wait_pio,
 230                            !atomic_read(&wait->pio_busy),
 231                            HZ);
 232 }
 233
 234 /**
 235  * iowait_pio_pending() - return pio pending count
 236  *
 237  * @wait: iowait structure
 238  *
 239  */
 240 static inline int iowait_pio_pending(struct iowait *wait)
 241 {
 242         return atomic_read(&wait->pio_busy);
 243 }
 244
 245 /**
 246  * iowait_pio_inc - note pio pending
 247  * @wait: iowait structure
 248  */
 249 static inline void iowait_pio_inc(struct iowait *wait)
 250 {
 251         atomic_inc(&wait->pio_busy);
 252 }
 253
 254 /**
 255  * iowait_pio_dec - note pio complete
 256  * @wait: iowait structure
 257  */
 258 static inline int iowait_pio_dec(struct iowait *wait)
 259 {
 260         if (!wait)
 261                 return 0;
 262         return atomic_dec_and_test(&wait->pio_busy);
 263 }
 264
 265 /**
 266  * iowait_drain_wakeup() - trigger iowait_drain() waiter
 267  *
 268  * @wait: iowait structure
 269  *
 270  * This will trigger any waiters.
 271  */
 272 static inline void iowait_drain_wakeup(struct iowait *wait)
 273 {
 274         wake_up(&wait->wait_dma);
 275         wake_up(&wait->wait_pio);
 276         if (wait->sdma_drained)
 277                 wait->sdma_drained(wait);
 278 }
 279
 280 /**
 281  * iowait_get_txhead() - get packet off of iowait list
 282  *
 283  * @wait: iowait_work structure
 284  */
 285 static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
 286 {
 287         struct sdma_txreq *tx = NULL;
 288
 289         if (!list_empty(&wait->tx_head)) {
 290                 tx = list_first_entry(
 291                         &wait->tx_head,
 292                         struct sdma_txreq,
 293                         list);
 294                 list_del_init(&tx->list);
 295         }
 296         return tx;
 297 }
 298
 299 static inline u16 iowait_get_desc(struct iowait_work *w)
 300 {
 301         u16 num_desc = 0;
 302         struct sdma_txreq *tx = NULL;
 303
 304         if (!list_empty(&w->tx_head)) {
 305                 tx = list_first_entry(&w->tx_head, struct sdma_txreq,
 306                                       list);
 307                 num_desc = tx->num_desc;
 308                 if (tx->flags & SDMA_TXREQ_F_VIP)
 309                         w->iow->priority++;
 310         }
 311         return num_desc;
 312 }
 313
 314 static inline u32 iowait_get_all_desc(struct iowait *w)
 315 {
 316         u32 num_desc = 0;
 317
 318         num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
 319         num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
 320         return num_desc;
 321 }
 322
 323 static inline void iowait_update_priority(struct iowait_work *w)
 324 {
 325         struct sdma_txreq *tx = NULL;
 326
 327         if (!list_empty(&w->tx_head)) {
 328                 tx = list_first_entry(&w->tx_head, struct sdma_txreq,
 329                                       list);
 330                 if (tx->flags & SDMA_TXREQ_F_VIP)
 331                         w->iow->priority++;
 332         }
 333 }
 334
 335 static inline void iowait_update_all_priority(struct iowait *w)
 336 {
 337         iowait_update_priority(&w->wait[IOWAIT_IB_SE]);
 338         iowait_update_priority(&w->wait[IOWAIT_TID_SE]);
 339 }
 340
 341 static inline void iowait_init_priority(struct iowait *w)
 342 {
 343         w->priority = 0;
 344         if (w->init_priority)
 345                 w->init_priority(w);
 346 }
 347
 348 static inline void iowait_get_priority(struct iowait *w)
 349 {
 350         iowait_init_priority(w);
 351         iowait_update_all_priority(w);
 352 }
 353
 354 /**
 355  * iowait_queue - Put the iowait on a wait queue
 356  * @pkts_sent: have some packets been sent before queuing?
 357  * @w: the iowait struct
 358  * @wait_head: the wait queue
 359  *
 360  * This function is called to insert an iowait struct into a
 361  * wait queue after a resource (eg, sdma descriptor or pio
 362  * buffer) is run out.
 363  */
 364 static inline void iowait_queue(bool pkts_sent, struct iowait *w,
 365                                 struct list_head *wait_head)
 366 {
 367         /*
 368          * To play fair, insert the iowait at the tail of the wait queue if it
 369          * has already sent some packets; Otherwise, put it at the head.
 370          * However, if it has priority packets to send, also put it at the
 371          * head.
 372          */
 373         if (pkts_sent)
 374                 w->starved_cnt = 0;
 375         else
 376                 w->starved_cnt++;
 377
 378         if (w->priority > 0 || !pkts_sent)
 379                 list_add(&w->list, wait_head);
 380         else
 381                 list_add_tail(&w->list, wait_head);
 382 }
 383
 384 /**
 385  * iowait_starve_clear - clear the wait queue's starve count
 386  * @pkts_sent: have some packets been sent?
 387  * @w: the iowait struct
 388  *
 389  * This function is called to clear the starve count. If no
 390  * packets have been sent, the starve count will not be cleared.
 391  */
 392 static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
 393 {
 394         if (pkts_sent)
 395                 w->starved_cnt = 0;
 396 }
 397
 398 /* Update the top priority index */
 399 uint iowait_priority_update_top(struct iowait *w,
 400                                 struct iowait *top,
 401                                 uint idx, uint top_idx);
 402
 403 /**
 404  * iowait_packet_queued() - determine if a packet is queued
 405  * @wait: the iowait_work structure
 406  */
 407 static inline bool iowait_packet_queued(struct iowait_work *wait)
 408 {
 409         return !list_empty(&wait->tx_head);
 410 }
 411
 412 /**
 413  * inc_wait_count - increment wait counts
 414  * @w: the log work struct
 415  * @n: the count
 416  */
 417 static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
 418 {
 419         if (!w)
 420                 return;
 421         w->iow->tx_count++;
 422         w->iow->count += n;
 423 }
 424
 425 /**
 426  * iowait_get_tid_work - return iowait_work for tid SE
 427  * @w: the iowait struct
 428  */
 429 static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
 430 {
 431         return &w->wait[IOWAIT_TID_SE];
 432 }
 433
 434 /**
 435  * iowait_get_ib_work - return iowait_work for ib SE
 436  * @w: the iowait struct
 437  */
 438 static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
 439 {
 440         return &w->wait[IOWAIT_IB_SE];
 441 }
 442
 443 /**
 444  * iowait_ioww_to_iow - return iowait given iowait_work
 445  * @w: the iowait_work struct
 446  */
 447 static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
 448 {
 449         if (likely(w))
 450                 return w->iow;
 451         return NULL;
 452 }
 453
 454 void iowait_cancel_work(struct iowait *w);
 455 int iowait_set_work_flag(struct iowait_work *w);
 456
 457 #endif