drivers/infiniband/hw/mlx5/wr.h

   1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
   2 /*
   3  * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
   4  */
   5
   6 #ifndef _MLX5_IB_WR_H
   7 #define _MLX5_IB_WR_H
   8
   9 #include "mlx5_ib.h"
  10
  11 enum {
  12         MLX5_IB_SQ_UMR_INLINE_THRESHOLD = 64,
  13 };
  14
  15 struct mlx5_wqe_eth_pad {
  16         u8 rsvd0[16];
  17 };
  18
  19
  20 /* get_sq_edge - Get the next nearby edge.
  21  *
  22  * An 'edge' is defined as the first following address after the end
  23  * of the fragment or the SQ. Accordingly, during the WQE construction
  24  * which repetitively increases the pointer to write the next data, it
  25  * simply should check if it gets to an edge.
  26  *
  27  * @sq - SQ buffer.
  28  * @idx - Stride index in the SQ buffer.
  29  *
  30  * Return:
  31  *      The new edge.
  32  */
  33 static inline void *get_sq_edge(struct mlx5_ib_wq *sq, u32 idx)
  34 {
  35         void *fragment_end;
  36
  37         fragment_end = mlx5_frag_buf_get_wqe
  38                 (&sq->fbc,
  39                  mlx5_frag_buf_get_idx_last_contig_stride(&sq->fbc, idx));
  40
  41         return fragment_end + MLX5_SEND_WQE_BB;
  42 }
  43
  44 /* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the
  45  * next nearby edge and get new address translation for current WQE position.
  46  * @sq: SQ buffer.
  47  * @seg: Current WQE position (16B aligned).
  48  * @wqe_sz: Total current WQE size [16B].
  49  * @cur_edge: Updated current edge.
  50  */
  51 static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,
  52                                          u32 wqe_sz, void **cur_edge)
  53 {
  54         u32 idx;
  55
  56         if (likely(*seg != *cur_edge))
  57                 return;
  58
  59         idx = (sq->cur_post + (wqe_sz >> 2)) & (sq->wqe_cnt - 1);
  60         *cur_edge = get_sq_edge(sq, idx);
  61
  62         *seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx);
  63 }
  64
  65 /* mlx5r_memcpy_send_wqe - copy data from src to WQE and update the relevant
  66  * WQ's pointers. At the end @seg is aligned to 16B regardless the copied size.
  67  * @sq: SQ buffer.
  68  * @cur_edge: Updated current edge.
  69  * @seg: Current WQE position (16B aligned).
  70  * @wqe_sz: Total current WQE size [16B].
  71  * @src: Pointer to copy from.
  72  * @n: Number of bytes to copy.
  73  */
  74 static inline void mlx5r_memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge,
  75                                          void **seg, u32 *wqe_sz,
  76                                          const void *src, size_t n)
  77 {
  78         while (likely(n)) {
  79                 size_t leftlen = *cur_edge - *seg;
  80                 size_t copysz = min_t(size_t, leftlen, n);
  81                 size_t stride;
  82
  83                 memcpy(*seg, src, copysz);
  84
  85                 n -= copysz;
  86                 src += copysz;
  87                 stride = !n ? ALIGN(copysz, 16) : copysz;
  88                 *seg += stride;
  89                 *wqe_sz += stride >> 4;
  90                 handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);
  91         }
  92 }
  93
  94 int mlx5r_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq);
  95 int mlx5r_begin_wqe(struct mlx5_ib_qp *qp, void **seg,
  96                     struct mlx5_wqe_ctrl_seg **ctrl, unsigned int *idx,
  97                     int *size, void **cur_edge, int nreq, __be32 general_id,
  98                     bool send_signaled, bool solicited);
  99 void mlx5r_finish_wqe(struct mlx5_ib_qp *qp, struct mlx5_wqe_ctrl_seg *ctrl,
 100                       void *seg, u8 size, void *cur_edge, unsigned int idx,
 101                       u64 wr_id, int nreq, u8 fence, u32 mlx5_opcode);
 102 void mlx5r_ring_db(struct mlx5_ib_qp *qp, unsigned int nreq,
 103                    struct mlx5_wqe_ctrl_seg *ctrl);
 104 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 105                       const struct ib_send_wr **bad_wr, bool drain);
 106 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
 107                       const struct ib_recv_wr **bad_wr, bool drain);
 108
 109 static inline int mlx5_ib_post_send_nodrain(struct ib_qp *ibqp,
 110                                             const struct ib_send_wr *wr,
 111                                             const struct ib_send_wr **bad_wr)
 112 {
 113         return mlx5_ib_post_send(ibqp, wr, bad_wr, false);
 114 }
 115
 116 static inline int mlx5_ib_post_send_drain(struct ib_qp *ibqp,
 117                                           const struct ib_send_wr *wr,
 118                                           const struct ib_send_wr **bad_wr)
 119 {
 120         return mlx5_ib_post_send(ibqp, wr, bad_wr, true);
 121 }
 122
 123 static inline int mlx5_ib_post_recv_nodrain(struct ib_qp *ibqp,
 124                                             const struct ib_recv_wr *wr,
 125                                             const struct ib_recv_wr **bad_wr)
 126 {
 127         return mlx5_ib_post_recv(ibqp, wr, bad_wr, false);
 128 }
 129
 130 static inline int mlx5_ib_post_recv_drain(struct ib_qp *ibqp,
 131                                           const struct ib_recv_wr *wr,
 132                                           const struct ib_recv_wr **bad_wr)
 133 {
 134         return mlx5_ib_post_recv(ibqp, wr, bad_wr, true);
 135 }
 136 #endif /* _MLX5_IB_WR_H */