include/net/busy_poll.h

   1 /*
   2  * net busy poll support
   3  * Copyright(c) 2013 Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Author: Eliezer Tamir
  19  *
  20  * Contact Information:
  21  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  22  */
  23
  24 #ifndef _LINUX_NET_BUSY_POLL_H
  25 #define _LINUX_NET_BUSY_POLL_H
  26
  27 #include <linux/netdevice.h>
  28 #include <linux/sched/clock.h>
  29 #include <linux/sched/signal.h>
  30 #include <net/ip.h>
  31
  32 /*              0 - Reserved to indicate value not set
  33  *     1..NR_CPUS - Reserved for sender_cpu
  34  *  NR_CPUS+1..~0 - Region available for NAPI IDs
  35  */
  36 #define MIN_NAPI_ID ((unsigned int)(NR_CPUS + 1))
  37
  38 #ifdef CONFIG_NET_RX_BUSY_POLL
  39
  40 struct napi_struct;
  41 extern unsigned int sysctl_net_busy_read __read_mostly;
  42 extern unsigned int sysctl_net_busy_poll __read_mostly;
  43
  44 static inline bool net_busy_loop_on(void)
  45 {
  46         return sysctl_net_busy_poll;
  47 }
  48
  49 static inline bool sk_can_busy_loop(const struct sock *sk)
  50 {
  51         return sk->sk_ll_usec && !signal_pending(current);
  52 }
  53
  54 bool sk_busy_loop_end(void *p, unsigned long start_time);
  55
  56 void napi_busy_loop(unsigned int napi_id,
  57                     bool (*loop_end)(void *, unsigned long),
  58                     void *loop_end_arg);
  59
  60 #else /* CONFIG_NET_RX_BUSY_POLL */
  61 static inline unsigned long net_busy_loop_on(void)
  62 {
  63         return 0;
  64 }
  65
  66 static inline bool sk_can_busy_loop(struct sock *sk)
  67 {
  68         return false;
  69 }
  70
  71 #endif /* CONFIG_NET_RX_BUSY_POLL */
  72
  73 static inline unsigned long busy_loop_current_time(void)
  74 {
  75 #ifdef CONFIG_NET_RX_BUSY_POLL
  76         return (unsigned long)(local_clock() >> 10);
  77 #else
  78         return 0;
  79 #endif
  80 }
  81
  82 /* in poll/select we use the global sysctl_net_ll_poll value */
  83 static inline bool busy_loop_timeout(unsigned long start_time)
  84 {
  85 #ifdef CONFIG_NET_RX_BUSY_POLL
  86         unsigned long bp_usec = READ_ONCE(sysctl_net_busy_poll);
  87
  88         if (bp_usec) {
  89                 unsigned long end_time = start_time + bp_usec;
  90                 unsigned long now = busy_loop_current_time();
  91
  92                 return time_after(now, end_time);
  93         }
  94 #endif
  95         return true;
  96 }
  97
  98 static inline bool sk_busy_loop_timeout(struct sock *sk,
  99                                         unsigned long start_time)
 100 {
 101 #ifdef CONFIG_NET_RX_BUSY_POLL
 102         unsigned long bp_usec = READ_ONCE(sk->sk_ll_usec);
 103
 104         if (bp_usec) {
 105                 unsigned long end_time = start_time + bp_usec;
 106                 unsigned long now = busy_loop_current_time();
 107
 108                 return time_after(now, end_time);
 109         }
 110 #endif
 111         return true;
 112 }
 113
 114 static inline void sk_busy_loop(struct sock *sk, int nonblock)
 115 {
 116 #ifdef CONFIG_NET_RX_BUSY_POLL
 117         unsigned int napi_id = READ_ONCE(sk->sk_napi_id);
 118
 119         if (napi_id >= MIN_NAPI_ID)
 120                 napi_busy_loop(napi_id, nonblock ? NULL : sk_busy_loop_end, sk);
 121 #endif
 122 }
 123
 124 static inline void sock_poll_busy_loop(struct socket *sock, __poll_t events)
 125 {
 126         if (sk_can_busy_loop(sock->sk) &&
 127             events && (events & POLL_BUSY_LOOP)) {
 128                 /* once, only if requested by syscall */
 129                 sk_busy_loop(sock->sk, 1);
 130         }
 131 }
 132
 133 /* if this socket can poll_ll, tell the system call */
 134 static inline __poll_t sock_poll_busy_flag(struct socket *sock)
 135 {
 136         return sk_can_busy_loop(sock->sk) ? POLL_BUSY_LOOP : 0;
 137 }
 138
 139 /* used in the NIC receive handler to mark the skb */
 140 static inline void skb_mark_napi_id(struct sk_buff *skb,
 141                                     struct napi_struct *napi)
 142 {
 143 #ifdef CONFIG_NET_RX_BUSY_POLL
 144         skb->napi_id = napi->napi_id;
 145 #endif
 146 }
 147
 148 /* used in the protocol hanlder to propagate the napi_id to the socket */
 149 static inline void sk_mark_napi_id(struct sock *sk, const struct sk_buff *skb)
 150 {
 151 #ifdef CONFIG_NET_RX_BUSY_POLL
 152         sk->sk_napi_id = skb->napi_id;
 153 #endif
 154         sk_rx_queue_set(sk, skb);
 155 }
 156
 157 /* variant used for unconnected sockets */
 158 static inline void sk_mark_napi_id_once(struct sock *sk,
 159                                         const struct sk_buff *skb)
 160 {
 161 #ifdef CONFIG_NET_RX_BUSY_POLL
 162         if (!sk->sk_napi_id)
 163                 sk->sk_napi_id = skb->napi_id;
 164 #endif
 165 }
 166
 167 #endif /* _LINUX_NET_BUSY_POLL_H */