net/ipv4/inet_timewait_sock.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              Generic TIME_WAIT sockets functions
   7  *
   8  *              From code orinally in TCP
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/kmemcheck.h>
  13 #include <linux/slab.h>
  14 #include <linux/module.h>
  15 #include <net/inet_hashtables.h>
  16 #include <net/inet_timewait_sock.h>
  17 #include <net/ip.h>
  18
  19
  20 /**
  21  *      inet_twsk_unhash - unhash a timewait socket from established hash
  22  *      @tw: timewait socket
  23  *
  24  *      unhash a timewait socket from established hash, if hashed.
  25  *      ehash lock must be held by caller.
  26  *      Returns 1 if caller should call inet_twsk_put() after lock release.
  27  */
  28 int inet_twsk_unhash(struct inet_timewait_sock *tw)
  29 {
  30         if (hlist_nulls_unhashed(&tw->tw_node))
  31                 return 0;
  32
  33         hlist_nulls_del_rcu(&tw->tw_node);
  34         sk_nulls_node_init(&tw->tw_node);
  35         /*
  36          * We cannot call inet_twsk_put() ourself under lock,
  37          * caller must call it for us.
  38          */
  39         return 1;
  40 }
  41
  42 /**
  43  *      inet_twsk_bind_unhash - unhash a timewait socket from bind hash
  44  *      @tw: timewait socket
  45  *      @hashinfo: hashinfo pointer
  46  *
  47  *      unhash a timewait socket from bind hash, if hashed.
  48  *      bind hash lock must be held by caller.
  49  *      Returns 1 if caller should call inet_twsk_put() after lock release.
  50  */
  51 int inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
  52                           struct inet_hashinfo *hashinfo)
  53 {
  54         struct inet_bind_bucket *tb = tw->tw_tb;
  55
  56         if (!tb)
  57                 return 0;
  58
  59         __hlist_del(&tw->tw_bind_node);
  60         tw->tw_tb = NULL;
  61         inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
  62         /*
  63          * We cannot call inet_twsk_put() ourself under lock,
  64          * caller must call it for us.
  65          */
  66         return 1;
  67 }
  68
  69 /* Must be called with locally disabled BHs. */
  70 static void inet_twsk_kill(struct inet_timewait_sock *tw)
  71 {
  72         struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
  73         struct inet_bind_hashbucket *bhead;
  74         int refcnt;
  75         /* Unlink from established hashes. */
  76         spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
  77
  78         spin_lock(lock);
  79         refcnt = inet_twsk_unhash(tw);
  80         spin_unlock(lock);
  81
  82         /* Disassociate with bind bucket. */
  83         bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
  84                         hashinfo->bhash_size)];
  85
  86         spin_lock(&bhead->lock);
  87         refcnt += inet_twsk_bind_unhash(tw, hashinfo);
  88         spin_unlock(&bhead->lock);
  89
  90         BUG_ON(refcnt >= atomic_read(&tw->tw_refcnt));
  91         atomic_sub(refcnt, &tw->tw_refcnt);
  92         atomic_dec(&tw->tw_dr->tw_count);
  93         inet_twsk_put(tw);
  94 }
  95
  96 void inet_twsk_free(struct inet_timewait_sock *tw)
  97 {
  98         struct module *owner = tw->tw_prot->owner;
  99         twsk_destructor((struct sock *)tw);
 100 #ifdef SOCK_REFCNT_DEBUG
 101         pr_debug("%s timewait_sock %p released\n", tw->tw_prot->name, tw);
 102 #endif
 103         kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
 104         module_put(owner);
 105 }
 106
 107 void inet_twsk_put(struct inet_timewait_sock *tw)
 108 {
 109         if (atomic_dec_and_test(&tw->tw_refcnt))
 110                 inet_twsk_free(tw);
 111 }
 112 EXPORT_SYMBOL_GPL(inet_twsk_put);
 113
 114 static void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,
 115                                    struct hlist_nulls_head *list)
 116 {
 117         hlist_nulls_add_head_rcu(&tw->tw_node, list);
 118 }
 119
 120 static void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,
 121                                     struct hlist_head *list)
 122 {
 123         hlist_add_head(&tw->tw_bind_node, list);
 124 }
 125
 126 /*
 127  * Enter the time wait state. This is called with locally disabled BH.
 128  * Essentially we whip up a timewait bucket, copy the relevant info into it
 129  * from the SK, and mess with hash chains and list linkage.
 130  */
 131 void __inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
 132                            struct inet_hashinfo *hashinfo)
 133 {
 134         const struct inet_sock *inet = inet_sk(sk);
 135         const struct inet_connection_sock *icsk = inet_csk(sk);
 136         struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
 137         spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
 138         struct inet_bind_hashbucket *bhead;
 139         /* Step 1: Put TW into bind hash. Original socket stays there too.
 140            Note, that any socket with inet->num != 0 MUST be bound in
 141            binding cache, even if it is closed.
 142          */
 143         bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->inet_num,
 144                         hashinfo->bhash_size)];
 145         spin_lock(&bhead->lock);
 146         tw->tw_tb = icsk->icsk_bind_hash;
 147         WARN_ON(!icsk->icsk_bind_hash);
 148         inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
 149         spin_unlock(&bhead->lock);
 150
 151         spin_lock(lock);
 152
 153         /*
 154          * Step 2: Hash TW into tcp ehash chain.
 155          * Notes :
 156          * - tw_refcnt is set to 4 because :
 157          * - We have one reference from bhash chain.
 158          * - We have one reference from ehash chain.
 159          * - We have one reference from timer.
 160          * - One reference for ourself (our caller will release it).
 161          * We can use atomic_set() because prior spin_lock()/spin_unlock()
 162          * committed into memory all tw fields.
 163          */
 164         atomic_set(&tw->tw_refcnt, 4);
 165         inet_twsk_add_node_rcu(tw, &ehead->chain);
 166
 167         /* Step 3: Remove SK from hash chain */
 168         if (__sk_nulls_del_node_init_rcu(sk))
 169                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 170
 171         spin_unlock(lock);
 172 }
 173 EXPORT_SYMBOL_GPL(__inet_twsk_hashdance);
 174
 175 void tw_timer_handler(unsigned long data)
 176 {
 177         struct inet_timewait_sock *tw = (struct inet_timewait_sock *)data;
 178
 179         if (tw->tw_kill)
 180                 NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITKILLED);
 181         else
 182                 NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITED);
 183         inet_twsk_kill(tw);
 184 }
 185
 186 struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
 187                                            struct inet_timewait_death_row *dr,
 188                                            const int state)
 189 {
 190         struct inet_timewait_sock *tw;
 191
 192         if (atomic_read(&dr->tw_count) >= dr->sysctl_max_tw_buckets)
 193                 return NULL;
 194
 195         tw = kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
 196                               GFP_ATOMIC);
 197         if (tw) {
 198                 const struct inet_sock *inet = inet_sk(sk);
 199
 200                 kmemcheck_annotate_bitfield(tw, flags);
 201
 202                 tw->tw_dr           = dr;
 203                 /* Give us an identity. */
 204                 tw->tw_daddr        = inet->inet_daddr;
 205                 tw->tw_rcv_saddr    = inet->inet_rcv_saddr;
 206                 tw->tw_bound_dev_if = sk->sk_bound_dev_if;
 207                 tw->tw_tos          = inet->tos;
 208                 tw->tw_num          = inet->inet_num;
 209                 tw->tw_state        = TCP_TIME_WAIT;
 210                 tw->tw_substate     = state;
 211                 tw->tw_sport        = inet->inet_sport;
 212                 tw->tw_dport        = inet->inet_dport;
 213                 tw->tw_family       = sk->sk_family;
 214                 tw->tw_reuse        = sk->sk_reuse;
 215                 tw->tw_hash         = sk->sk_hash;
 216                 tw->tw_ipv6only     = 0;
 217                 tw->tw_transparent  = inet->transparent;
 218                 tw->tw_prot         = sk->sk_prot_creator;
 219                 atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
 220                 twsk_net_set(tw, sock_net(sk));
 221                 setup_timer(&tw->tw_timer, tw_timer_handler, (unsigned long)tw);
 222                 /*
 223                  * Because we use RCU lookups, we should not set tw_refcnt
 224                  * to a non null value before everything is setup for this
 225                  * timewait socket.
 226                  */
 227                 atomic_set(&tw->tw_refcnt, 0);
 228
 229                 __module_get(tw->tw_prot->owner);
 230         }
 231
 232         return tw;
 233 }
 234 EXPORT_SYMBOL_GPL(inet_twsk_alloc);
 235
 236 /* These are always called from BH context.  See callers in
 237  * tcp_input.c to verify this.
 238  */
 239
 240 /* This is for handling early-kills of TIME_WAIT sockets. */
 241 void inet_twsk_deschedule(struct inet_timewait_sock *tw)
 242 {
 243         if (del_timer_sync(&tw->tw_timer))
 244                 inet_twsk_kill(tw);
 245 }
 246 EXPORT_SYMBOL(inet_twsk_deschedule);
 247
 248 void __inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo, bool rearm)
 249 {
 250         /* timeout := RTO * 3.5
 251          *
 252          * 3.5 = 1+2+0.5 to wait for two retransmits.
 253          *
 254          * RATIONALE: if FIN arrived and we entered TIME-WAIT state,
 255          * our ACK acking that FIN can be lost. If N subsequent retransmitted
 256          * FINs (or previous seqments) are lost (probability of such event
 257          * is p^(N+1), where p is probability to lose single packet and
 258          * time to detect the loss is about RTO*(2^N - 1) with exponential
 259          * backoff). Normal timewait length is calculated so, that we
 260          * waited at least for one retransmitted FIN (maximal RTO is 120sec).
 261          * [ BTW Linux. following BSD, violates this requirement waiting
 262          *   only for 60sec, we should wait at least for 240 secs.
 263          *   Well, 240 consumes too much of resources 8)
 264          * ]
 265          * This interval is not reduced to catch old duplicate and
 266          * responces to our wandering segments living for two MSLs.
 267          * However, if we use PAWS to detect
 268          * old duplicates, we can reduce the interval to bounds required
 269          * by RTO, rather than MSL. So, if peer understands PAWS, we
 270          * kill tw bucket after 3.5*RTO (it is important that this number
 271          * is greater than TS tick!) and detect old duplicates with help
 272          * of PAWS.
 273          */
 274
 275         tw->tw_kill = timeo <= 4*HZ;
 276         if (!rearm) {
 277                 BUG_ON(mod_timer_pinned(&tw->tw_timer, jiffies + timeo));
 278                 atomic_inc(&tw->tw_dr->tw_count);
 279         } else {
 280                 mod_timer_pending(&tw->tw_timer, jiffies + timeo);
 281         }
 282 }
 283 EXPORT_SYMBOL_GPL(__inet_twsk_schedule);
 284
 285 void inet_twsk_purge(struct inet_hashinfo *hashinfo,
 286                      struct inet_timewait_death_row *twdr, int family)
 287 {
 288         struct inet_timewait_sock *tw;
 289         struct sock *sk;
 290         struct hlist_nulls_node *node;
 291         unsigned int slot;
 292
 293         for (slot = 0; slot <= hashinfo->ehash_mask; slot++) {
 294                 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
 295 restart_rcu:
 296                 cond_resched();
 297                 rcu_read_lock();
 298 restart:
 299                 sk_nulls_for_each_rcu(sk, node, &head->chain) {
 300                         if (sk->sk_state != TCP_TIME_WAIT)
 301                                 continue;
 302                         tw = inet_twsk(sk);
 303                         if ((tw->tw_family != family) ||
 304                                 atomic_read(&twsk_net(tw)->count))
 305                                 continue;
 306
 307                         if (unlikely(!atomic_inc_not_zero(&tw->tw_refcnt)))
 308                                 continue;
 309
 310                         if (unlikely((tw->tw_family != family) ||
 311                                      atomic_read(&twsk_net(tw)->count))) {
 312                                 inet_twsk_put(tw);
 313                                 goto restart;
 314                         }
 315
 316                         rcu_read_unlock();
 317                         local_bh_disable();
 318                         inet_twsk_deschedule(tw);
 319                         local_bh_enable();
 320                         inet_twsk_put(tw);
 321                         goto restart_rcu;
 322                 }
 323                 /* If the nulls value we got at the end of this lookup is
 324                  * not the expected one, we must restart lookup.
 325                  * We probably met an item that was moved to another chain.
 326                  */
 327                 if (get_nulls_value(node) != slot)
 328                         goto restart;
 329                 rcu_read_unlock();
 330         }
 331 }
 332 EXPORT_SYMBOL_GPL(inet_twsk_purge);