drivers/net/wireguard/peer.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  */
   5
   6 #include "peer.h"
   7 #include "device.h"
   8 #include "queueing.h"
   9 #include "timers.h"
  10 #include "peerlookup.h"
  11 #include "noise.h"
  12
  13 #include <linux/kref.h>
  14 #include <linux/lockdep.h>
  15 #include <linux/rcupdate.h>
  16 #include <linux/list.h>
  17
  18 static atomic64_t peer_counter = ATOMIC64_INIT(0);
  19
  20 struct wg_peer *wg_peer_create(struct wg_device *wg,
  21                                const u8 public_key[NOISE_PUBLIC_KEY_LEN],
  22                                const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN])
  23 {
  24         struct wg_peer *peer;
  25         int ret = -ENOMEM;
  26
  27         lockdep_assert_held(&wg->device_update_lock);
  28
  29         if (wg->num_peers >= MAX_PEERS_PER_DEVICE)
  30                 return ERR_PTR(ret);
  31
  32         peer = kzalloc(sizeof(*peer), GFP_KERNEL);
  33         if (unlikely(!peer))
  34                 return ERR_PTR(ret);
  35         peer->device = wg;
  36
  37         if (!wg_noise_handshake_init(&peer->handshake, &wg->static_identity,
  38                                      public_key, preshared_key, peer)) {
  39                 ret = -EKEYREJECTED;
  40                 goto err_1;
  41         }
  42         if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL))
  43                 goto err_1;
  44         if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false,
  45                                  MAX_QUEUED_PACKETS))
  46                 goto err_2;
  47         if (wg_packet_queue_init(&peer->rx_queue, NULL, false,
  48                                  MAX_QUEUED_PACKETS))
  49                 goto err_3;
  50
  51         peer->internal_id = atomic64_inc_return(&peer_counter);
  52         peer->serial_work_cpu = nr_cpumask_bits;
  53         wg_cookie_init(&peer->latest_cookie);
  54         wg_timers_init(peer);
  55         wg_cookie_checker_precompute_peer_keys(peer);
  56         spin_lock_init(&peer->keypairs.keypair_update_lock);
  57         INIT_WORK(&peer->transmit_handshake_work,
  58                   wg_packet_handshake_send_worker);
  59         rwlock_init(&peer->endpoint_lock);
  60         kref_init(&peer->refcount);
  61         skb_queue_head_init(&peer->staged_packet_queue);
  62         wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
  63         set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state);
  64         netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll,
  65                        NAPI_POLL_WEIGHT);
  66         napi_enable(&peer->napi);
  67         list_add_tail(&peer->peer_list, &wg->peer_list);
  68         INIT_LIST_HEAD(&peer->allowedips_list);
  69         wg_pubkey_hashtable_add(wg->peer_hashtable, peer);
  70         ++wg->num_peers;
  71         pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id);
  72         return peer;
  73
  74 err_3:
  75         wg_packet_queue_free(&peer->tx_queue, false);
  76 err_2:
  77         dst_cache_destroy(&peer->endpoint_cache);
  78 err_1:
  79         kfree(peer);
  80         return ERR_PTR(ret);
  81 }
  82
  83 struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer)
  84 {
  85         RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(),
  86                          "Taking peer reference without holding the RCU read lock");
  87         if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount)))
  88                 return NULL;
  89         return peer;
  90 }
  91
  92 static void peer_make_dead(struct wg_peer *peer)
  93 {
  94         /* Remove from configuration-time lookup structures. */
  95         list_del_init(&peer->peer_list);
  96         wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer,
  97                                      &peer->device->device_update_lock);
  98         wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer);
  99
 100         /* Mark as dead, so that we don't allow jumping contexts after. */
 101         WRITE_ONCE(peer->is_dead, true);
 102
 103         /* The caller must now synchronize_rcu() for this to take effect. */
 104 }
 105
 106 static void peer_remove_after_dead(struct wg_peer *peer)
 107 {
 108         WARN_ON(!peer->is_dead);
 109
 110         /* No more keypairs can be created for this peer, since is_dead protects
 111          * add_new_keypair, so we can now destroy existing ones.
 112          */
 113         wg_noise_keypairs_clear(&peer->keypairs);
 114
 115         /* Destroy all ongoing timers that were in-flight at the beginning of
 116          * this function.
 117          */
 118         wg_timers_stop(peer);
 119
 120         /* The transition between packet encryption/decryption queues isn't
 121          * guarded by is_dead, but each reference's life is strictly bounded by
 122          * two generations: once for parallel crypto and once for serial
 123          * ingestion, so we can simply flush twice, and be sure that we no
 124          * longer have references inside these queues.
 125          */
 126
 127         /* a) For encrypt/decrypt. */
 128         flush_workqueue(peer->device->packet_crypt_wq);
 129         /* b.1) For send (but not receive, since that's napi). */
 130         flush_workqueue(peer->device->packet_crypt_wq);
 131         /* b.2.1) For receive (but not send, since that's wq). */
 132         napi_disable(&peer->napi);
 133         /* b.2.1) It's now safe to remove the napi struct, which must be done
 134          * here from process context.
 135          */
 136         netif_napi_del(&peer->napi);
 137
 138         /* Ensure any workstructs we own (like transmit_handshake_work or
 139          * clear_peer_work) no longer are in use.
 140          */
 141         flush_workqueue(peer->device->handshake_send_wq);
 142
 143         /* After the above flushes, a peer might still be active in a few
 144          * different contexts: 1) from xmit(), before hitting is_dead and
 145          * returning, 2) from wg_packet_consume_data(), before hitting is_dead
 146          * and returning, 3) from wg_receive_handshake_packet() after a point
 147          * where it has processed an incoming handshake packet, but where
 148          * all calls to pass it off to timers fails because of is_dead. We won't
 149          * have new references in (1) eventually, because we're removed from
 150          * allowedips; we won't have new references in (2) eventually, because
 151          * wg_index_hashtable_lookup will always return NULL, since we removed
 152          * all existing keypairs and no more can be created; we won't have new
 153          * references in (3) eventually, because we're removed from the pubkey
 154          * hash table, which allows for a maximum of one handshake response,
 155          * via the still-uncleared index hashtable entry, but not more than one,
 156          * and in wg_cookie_message_consume, the lookup eventually gets a peer
 157          * with a refcount of zero, so no new reference is taken.
 158          */
 159
 160         --peer->device->num_peers;
 161         wg_peer_put(peer);
 162 }
 163
 164 /* We have a separate "remove" function make sure that all active places where
 165  * a peer is currently operating will eventually come to an end and not pass
 166  * their reference onto another context.
 167  */
 168 void wg_peer_remove(struct wg_peer *peer)
 169 {
 170         if (unlikely(!peer))
 171                 return;
 172         lockdep_assert_held(&peer->device->device_update_lock);
 173
 174         peer_make_dead(peer);
 175         synchronize_rcu();
 176         peer_remove_after_dead(peer);
 177 }
 178
 179 void wg_peer_remove_all(struct wg_device *wg)
 180 {
 181         struct wg_peer *peer, *temp;
 182         LIST_HEAD(dead_peers);
 183
 184         lockdep_assert_held(&wg->device_update_lock);
 185
 186         /* Avoid having to traverse individually for each one. */
 187         wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock);
 188
 189         list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) {
 190                 peer_make_dead(peer);
 191                 list_add_tail(&peer->peer_list, &dead_peers);
 192         }
 193         synchronize_rcu();
 194         list_for_each_entry_safe(peer, temp, &dead_peers, peer_list)
 195                 peer_remove_after_dead(peer);
 196 }
 197
 198 static void rcu_release(struct rcu_head *rcu)
 199 {
 200         struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu);
 201
 202         dst_cache_destroy(&peer->endpoint_cache);
 203         wg_packet_queue_free(&peer->rx_queue, false);
 204         wg_packet_queue_free(&peer->tx_queue, false);
 205
 206         /* The final zeroing takes care of clearing any remaining handshake key
 207          * material and other potentially sensitive information.
 208          */
 209         kzfree(peer);
 210 }
 211
 212 static void kref_release(struct kref *refcount)
 213 {
 214         struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount);
 215
 216         pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n",
 217                  peer->device->dev->name, peer->internal_id,
 218                  &peer->endpoint.addr);
 219
 220         /* Remove ourself from dynamic runtime lookup structures, now that the
 221          * last reference is gone.
 222          */
 223         wg_index_hashtable_remove(peer->device->index_hashtable,
 224                                   &peer->handshake.entry);
 225
 226         /* Remove any lingering packets that didn't have a chance to be
 227          * transmitted.
 228          */
 229         wg_packet_purge_staged_packets(peer);
 230
 231         /* Free the memory used. */
 232         call_rcu(&peer->rcu, rcu_release);
 233 }
 234
 235 void wg_peer_put(struct wg_peer *peer)
 236 {
 237         if (unlikely(!peer))
 238                 return;
 239         kref_put(&peer->refcount, kref_release);
 240 }