drivers/net/wireguard/send.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  */
   5
   6 #include "queueing.h"
   7 #include "timers.h"
   8 #include "device.h"
   9 #include "peer.h"
  10 #include "socket.h"
  11 #include "messages.h"
  12 #include "cookie.h"
  13
  14 #include <linux/uio.h>
  15 #include <linux/inetdevice.h>
  16 #include <linux/socket.h>
  17 #include <net/ip_tunnels.h>
  18 #include <net/udp.h>
  19 #include <net/sock.h>
  20
  21 static void wg_packet_send_handshake_initiation(struct wg_peer *peer)
  22 {
  23         struct message_handshake_initiation packet;
  24
  25         if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
  26                                       REKEY_TIMEOUT))
  27                 return; /* This function is rate limited. */
  28
  29         atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
  30         net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n",
  31                             peer->device->dev->name, peer->internal_id,
  32                             &peer->endpoint.addr);
  33
  34         if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) {
  35                 wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
  36                 wg_timers_any_authenticated_packet_traversal(peer);
  37                 wg_timers_any_authenticated_packet_sent(peer);
  38                 atomic64_set(&peer->last_sent_handshake,
  39                              ktime_get_coarse_boottime_ns());
  40                 wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet),
  41                                               HANDSHAKE_DSCP);
  42                 wg_timers_handshake_initiated(peer);
  43         }
  44 }
  45
  46 void wg_packet_handshake_send_worker(struct work_struct *work)
  47 {
  48         struct wg_peer *peer = container_of(work, struct wg_peer,
  49                                             transmit_handshake_work);
  50
  51         wg_packet_send_handshake_initiation(peer);
  52         wg_peer_put(peer);
  53 }
  54
  55 void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer,
  56                                                 bool is_retry)
  57 {
  58         if (!is_retry)
  59                 peer->timer_handshake_attempts = 0;
  60
  61         rcu_read_lock_bh();
  62         /* We check last_sent_handshake here in addition to the actual function
  63          * we're queueing up, so that we don't queue things if not strictly
  64          * necessary:
  65          */
  66         if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
  67                                       REKEY_TIMEOUT) ||
  68                         unlikely(READ_ONCE(peer->is_dead)))
  69                 goto out;
  70
  71         wg_peer_get(peer);
  72         /* Queues up calling packet_send_queued_handshakes(peer), where we do a
  73          * peer_put(peer) after:
  74          */
  75         if (!queue_work(peer->device->handshake_send_wq,
  76                         &peer->transmit_handshake_work))
  77                 /* If the work was already queued, we want to drop the
  78                  * extra reference:
  79                  */
  80                 wg_peer_put(peer);
  81 out:
  82         rcu_read_unlock_bh();
  83 }
  84
  85 void wg_packet_send_handshake_response(struct wg_peer *peer)
  86 {
  87         struct message_handshake_response packet;
  88
  89         atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
  90         net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n",
  91                             peer->device->dev->name, peer->internal_id,
  92                             &peer->endpoint.addr);
  93
  94         if (wg_noise_handshake_create_response(&packet, &peer->handshake)) {
  95                 wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
  96                 if (wg_noise_handshake_begin_session(&peer->handshake,
  97                                                      &peer->keypairs)) {
  98                         wg_timers_session_derived(peer);
  99                         wg_timers_any_authenticated_packet_traversal(peer);
 100                         wg_timers_any_authenticated_packet_sent(peer);
 101                         atomic64_set(&peer->last_sent_handshake,
 102                                      ktime_get_coarse_boottime_ns());
 103                         wg_socket_send_buffer_to_peer(peer, &packet,
 104                                                       sizeof(packet),
 105                                                       HANDSHAKE_DSCP);
 106                 }
 107         }
 108 }
 109
 110 void wg_packet_send_handshake_cookie(struct wg_device *wg,
 111                                      struct sk_buff *initiating_skb,
 112                                      __le32 sender_index)
 113 {
 114         struct message_handshake_cookie packet;
 115
 116         net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n",
 117                                 wg->dev->name, initiating_skb);
 118         wg_cookie_message_create(&packet, initiating_skb, sender_index,
 119                                  &wg->cookie_checker);
 120         wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet,
 121                                               sizeof(packet));
 122 }
 123
 124 static void keep_key_fresh(struct wg_peer *peer)
 125 {
 126         struct noise_keypair *keypair;
 127         bool send = false;
 128
 129         rcu_read_lock_bh();
 130         keypair = rcu_dereference_bh(peer->keypairs.current_keypair);
 131         if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) &&
 132             (unlikely(atomic64_read(&keypair->sending.counter.counter) >
 133                       REKEY_AFTER_MESSAGES) ||
 134              (keypair->i_am_the_initiator &&
 135               unlikely(wg_birthdate_has_expired(keypair->sending.birthdate,
 136                                                 REKEY_AFTER_TIME)))))
 137                 send = true;
 138         rcu_read_unlock_bh();
 139
 140         if (send)
 141                 wg_packet_send_queued_handshake_initiation(peer, false);
 142 }
 143
 144 static unsigned int calculate_skb_padding(struct sk_buff *skb)
 145 {
 146         /* We do this modulo business with the MTU, just in case the networking
 147          * layer gives us a packet that's bigger than the MTU. In that case, we
 148          * wouldn't want the final subtraction to overflow in the case of the
 149          * padded_size being clamped.
 150          */
 151         unsigned int last_unit = skb->len % PACKET_CB(skb)->mtu;
 152         unsigned int padded_size = ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE);
 153
 154         if (padded_size > PACKET_CB(skb)->mtu)
 155                 padded_size = PACKET_CB(skb)->mtu;
 156         return padded_size - last_unit;
 157 }
 158
 159 static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair)
 160 {
 161         unsigned int padding_len, plaintext_len, trailer_len;
 162         struct scatterlist sg[MAX_SKB_FRAGS + 8];
 163         struct message_data *header;
 164         struct sk_buff *trailer;
 165         int num_frags;
 166
 167         /* Calculate lengths. */
 168         padding_len = calculate_skb_padding(skb);
 169         trailer_len = padding_len + noise_encrypted_len(0);
 170         plaintext_len = skb->len + padding_len;
 171
 172         /* Expand data section to have room for padding and auth tag. */
 173         num_frags = skb_cow_data(skb, trailer_len, &trailer);
 174         if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg)))
 175                 return false;
 176
 177         /* Set the padding to zeros, and make sure it and the auth tag are part
 178          * of the skb.
 179          */
 180         memset(skb_tail_pointer(trailer), 0, padding_len);
 181
 182         /* Expand head section to have room for our header and the network
 183          * stack's headers.
 184          */
 185         if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0))
 186                 return false;
 187
 188         /* Finalize checksum calculation for the inner packet, if required. */
 189         if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL &&
 190                      skb_checksum_help(skb)))
 191                 return false;
 192
 193         /* Only after checksumming can we safely add on the padding at the end
 194          * and the header.
 195          */
 196         skb_set_inner_network_header(skb, 0);
 197         header = (struct message_data *)skb_push(skb, sizeof(*header));
 198         header->header.type = cpu_to_le32(MESSAGE_DATA);
 199         header->key_idx = keypair->remote_index;
 200         header->counter = cpu_to_le64(PACKET_CB(skb)->nonce);
 201         pskb_put(skb, trailer, trailer_len);
 202
 203         /* Now we can encrypt the scattergather segments */
 204         sg_init_table(sg, num_frags);
 205         if (skb_to_sgvec(skb, sg, sizeof(struct message_data),
 206                          noise_encrypted_len(plaintext_len)) <= 0)
 207                 return false;
 208         return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0,
 209                                                    PACKET_CB(skb)->nonce,
 210                                                    keypair->sending.key);
 211 }
 212
 213 void wg_packet_send_keepalive(struct wg_peer *peer)
 214 {
 215         struct sk_buff *skb;
 216
 217         if (skb_queue_empty(&peer->staged_packet_queue)) {
 218                 skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH,
 219                                 GFP_ATOMIC);
 220                 if (unlikely(!skb))
 221                         return;
 222                 skb_reserve(skb, DATA_PACKET_HEAD_ROOM);
 223                 skb->dev = peer->device->dev;
 224                 PACKET_CB(skb)->mtu = skb->dev->mtu;
 225                 skb_queue_tail(&peer->staged_packet_queue, skb);
 226                 net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n",
 227                                     peer->device->dev->name, peer->internal_id,
 228                                     &peer->endpoint.addr);
 229         }
 230
 231         wg_packet_send_staged_packets(peer);
 232 }
 233
 234 static void wg_packet_create_data_done(struct sk_buff *first,
 235                                        struct wg_peer *peer)
 236 {
 237         struct sk_buff *skb, *next;
 238         bool is_keepalive, data_sent = false;
 239
 240         wg_timers_any_authenticated_packet_traversal(peer);
 241         wg_timers_any_authenticated_packet_sent(peer);
 242         skb_list_walk_safe(first, skb, next) {
 243                 is_keepalive = skb->len == message_data_len(0);
 244                 if (likely(!wg_socket_send_skb_to_peer(peer, skb,
 245                                 PACKET_CB(skb)->ds) && !is_keepalive))
 246                         data_sent = true;
 247         }
 248
 249         if (likely(data_sent))
 250                 wg_timers_data_sent(peer);
 251
 252         keep_key_fresh(peer);
 253 }
 254
 255 void wg_packet_tx_worker(struct work_struct *work)
 256 {
 257         struct crypt_queue *queue = container_of(work, struct crypt_queue,
 258                                                  work);
 259         struct noise_keypair *keypair;
 260         enum packet_state state;
 261         struct sk_buff *first;
 262         struct wg_peer *peer;
 263
 264         while ((first = __ptr_ring_peek(&queue->ring)) != NULL &&
 265                (state = atomic_read_acquire(&PACKET_CB(first)->state)) !=
 266                        PACKET_STATE_UNCRYPTED) {
 267                 __ptr_ring_discard_one(&queue->ring);
 268                 peer = PACKET_PEER(first);
 269                 keypair = PACKET_CB(first)->keypair;
 270
 271                 if (likely(state == PACKET_STATE_CRYPTED))
 272                         wg_packet_create_data_done(first, peer);
 273                 else
 274                         kfree_skb_list(first);
 275
 276                 wg_noise_keypair_put(keypair, false);
 277                 wg_peer_put(peer);
 278         }
 279 }
 280
 281 void wg_packet_encrypt_worker(struct work_struct *work)
 282 {
 283         struct crypt_queue *queue = container_of(work, struct multicore_worker,
 284                                                  work)->ptr;
 285         struct sk_buff *first, *skb, *next;
 286
 287         while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) {
 288                 enum packet_state state = PACKET_STATE_CRYPTED;
 289
 290                 skb_list_walk_safe(first, skb, next) {
 291                         if (likely(encrypt_packet(skb,
 292                                         PACKET_CB(first)->keypair))) {
 293                                 wg_reset_packet(skb);
 294                         } else {
 295                                 state = PACKET_STATE_DEAD;
 296                                 break;
 297                         }
 298                 }
 299                 wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first,
 300                                           state);
 301
 302         }
 303 }
 304
 305 static void wg_packet_create_data(struct sk_buff *first)
 306 {
 307         struct wg_peer *peer = PACKET_PEER(first);
 308         struct wg_device *wg = peer->device;
 309         int ret = -EINVAL;
 310
 311         rcu_read_lock_bh();
 312         if (unlikely(READ_ONCE(peer->is_dead)))
 313                 goto err;
 314
 315         ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue,
 316                                                    &peer->tx_queue, first,
 317                                                    wg->packet_crypt_wq,
 318                                                    &wg->encrypt_queue.last_cpu);
 319         if (unlikely(ret == -EPIPE))
 320                 wg_queue_enqueue_per_peer(&peer->tx_queue, first,
 321                                           PACKET_STATE_DEAD);
 322 err:
 323         rcu_read_unlock_bh();
 324         if (likely(!ret || ret == -EPIPE))
 325                 return;
 326         wg_noise_keypair_put(PACKET_CB(first)->keypair, false);
 327         wg_peer_put(peer);
 328         kfree_skb_list(first);
 329 }
 330
 331 void wg_packet_purge_staged_packets(struct wg_peer *peer)
 332 {
 333         spin_lock_bh(&peer->staged_packet_queue.lock);
 334         peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen;
 335         __skb_queue_purge(&peer->staged_packet_queue);
 336         spin_unlock_bh(&peer->staged_packet_queue.lock);
 337 }
 338
 339 void wg_packet_send_staged_packets(struct wg_peer *peer)
 340 {
 341         struct noise_symmetric_key *key;
 342         struct noise_keypair *keypair;
 343         struct sk_buff_head packets;
 344         struct sk_buff *skb;
 345
 346         /* Steal the current queue into our local one. */
 347         __skb_queue_head_init(&packets);
 348         spin_lock_bh(&peer->staged_packet_queue.lock);
 349         skb_queue_splice_init(&peer->staged_packet_queue, &packets);
 350         spin_unlock_bh(&peer->staged_packet_queue.lock);
 351         if (unlikely(skb_queue_empty(&packets)))
 352                 return;
 353
 354         /* First we make sure we have a valid reference to a valid key. */
 355         rcu_read_lock_bh();
 356         keypair = wg_noise_keypair_get(
 357                 rcu_dereference_bh(peer->keypairs.current_keypair));
 358         rcu_read_unlock_bh();
 359         if (unlikely(!keypair))
 360                 goto out_nokey;
 361         key = &keypair->sending;
 362         if (unlikely(!READ_ONCE(key->is_valid)))
 363                 goto out_nokey;
 364         if (unlikely(wg_birthdate_has_expired(key->birthdate,
 365                                               REJECT_AFTER_TIME)))
 366                 goto out_invalid;
 367
 368         /* After we know we have a somewhat valid key, we now try to assign
 369          * nonces to all of the packets in the queue. If we can't assign nonces
 370          * for all of them, we just consider it a failure and wait for the next
 371          * handshake.
 372          */
 373         skb_queue_walk(&packets, skb) {
 374                 /* 0 for no outer TOS: no leak. TODO: at some later point, we
 375                  * might consider using flowi->tos as outer instead.
 376                  */
 377                 PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb);
 378                 PACKET_CB(skb)->nonce =
 379                                 atomic64_inc_return(&key->counter.counter) - 1;
 380                 if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES))
 381                         goto out_invalid;
 382         }
 383
 384         packets.prev->next = NULL;
 385         wg_peer_get(keypair->entry.peer);
 386         PACKET_CB(packets.next)->keypair = keypair;
 387         wg_packet_create_data(packets.next);
 388         return;
 389
 390 out_invalid:
 391         WRITE_ONCE(key->is_valid, false);
 392 out_nokey:
 393         wg_noise_keypair_put(keypair, false);
 394
 395         /* We orphan the packets if we're waiting on a handshake, so that they
 396          * don't block a socket's pool.
 397          */
 398         skb_queue_walk(&packets, skb)
 399                 skb_orphan(skb);
 400         /* Then we put them back on the top of the queue. We're not too
 401          * concerned about accidentally getting things a little out of order if
 402          * packets are being added really fast, because this queue is for before
 403          * packets can even be sent and it's small anyway.
 404          */
 405         spin_lock_bh(&peer->staged_packet_queue.lock);
 406         skb_queue_splice(&packets, &peer->staged_packet_queue);
 407         spin_unlock_bh(&peer->staged_packet_queue.lock);
 408
 409         /* If we're exiting because there's something wrong with the key, it
 410          * means we should initiate a new handshake.
 411          */
 412         wg_packet_send_queued_handshake_initiation(peer, false);
 413 }