drivers/crypto/chelsio/chtls/chtls_main.c

   1 /*
   2  * Copyright (c) 2018 Chelsio Communications, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * Written by: Atul Gupta (atul.gupta@chelsio.com)
   9  */
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/skbuff.h>
  13 #include <linux/socket.h>
  14 #include <linux/hash.h>
  15 #include <linux/in.h>
  16 #include <linux/net.h>
  17 #include <linux/ip.h>
  18 #include <linux/tcp.h>
  19 #include <net/tcp.h>
  20 #include <net/tls.h>
  21
  22 #include "chtls.h"
  23 #include "chtls_cm.h"
  24
  25 #define DRV_NAME "chtls"
  26
  27 /*
  28  * chtls device management
  29  * maintains a list of the chtls devices
  30  */
  31 static LIST_HEAD(cdev_list);
  32 static DEFINE_MUTEX(cdev_mutex);
  33 static DEFINE_MUTEX(cdev_list_lock);
  34
  35 static DEFINE_MUTEX(notify_mutex);
  36 static RAW_NOTIFIER_HEAD(listen_notify_list);
  37 static struct proto chtls_cpl_prot;
  38 struct request_sock_ops chtls_rsk_ops;
  39 static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
  40
  41 static void register_listen_notifier(struct notifier_block *nb)
  42 {
  43         mutex_lock(&notify_mutex);
  44         raw_notifier_chain_register(&listen_notify_list, nb);
  45         mutex_unlock(&notify_mutex);
  46 }
  47
  48 static void unregister_listen_notifier(struct notifier_block *nb)
  49 {
  50         mutex_lock(&notify_mutex);
  51         raw_notifier_chain_unregister(&listen_notify_list, nb);
  52         mutex_unlock(&notify_mutex);
  53 }
  54
  55 static int listen_notify_handler(struct notifier_block *this,
  56                                  unsigned long event, void *data)
  57 {
  58         struct chtls_dev *cdev;
  59         struct sock *sk;
  60         int ret;
  61
  62         sk = data;
  63         ret =  NOTIFY_DONE;
  64
  65         switch (event) {
  66         case CHTLS_LISTEN_START:
  67         case CHTLS_LISTEN_STOP:
  68                 mutex_lock(&cdev_list_lock);
  69                 list_for_each_entry(cdev, &cdev_list, list) {
  70                         if (event == CHTLS_LISTEN_START)
  71                                 ret = chtls_listen_start(cdev, sk);
  72                         else
  73                                 chtls_listen_stop(cdev, sk);
  74                 }
  75                 mutex_unlock(&cdev_list_lock);
  76                 break;
  77         }
  78         return ret;
  79 }
  80
  81 static struct notifier_block listen_notifier = {
  82         .notifier_call = listen_notify_handler
  83 };
  84
  85 static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
  86 {
  87         if (likely(skb_transport_header(skb) != skb_network_header(skb)))
  88                 return tcp_v4_do_rcv(sk, skb);
  89         BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
  90         return 0;
  91 }
  92
  93 static int chtls_start_listen(struct sock *sk)
  94 {
  95         int err;
  96
  97         if (sk->sk_protocol != IPPROTO_TCP)
  98                 return -EPROTONOSUPPORT;
  99
 100         if (sk->sk_family == PF_INET &&
 101             LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
 102                 return -EADDRNOTAVAIL;
 103
 104         sk->sk_backlog_rcv = listen_backlog_rcv;
 105         mutex_lock(&notify_mutex);
 106         err = raw_notifier_call_chain(&listen_notify_list,
 107                                       CHTLS_LISTEN_START, sk);
 108         mutex_unlock(&notify_mutex);
 109         return err;
 110 }
 111
 112 static void chtls_stop_listen(struct sock *sk)
 113 {
 114         if (sk->sk_protocol != IPPROTO_TCP)
 115                 return;
 116
 117         mutex_lock(&notify_mutex);
 118         raw_notifier_call_chain(&listen_notify_list,
 119                                 CHTLS_LISTEN_STOP, sk);
 120         mutex_unlock(&notify_mutex);
 121 }
 122
 123 static int chtls_inline_feature(struct tls_device *dev)
 124 {
 125         struct net_device *netdev;
 126         struct chtls_dev *cdev;
 127         int i;
 128
 129         cdev = to_chtls_dev(dev);
 130
 131         for (i = 0; i < cdev->lldi->nports; i++) {
 132                 netdev = cdev->ports[i];
 133                 if (netdev->features & NETIF_F_HW_TLS_RECORD)
 134                         return 1;
 135         }
 136         return 0;
 137 }
 138
 139 static int chtls_create_hash(struct tls_device *dev, struct sock *sk)
 140 {
 141         if (sk->sk_state == TCP_LISTEN)
 142                 return chtls_start_listen(sk);
 143         return 0;
 144 }
 145
 146 static void chtls_destroy_hash(struct tls_device *dev, struct sock *sk)
 147 {
 148         if (sk->sk_state == TCP_LISTEN)
 149                 chtls_stop_listen(sk);
 150 }
 151
 152 static void chtls_register_dev(struct chtls_dev *cdev)
 153 {
 154         struct tls_device *tlsdev = &cdev->tlsdev;
 155
 156         strlcpy(tlsdev->name, "chtls", TLS_DEVICE_NAME_MAX);
 157         strlcat(tlsdev->name, cdev->lldi->ports[0]->name,
 158                 TLS_DEVICE_NAME_MAX);
 159         tlsdev->feature = chtls_inline_feature;
 160         tlsdev->hash = chtls_create_hash;
 161         tlsdev->unhash = chtls_destroy_hash;
 162         tls_register_device(&cdev->tlsdev);
 163         cdev->cdev_state = CHTLS_CDEV_STATE_UP;
 164 }
 165
 166 static void chtls_unregister_dev(struct chtls_dev *cdev)
 167 {
 168         tls_unregister_device(&cdev->tlsdev);
 169 }
 170
 171 static void process_deferq(struct work_struct *task_param)
 172 {
 173         struct chtls_dev *cdev = container_of(task_param,
 174                                 struct chtls_dev, deferq_task);
 175         struct sk_buff *skb;
 176
 177         spin_lock_bh(&cdev->deferq.lock);
 178         while ((skb = __skb_dequeue(&cdev->deferq)) != NULL) {
 179                 spin_unlock_bh(&cdev->deferq.lock);
 180                 DEFERRED_SKB_CB(skb)->handler(cdev, skb);
 181                 spin_lock_bh(&cdev->deferq.lock);
 182         }
 183         spin_unlock_bh(&cdev->deferq.lock);
 184 }
 185
 186 static int chtls_get_skb(struct chtls_dev *cdev)
 187 {
 188         cdev->askb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL);
 189         if (!cdev->askb)
 190                 return -ENOMEM;
 191
 192         skb_put(cdev->askb, sizeof(struct tcphdr));
 193         skb_reset_transport_header(cdev->askb);
 194         memset(cdev->askb->data, 0, cdev->askb->len);
 195         return 0;
 196 }
 197
 198 static void *chtls_uld_add(const struct cxgb4_lld_info *info)
 199 {
 200         struct cxgb4_lld_info *lldi;
 201         struct chtls_dev *cdev;
 202         int i, j;
 203
 204         cdev = kzalloc(sizeof(*cdev) + info->nports *
 205                       (sizeof(struct net_device *)), GFP_KERNEL);
 206         if (!cdev)
 207                 goto out;
 208
 209         lldi = kzalloc(sizeof(*lldi), GFP_KERNEL);
 210         if (!lldi)
 211                 goto out_lldi;
 212
 213         if (chtls_get_skb(cdev))
 214                 goto out_skb;
 215
 216         *lldi = *info;
 217         cdev->lldi = lldi;
 218         cdev->pdev = lldi->pdev;
 219         cdev->tids = lldi->tids;
 220         cdev->ports = lldi->ports;
 221         cdev->mtus = lldi->mtus;
 222         cdev->tids = lldi->tids;
 223         cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0]))
 224                         << FW_VIID_PFN_S;
 225
 226         for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) {
 227                 unsigned int size = 64 - sizeof(struct rsp_ctrl) - 8;
 228
 229                 cdev->rspq_skb_cache[i] = __alloc_skb(size,
 230                                                       gfp_any(), 0,
 231                                                       lldi->nodeid);
 232                 if (unlikely(!cdev->rspq_skb_cache[i]))
 233                         goto out_rspq_skb;
 234         }
 235
 236         idr_init(&cdev->hwtid_idr);
 237         INIT_WORK(&cdev->deferq_task, process_deferq);
 238         spin_lock_init(&cdev->listen_lock);
 239         spin_lock_init(&cdev->idr_lock);
 240         cdev->send_page_order = min_t(uint, get_order(32768),
 241                                       send_page_order);
 242         cdev->max_host_sndbuf = 48 * 1024;
 243
 244         if (lldi->vr->key.size)
 245                 if (chtls_init_kmap(cdev, lldi))
 246                         goto out_rspq_skb;
 247
 248         mutex_lock(&cdev_mutex);
 249         list_add_tail(&cdev->list, &cdev_list);
 250         mutex_unlock(&cdev_mutex);
 251
 252         return cdev;
 253 out_rspq_skb:
 254         for (j = 0; j < i; j++)
 255                 kfree_skb(cdev->rspq_skb_cache[j]);
 256         kfree_skb(cdev->askb);
 257 out_skb:
 258         kfree(lldi);
 259 out_lldi:
 260         kfree(cdev);
 261 out:
 262         return NULL;
 263 }
 264
 265 static void chtls_free_uld(struct chtls_dev *cdev)
 266 {
 267         int i;
 268
 269         chtls_unregister_dev(cdev);
 270         kvfree(cdev->kmap.addr);
 271         idr_destroy(&cdev->hwtid_idr);
 272         for (i = 0; i < (1 << RSPQ_HASH_BITS); i++)
 273                 kfree_skb(cdev->rspq_skb_cache[i]);
 274         kfree(cdev->lldi);
 275         if (cdev->askb)
 276                 kfree_skb(cdev->askb);
 277         kfree(cdev);
 278 }
 279
 280 static void chtls_free_all_uld(void)
 281 {
 282         struct chtls_dev *cdev, *tmp;
 283
 284         mutex_lock(&cdev_mutex);
 285         list_for_each_entry_safe(cdev, tmp, &cdev_list, list) {
 286                 if (cdev->cdev_state == CHTLS_CDEV_STATE_UP)
 287                         chtls_free_uld(cdev);
 288         }
 289         mutex_unlock(&cdev_mutex);
 290 }
 291
 292 static int chtls_uld_state_change(void *handle, enum cxgb4_state new_state)
 293 {
 294         struct chtls_dev *cdev = handle;
 295
 296         switch (new_state) {
 297         case CXGB4_STATE_UP:
 298                 chtls_register_dev(cdev);
 299                 break;
 300         case CXGB4_STATE_DOWN:
 301                 break;
 302         case CXGB4_STATE_START_RECOVERY:
 303                 break;
 304         case CXGB4_STATE_DETACH:
 305                 mutex_lock(&cdev_mutex);
 306                 list_del(&cdev->list);
 307                 mutex_unlock(&cdev_mutex);
 308                 chtls_free_uld(cdev);
 309                 break;
 310         default:
 311                 break;
 312         }
 313         return 0;
 314 }
 315
 316 static struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl,
 317                                           const __be64 *rsp,
 318                                           u32 pktshift)
 319 {
 320         struct sk_buff *skb;
 321
 322         /* Allocate space for cpl_pass_accpet_req which will be synthesized by
 323          * driver. Once driver synthesizes cpl_pass_accpet_req the skb will go
 324          * through the regular cpl_pass_accept_req processing in TOM.
 325          */
 326         skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req)
 327                         - pktshift, GFP_ATOMIC);
 328         if (unlikely(!skb))
 329                 return NULL;
 330         __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req)
 331                    - pktshift);
 332         /* For now we will copy  cpl_rx_pkt in the skb */
 333         skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_rx_pkt));
 334         skb_copy_to_linear_data_offset(skb, sizeof(struct cpl_pass_accept_req)
 335                                        , gl->va + pktshift,
 336                                        gl->tot_len - pktshift);
 337
 338         return skb;
 339 }
 340
 341 static int chtls_recv_packet(struct chtls_dev *cdev,
 342                              const struct pkt_gl *gl, const __be64 *rsp)
 343 {
 344         unsigned int opcode = *(u8 *)rsp;
 345         struct sk_buff *skb;
 346         int ret;
 347
 348         skb = copy_gl_to_skb_pkt(gl, rsp, cdev->lldi->sge_pktshift);
 349         if (!skb)
 350                 return -ENOMEM;
 351
 352         ret = chtls_handlers[opcode](cdev, skb);
 353         if (ret & CPL_RET_BUF_DONE)
 354                 kfree_skb(skb);
 355
 356         return 0;
 357 }
 358
 359 static int chtls_recv_rsp(struct chtls_dev *cdev, const __be64 *rsp)
 360 {
 361         unsigned long rspq_bin;
 362         unsigned int opcode;
 363         struct sk_buff *skb;
 364         unsigned int len;
 365         int ret;
 366
 367         len = 64 - sizeof(struct rsp_ctrl) - 8;
 368         opcode = *(u8 *)rsp;
 369
 370         rspq_bin = hash_ptr((void *)rsp, RSPQ_HASH_BITS);
 371         skb = cdev->rspq_skb_cache[rspq_bin];
 372         if (skb && !skb_is_nonlinear(skb) &&
 373             !skb_shared(skb) && !skb_cloned(skb)) {
 374                 refcount_inc(&skb->users);
 375                 if (refcount_read(&skb->users) == 2) {
 376                         __skb_trim(skb, 0);
 377                         if (skb_tailroom(skb) >= len)
 378                                 goto copy_out;
 379                 }
 380                 refcount_dec(&skb->users);
 381         }
 382         skb = alloc_skb(len, GFP_ATOMIC);
 383         if (unlikely(!skb))
 384                 return -ENOMEM;
 385
 386 copy_out:
 387         __skb_put(skb, len);
 388         skb_copy_to_linear_data(skb, rsp, len);
 389         skb_reset_network_header(skb);
 390         skb_reset_transport_header(skb);
 391         ret = chtls_handlers[opcode](cdev, skb);
 392
 393         if (ret & CPL_RET_BUF_DONE)
 394                 kfree_skb(skb);
 395         return 0;
 396 }
 397
 398 static void chtls_recv(struct chtls_dev *cdev,
 399                        struct sk_buff **skbs, const __be64 *rsp)
 400 {
 401         struct sk_buff *skb = *skbs;
 402         unsigned int opcode;
 403         int ret;
 404
 405         opcode = *(u8 *)rsp;
 406
 407         __skb_push(skb, sizeof(struct rss_header));
 408         skb_copy_to_linear_data(skb, rsp, sizeof(struct rss_header));
 409
 410         ret = chtls_handlers[opcode](cdev, skb);
 411         if (ret & CPL_RET_BUF_DONE)
 412                 kfree_skb(skb);
 413 }
 414
 415 static int chtls_uld_rx_handler(void *handle, const __be64 *rsp,
 416                                 const struct pkt_gl *gl)
 417 {
 418         struct chtls_dev *cdev = handle;
 419         unsigned int opcode;
 420         struct sk_buff *skb;
 421
 422         opcode = *(u8 *)rsp;
 423
 424         if (unlikely(opcode == CPL_RX_PKT)) {
 425                 if (chtls_recv_packet(cdev, gl, rsp) < 0)
 426                         goto nomem;
 427                 return 0;
 428         }
 429
 430         if (!gl)
 431                 return chtls_recv_rsp(cdev, rsp);
 432
 433 #define RX_PULL_LEN 128
 434         skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
 435         if (unlikely(!skb))
 436                 goto nomem;
 437         chtls_recv(cdev, &skb, rsp);
 438         return 0;
 439
 440 nomem:
 441         return -ENOMEM;
 442 }
 443
 444 static int do_chtls_getsockopt(struct sock *sk, char __user *optval,
 445                                int __user *optlen)
 446 {
 447         struct tls_crypto_info crypto_info = { 0 };
 448
 449         crypto_info.version = TLS_1_2_VERSION;
 450         if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info)))
 451                 return -EFAULT;
 452         return 0;
 453 }
 454
 455 static int chtls_getsockopt(struct sock *sk, int level, int optname,
 456                             char __user *optval, int __user *optlen)
 457 {
 458         struct tls_context *ctx = tls_get_ctx(sk);
 459
 460         if (level != SOL_TLS)
 461                 return ctx->getsockopt(sk, level, optname, optval, optlen);
 462
 463         return do_chtls_getsockopt(sk, optval, optlen);
 464 }
 465
 466 static int do_chtls_setsockopt(struct sock *sk, int optname,
 467                                char __user *optval, unsigned int optlen)
 468 {
 469         struct tls_crypto_info *crypto_info, tmp_crypto_info;
 470         struct chtls_sock *csk;
 471         int keylen;
 472         int rc = 0;
 473
 474         csk = rcu_dereference_sk_user_data(sk);
 475
 476         if (!optval || optlen < sizeof(*crypto_info)) {
 477                 rc = -EINVAL;
 478                 goto out;
 479         }
 480
 481         rc = copy_from_user(&tmp_crypto_info, optval, sizeof(*crypto_info));
 482         if (rc) {
 483                 rc = -EFAULT;
 484                 goto out;
 485         }
 486
 487         /* check version */
 488         if (tmp_crypto_info.version != TLS_1_2_VERSION) {
 489                 rc = -ENOTSUPP;
 490                 goto out;
 491         }
 492
 493         crypto_info = (struct tls_crypto_info *)&csk->tlshws.crypto_info;
 494
 495         switch (tmp_crypto_info.cipher_type) {
 496         case TLS_CIPHER_AES_GCM_128: {
 497                 /* Obtain version and type from previous copy */
 498                 crypto_info[0] = tmp_crypto_info;
 499                 /* Now copy the following data */
 500                 rc = copy_from_user((char *)crypto_info + sizeof(*crypto_info),
 501                                 optval + sizeof(*crypto_info),
 502                                 sizeof(struct tls12_crypto_info_aes_gcm_128)
 503                                 - sizeof(*crypto_info));
 504
 505                 if (rc) {
 506                         rc = -EFAULT;
 507                         goto out;
 508                 }
 509
 510                 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
 511                 rc = chtls_setkey(csk, keylen, optname);
 512                 break;
 513         }
 514         default:
 515                 rc = -EINVAL;
 516                 goto out;
 517         }
 518 out:
 519         return rc;
 520 }
 521
 522 static int chtls_setsockopt(struct sock *sk, int level, int optname,
 523                             char __user *optval, unsigned int optlen)
 524 {
 525         struct tls_context *ctx = tls_get_ctx(sk);
 526
 527         if (level != SOL_TLS)
 528                 return ctx->setsockopt(sk, level, optname, optval, optlen);
 529
 530         return do_chtls_setsockopt(sk, optname, optval, optlen);
 531 }
 532
 533 static struct cxgb4_uld_info chtls_uld_info = {
 534         .name = DRV_NAME,
 535         .nrxq = MAX_ULD_QSETS,
 536         .ntxq = MAX_ULD_QSETS,
 537         .rxq_size = 1024,
 538         .add = chtls_uld_add,
 539         .state_change = chtls_uld_state_change,
 540         .rx_handler = chtls_uld_rx_handler,
 541 };
 542
 543 void chtls_install_cpl_ops(struct sock *sk)
 544 {
 545         sk->sk_prot = &chtls_cpl_prot;
 546 }
 547
 548 static void __init chtls_init_ulp_ops(void)
 549 {
 550         chtls_cpl_prot                  = tcp_prot;
 551         chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops,
 552                            &tcp_prot, PF_INET);
 553         chtls_cpl_prot.close            = chtls_close;
 554         chtls_cpl_prot.disconnect       = chtls_disconnect;
 555         chtls_cpl_prot.destroy          = chtls_destroy_sock;
 556         chtls_cpl_prot.shutdown         = chtls_shutdown;
 557         chtls_cpl_prot.sendmsg          = chtls_sendmsg;
 558         chtls_cpl_prot.sendpage         = chtls_sendpage;
 559         chtls_cpl_prot.recvmsg          = chtls_recvmsg;
 560         chtls_cpl_prot.setsockopt       = chtls_setsockopt;
 561         chtls_cpl_prot.getsockopt       = chtls_getsockopt;
 562 }
 563
 564 static int __init chtls_register(void)
 565 {
 566         chtls_init_ulp_ops();
 567         register_listen_notifier(&listen_notifier);
 568         cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info);
 569         return 0;
 570 }
 571
 572 static void __exit chtls_unregister(void)
 573 {
 574         unregister_listen_notifier(&listen_notifier);
 575         chtls_free_all_uld();
 576         cxgb4_unregister_uld(CXGB4_ULD_TLS);
 577 }
 578
 579 module_init(chtls_register);
 580 module_exit(chtls_unregister);
 581
 582 MODULE_DESCRIPTION("Chelsio TLS Inline driver");
 583 MODULE_LICENSE("GPL");
 584 MODULE_AUTHOR("Chelsio Communications");
 585 MODULE_VERSION(DRV_VERSION);