zc.c

   1 /*
   2  * Driver for /dev/crypto device (aka CryptoDev)
   3  *
   4  * Copyright (c) 2009-2011 Nikos Mavrogiannopoulos <nmav@gnutls.org>
   5  * Copyright (c) 2010 Phil Sutter
   6  * Copyright (c) 2011, 2012 OpenSSL Software Foundation, Inc.
   7  *
   8  * This file is part of linux cryptodev.
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public License
  12  * as published by the Free Software Foundation; either version 2
  13  * of the License, or (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23  * 02110-1301, USA.
  24  */
  25
  26 #include <crypto/hash.h>
  27 #include <linux/crypto.h>
  28 #include <linux/mm.h>
  29 #include <linux/highmem.h>
  30 #include <linux/ioctl.h>
  31 #include <linux/random.h>
  32 #include <linux/syscalls.h>
  33 #include <linux/pagemap.h>
  34 #include <linux/uaccess.h>
  35 #include <crypto/scatterwalk.h>
  36 #include <linux/scatterlist.h>
  37 #include "cryptodev_int.h"
  38 #include "zc.h"
  39 #include "version.h"
  40
  41 /* Helper functions to assist zero copy.
  42  * This needs to be redesigned and moved out of the session. --nmav
  43  */
  44
  45 /* offset of buf in it's first page */
  46 #define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK)
  47
  48 /* fetch the pages addr resides in into pg and initialise sg with them */
  49 int __get_userbuf(uint8_t __user *addr, uint32_t len, int write,
  50                 unsigned int pgcount, struct page **pg, struct scatterlist *sg,
  51                 struct task_struct *task, struct mm_struct *mm)
  52 {
  53         int ret, pglen, i = 0;
  54         struct scatterlist *sgp;
  55
  56         if (unlikely(!pgcount || !len || !addr)) {
  57                 sg_mark_end(sg);
  58         }
  59         else {
  60                 down_write(&mm->mmap_sem);
  61                 ret = get_user_pages(task, mm,
  62                                 (unsigned long)addr, pgcount, write, 0, pg, NULL);
  63                 up_write(&mm->mmap_sem);
  64                 if (ret != pgcount)
  65                         return -EINVAL;
  66
  67                 sg_init_table(sg, pgcount);
  68
  69                 pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len);
  70                 sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr));
  71
  72                 len -= pglen;
  73                 for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) {
  74                         pglen = min((uint32_t)PAGE_SIZE, len);
  75                         sg_set_page(sgp, pg[i++], pglen, 0);
  76                         len -= pglen;
  77                 }
  78                 sg_mark_end(sg_last(sg, pgcount));
  79         }
  80         return 0;
  81 }
  82
  83 int adjust_sg_array(struct csession * ses, int pagecount)
  84 {
  85         struct scatterlist *sg;
  86         struct page **pages;
  87         int array_size;
  88
  89         for (array_size = ses->array_size; array_size < pagecount;
  90              array_size *= 2)
  91                 ;
  92         dprintk(0, KERN_DEBUG, "reallocating from %d to %d pages\n",
  93                         ses->array_size, array_size);
  94         pages = krealloc(ses->pages, array_size * sizeof(struct page *),
  95                          GFP_KERNEL);
  96         if (unlikely(!pages))
  97                 return -ENOMEM;
  98         ses->pages = pages;
  99         sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist),
 100                       GFP_KERNEL);
 101         if (unlikely(!sg))
 102                 return -ENOMEM;
 103         ses->sg = sg;
 104         ses->array_size = array_size;
 105
 106         return 0;
 107 }
 108
 109 void release_user_pages(struct csession *ses)
 110 {
 111 unsigned int i;
 112
 113         for (i=0;i<ses->used_pages;i++) {
 114                 if (!PageReserved(ses->pages[i]))
 115                         SetPageDirty(ses->pages[i]);
 116
 117                 if (ses->readonly_pages == 0)
 118                         flush_dcache_page(ses->pages[i]);
 119                 else
 120                         ses->readonly_pages--;
 121
 122                 page_cache_release(ses->pages[i]);
 123         }
 124         ses->used_pages = 0;
 125 }
 126
 127 /* make src and dst available in scatterlists.
 128  * dst might be the same as src.
 129  */
 130 int get_userbuf(struct csession *ses,
 131                 void* __user src, unsigned int src_len,
 132                 void* __user dst, unsigned int dst_len,
 133                 struct task_struct *task, struct mm_struct *mm,
 134                 struct scatterlist **src_sg,
 135                 struct scatterlist **dst_sg)
 136 {
 137         int src_pagecount, dst_pagecount;
 138         int rc;
 139
 140         /* Empty input is a valid option to many algorithms & is tested by NIST/FIPS */
 141         /* Make sure NULL input has 0 length */
 142         if (!src && src_len) { src_len = 0; }
 143
 144         /* I don't know that null output is ever useful, but we can handle it gracefully */
 145         /* Make sure NULL output has 0 length */
 146         if (!dst && dst_len) { dst_len = 0; }
 147
 148         if (ses->alignmask && !IS_ALIGNED((unsigned long)src, ses->alignmask)) {
 149                 dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
 150                                 __func__, (unsigned long)src, ses->alignmask + 1);
 151         }
 152
 153         if (ses->alignmask && !IS_ALIGNED((unsigned long)dst, ses->alignmask)) {
 154                 dprintk(2, KERN_WARNING, "%s: careful - destination address %lx is not %d byte aligned\n",
 155                                 __func__, (unsigned long)dst, ses->alignmask + 1);
 156         }
 157
 158         src_pagecount = PAGECOUNT(src, src_len);
 159         dst_pagecount = PAGECOUNT(dst, dst_len);
 160
 161         ses->used_pages = (src == dst) ? max(src_pagecount, dst_pagecount)
 162                                        : src_pagecount + dst_pagecount;
 163
 164         ses->readonly_pages = (src == dst) ? 0 : src_pagecount;
 165
 166         if (ses->used_pages > ses->array_size) {
 167                 rc = adjust_sg_array(ses, ses->used_pages);
 168                 if (rc)
 169                         return rc;
 170         }
 171
 172         if (src == dst) {
 173                 rc = __get_userbuf(src, src_len, 1, ses->used_pages,
 174                                        ses->pages, ses->sg, task, mm);
 175                 if (unlikely(rc)) {
 176                         dprintk(1, KERN_ERR,
 177                                 "failed to get user pages for data IO\n");
 178                         return rc;
 179                 }
 180                 (*src_sg) = (*dst_sg) = ses->sg;
 181         }
 182         else {
 183                 const unsigned int readonly_pages = ses->readonly_pages;
 184                 const unsigned int writable_pages = ses->used_pages - readonly_pages;
 185
 186                 if(likely(src)) {
 187                         rc = __get_userbuf(src, src_len, 0, readonly_pages,
 188                                                    ses->pages, ses->sg, task, mm);
 189                         if (unlikely(rc)) {
 190                                 dprintk(1, KERN_ERR,
 191                                         "failed to get user pages for data input\n");
 192                                 return rc;
 193                         }
 194                         *src_sg = ses->sg;
 195                 }
 196                 else {
 197                         *src_sg = NULL; // no input
 198                 }
 199
 200                 if(likely(dst)) {
 201                         struct page **dst_pages = ses->pages + readonly_pages;
 202                         *dst_sg = ses->sg + readonly_pages;
 203
 204                         rc = __get_userbuf(dst, dst_len, 1, writable_pages,
 205                                                    dst_pages, *dst_sg, task, mm);
 206                         if (unlikely(rc)) {
 207                                 dprintk(1, KERN_ERR,
 208                                                 "failed to get user pages for data output\n");
 209                                 release_user_pages(ses);  /* FIXME: use __release_userbuf(src, ...) */
 210                                 return rc;
 211                         }
 212                 }
 213                 else {
 214                         *dst_sg = NULL; // ignore output
 215                 }
 216         }
 217         return 0;
 218 }
 219