drivers/block/zram/zcomp.c

   1 /*
   2  * Copyright (C) 2014 Sergey Senozhatsky.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; either version
   7  * 2 of the License, or (at your option) any later version.
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/string.h>
  12 #include <linux/err.h>
  13 #include <linux/slab.h>
  14 #include <linux/wait.h>
  15 #include <linux/sched.h>
  16
  17 #include "zcomp.h"
  18 #include "zcomp_lzo.h"
  19 #ifdef CONFIG_ZRAM_LZ4_COMPRESS
  20 #include "zcomp_lz4.h"
  21 #endif
  22
  23 /*
  24  * single zcomp_strm backend
  25  */
  26 struct zcomp_strm_single {
  27         struct mutex strm_lock;
  28         struct zcomp_strm *zstrm;
  29 };
  30
  31 /*
  32  * multi zcomp_strm backend
  33  */
  34 struct zcomp_strm_multi {
  35         /* protect strm list */
  36         spinlock_t strm_lock;
  37         /* max possible number of zstrm streams */
  38         int max_strm;
  39         /* number of available zstrm streams */
  40         int avail_strm;
  41         /* list of available strms */
  42         struct list_head idle_strm;
  43         wait_queue_head_t strm_wait;
  44 };
  45
  46 static struct zcomp_backend *backends[] = {
  47         &zcomp_lzo,
  48 #ifdef CONFIG_ZRAM_LZ4_COMPRESS
  49         &zcomp_lz4,
  50 #endif
  51         NULL
  52 };
  53
  54 static struct zcomp_backend *find_backend(const char *compress)
  55 {
  56         int i = 0;
  57         while (backends[i]) {
  58                 if (sysfs_streq(compress, backends[i]->name))
  59                         break;
  60                 i++;
  61         }
  62         return backends[i];
  63 }
  64
  65 static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm)
  66 {
  67         if (zstrm->private)
  68                 comp->backend->destroy(zstrm->private);
  69         free_pages((unsigned long)zstrm->buffer, 1);
  70         kfree(zstrm);
  71 }
  72
  73 /*
  74  * allocate new zcomp_strm structure with ->private initialized by
  75  * backend, return NULL on error
  76  */
  77 static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp, gfp_t flags)
  78 {
  79         struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), flags);
  80         if (!zstrm)
  81                 return NULL;
  82
  83         zstrm->private = comp->backend->create(flags);
  84         /*
  85          * allocate 2 pages. 1 for compressed data, plus 1 extra for the
  86          * case when compressed size is larger than the original one
  87          */
  88         zstrm->buffer = (void *)__get_free_pages(flags | __GFP_ZERO, 1);
  89         if (!zstrm->private || !zstrm->buffer) {
  90                 zcomp_strm_free(comp, zstrm);
  91                 zstrm = NULL;
  92         }
  93         return zstrm;
  94 }
  95
  96 /*
  97  * get idle zcomp_strm or wait until other process release
  98  * (zcomp_strm_release()) one for us
  99  */
 100 static struct zcomp_strm *zcomp_strm_multi_find(struct zcomp *comp)
 101 {
 102         struct zcomp_strm_multi *zs = comp->stream;
 103         struct zcomp_strm *zstrm;
 104
 105         while (1) {
 106                 spin_lock(&zs->strm_lock);
 107                 if (!list_empty(&zs->idle_strm)) {
 108                         zstrm = list_entry(zs->idle_strm.next,
 109                                         struct zcomp_strm, list);
 110                         list_del(&zstrm->list);
 111                         spin_unlock(&zs->strm_lock);
 112                         return zstrm;
 113                 }
 114                 /* zstrm streams limit reached, wait for idle stream */
 115                 if (zs->avail_strm >= zs->max_strm) {
 116                         spin_unlock(&zs->strm_lock);
 117                         wait_event(zs->strm_wait, !list_empty(&zs->idle_strm));
 118                         continue;
 119                 }
 120                 /* allocate new zstrm stream */
 121                 zs->avail_strm++;
 122                 spin_unlock(&zs->strm_lock);
 123                 /*
 124                  * This function can be called in swapout/fs write path
 125                  * so we can't use GFP_FS|IO. And it assumes we already
 126                  * have at least one stream in zram initialization so we
 127                  * don't do best effort to allocate more stream in here.
 128                  * A default stream will work well without further multiple
 129                  * streams. That's why we use NORETRY | NOWARN.
 130                  */
 131                 zstrm = zcomp_strm_alloc(comp, GFP_NOIO | __GFP_NORETRY |
 132                                         __GFP_NOWARN);
 133                 if (!zstrm) {
 134                         spin_lock(&zs->strm_lock);
 135                         zs->avail_strm--;
 136                         spin_unlock(&zs->strm_lock);
 137                         wait_event(zs->strm_wait, !list_empty(&zs->idle_strm));
 138                         continue;
 139                 }
 140                 break;
 141         }
 142         return zstrm;
 143 }
 144
 145 /* add stream back to idle list and wake up waiter or free the stream */
 146 static void zcomp_strm_multi_release(struct zcomp *comp, struct zcomp_strm *zstrm)
 147 {
 148         struct zcomp_strm_multi *zs = comp->stream;
 149
 150         spin_lock(&zs->strm_lock);
 151         if (zs->avail_strm <= zs->max_strm) {
 152                 list_add(&zstrm->list, &zs->idle_strm);
 153                 spin_unlock(&zs->strm_lock);
 154                 wake_up(&zs->strm_wait);
 155                 return;
 156         }
 157
 158         zs->avail_strm--;
 159         spin_unlock(&zs->strm_lock);
 160         zcomp_strm_free(comp, zstrm);
 161 }
 162
 163 /* change max_strm limit */
 164 static bool zcomp_strm_multi_set_max_streams(struct zcomp *comp, int num_strm)
 165 {
 166         struct zcomp_strm_multi *zs = comp->stream;
 167         struct zcomp_strm *zstrm;
 168
 169         spin_lock(&zs->strm_lock);
 170         zs->max_strm = num_strm;
 171         /*
 172          * if user has lowered the limit and there are idle streams,
 173          * immediately free as much streams (and memory) as we can.
 174          */
 175         while (zs->avail_strm > num_strm && !list_empty(&zs->idle_strm)) {
 176                 zstrm = list_entry(zs->idle_strm.next,
 177                                 struct zcomp_strm, list);
 178                 list_del(&zstrm->list);
 179                 zcomp_strm_free(comp, zstrm);
 180                 zs->avail_strm--;
 181         }
 182         spin_unlock(&zs->strm_lock);
 183         return true;
 184 }
 185
 186 static void zcomp_strm_multi_destroy(struct zcomp *comp)
 187 {
 188         struct zcomp_strm_multi *zs = comp->stream;
 189         struct zcomp_strm *zstrm;
 190
 191         while (!list_empty(&zs->idle_strm)) {
 192                 zstrm = list_entry(zs->idle_strm.next,
 193                                 struct zcomp_strm, list);
 194                 list_del(&zstrm->list);
 195                 zcomp_strm_free(comp, zstrm);
 196         }
 197         kfree(zs);
 198 }
 199
 200 static int zcomp_strm_multi_create(struct zcomp *comp, int max_strm)
 201 {
 202         struct zcomp_strm *zstrm;
 203         struct zcomp_strm_multi *zs;
 204
 205         comp->destroy = zcomp_strm_multi_destroy;
 206         comp->strm_find = zcomp_strm_multi_find;
 207         comp->strm_release = zcomp_strm_multi_release;
 208         comp->set_max_streams = zcomp_strm_multi_set_max_streams;
 209         zs = kmalloc(sizeof(struct zcomp_strm_multi), GFP_KERNEL);
 210         if (!zs)
 211                 return -ENOMEM;
 212
 213         comp->stream = zs;
 214         spin_lock_init(&zs->strm_lock);
 215         INIT_LIST_HEAD(&zs->idle_strm);
 216         init_waitqueue_head(&zs->strm_wait);
 217         zs->max_strm = max_strm;
 218         zs->avail_strm = 1;
 219
 220         zstrm = zcomp_strm_alloc(comp, GFP_KERNEL);
 221         if (!zstrm) {
 222                 kfree(zs);
 223                 return -ENOMEM;
 224         }
 225         list_add(&zstrm->list, &zs->idle_strm);
 226         return 0;
 227 }
 228
 229 static struct zcomp_strm *zcomp_strm_single_find(struct zcomp *comp)
 230 {
 231         struct zcomp_strm_single *zs = comp->stream;
 232         mutex_lock(&zs->strm_lock);
 233         return zs->zstrm;
 234 }
 235
 236 static void zcomp_strm_single_release(struct zcomp *comp,
 237                 struct zcomp_strm *zstrm)
 238 {
 239         struct zcomp_strm_single *zs = comp->stream;
 240         mutex_unlock(&zs->strm_lock);
 241 }
 242
 243 static bool zcomp_strm_single_set_max_streams(struct zcomp *comp, int num_strm)
 244 {
 245         /* zcomp_strm_single support only max_comp_streams == 1 */
 246         return false;
 247 }
 248
 249 static void zcomp_strm_single_destroy(struct zcomp *comp)
 250 {
 251         struct zcomp_strm_single *zs = comp->stream;
 252         zcomp_strm_free(comp, zs->zstrm);
 253         kfree(zs);
 254 }
 255
 256 static int zcomp_strm_single_create(struct zcomp *comp)
 257 {
 258         struct zcomp_strm_single *zs;
 259
 260         comp->destroy = zcomp_strm_single_destroy;
 261         comp->strm_find = zcomp_strm_single_find;
 262         comp->strm_release = zcomp_strm_single_release;
 263         comp->set_max_streams = zcomp_strm_single_set_max_streams;
 264         zs = kmalloc(sizeof(struct zcomp_strm_single), GFP_KERNEL);
 265         if (!zs)
 266                 return -ENOMEM;
 267
 268         comp->stream = zs;
 269         mutex_init(&zs->strm_lock);
 270         zs->zstrm = zcomp_strm_alloc(comp, GFP_KERNEL);
 271         if (!zs->zstrm) {
 272                 kfree(zs);
 273                 return -ENOMEM;
 274         }
 275         return 0;
 276 }
 277
 278 /* show available compressors */
 279 ssize_t zcomp_available_show(const char *comp, char *buf)
 280 {
 281         ssize_t sz = 0;
 282         int i = 0;
 283
 284         while (backends[i]) {
 285                 if (!strcmp(comp, backends[i]->name))
 286                         sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
 287                                         "[%s] ", backends[i]->name);
 288                 else
 289                         sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
 290                                         "%s ", backends[i]->name);
 291                 i++;
 292         }
 293         sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n");
 294         return sz;
 295 }
 296
 297 bool zcomp_available_algorithm(const char *comp)
 298 {
 299         return find_backend(comp) != NULL;
 300 }
 301
 302 bool zcomp_set_max_streams(struct zcomp *comp, int num_strm)
 303 {
 304         return comp->set_max_streams(comp, num_strm);
 305 }
 306
 307 struct zcomp_strm *zcomp_strm_find(struct zcomp *comp)
 308 {
 309         return comp->strm_find(comp);
 310 }
 311
 312 void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm)
 313 {
 314         comp->strm_release(comp, zstrm);
 315 }
 316
 317 int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm,
 318                 const unsigned char *src, size_t *dst_len)
 319 {
 320         return comp->backend->compress(src, zstrm->buffer, dst_len,
 321                         zstrm->private);
 322 }
 323
 324 int zcomp_decompress(struct zcomp *comp, const unsigned char *src,
 325                 size_t src_len, unsigned char *dst)
 326 {
 327         return comp->backend->decompress(src, src_len, dst);
 328 }
 329
 330 void zcomp_destroy(struct zcomp *comp)
 331 {
 332         comp->destroy(comp);
 333         kfree(comp);
 334 }
 335
 336 /*
 337  * search available compressors for requested algorithm.
 338  * allocate new zcomp and initialize it. return compressing
 339  * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL)
 340  * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in
 341  * case of allocation error, or any other error potentially
 342  * returned by functions zcomp_strm_{multi,single}_create.
 343  */
 344 struct zcomp *zcomp_create(const char *compress, int max_strm)
 345 {
 346         struct zcomp *comp;
 347         struct zcomp_backend *backend;
 348         int error;
 349
 350         backend = find_backend(compress);
 351         if (!backend)
 352                 return ERR_PTR(-EINVAL);
 353
 354         comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL);
 355         if (!comp)
 356                 return ERR_PTR(-ENOMEM);
 357
 358         comp->backend = backend;
 359         if (max_strm > 1)
 360                 error = zcomp_strm_multi_create(comp, max_strm);
 361         else
 362                 error = zcomp_strm_single_create(comp);
 363         if (error) {
 364                 kfree(comp);
 365                 return ERR_PTR(error);
 366         }
 367         return comp;
 368 }