drivers/block/zram/backend_zstd.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2
   3 #include <linux/kernel.h>
   4 #include <linux/slab.h>
   5 #include <linux/vmalloc.h>
   6 #include <linux/zstd.h>
   7
   8 #include "backend_zstd.h"
   9
  10 struct zstd_ctx {
  11         zstd_cctx *cctx;
  12         zstd_dctx *dctx;
  13         void *cctx_mem;
  14         void *dctx_mem;
  15 };
  16
  17 struct zstd_params {
  18         zstd_custom_mem custom_mem;
  19         zstd_cdict *cdict;
  20         zstd_ddict *ddict;
  21         zstd_parameters cprm;
  22 };
  23
  24 /*
  25  * For C/D dictionaries we need to provide zstd with zstd_custom_mem,
  26  * which zstd uses internally to allocate/free memory when needed.
  27  *
  28  * This means that allocator.customAlloc() can be called from zcomp_compress()
  29  * under local-lock (per-CPU compression stream), in which case we must use
  30  * GFP_ATOMIC.
  31  *
  32  * Another complication here is that we can be configured as a swap device.
  33  */
  34 static void *zstd_custom_alloc(void *opaque, size_t size)
  35 {
  36         if (!preemptible())
  37                 return kvzalloc(size, GFP_ATOMIC);
  38
  39         return kvzalloc(size, __GFP_KSWAPD_RECLAIM | __GFP_NOWARN);
  40 }
  41
  42 static void zstd_custom_free(void *opaque, void *address)
  43 {
  44         kvfree(address);
  45 }
  46
  47 static void zstd_release_params(struct zcomp_params *params)
  48 {
  49         struct zstd_params *zp = params->drv_data;
  50
  51         params->drv_data = NULL;
  52         if (!zp)
  53                 return;
  54
  55         zstd_free_cdict(zp->cdict);
  56         zstd_free_ddict(zp->ddict);
  57         kfree(zp);
  58 }
  59
  60 static int zstd_setup_params(struct zcomp_params *params)
  61 {
  62         zstd_compression_parameters prm;
  63         struct zstd_params *zp;
  64
  65         zp = kzalloc(sizeof(*zp), GFP_KERNEL);
  66         if (!zp)
  67                 return -ENOMEM;
  68
  69         params->drv_data = zp;
  70         if (params->level == ZCOMP_PARAM_NO_LEVEL)
  71                 params->level = zstd_default_clevel();
  72
  73         zp->cprm = zstd_get_params(params->level, PAGE_SIZE);
  74
  75         zp->custom_mem.customAlloc = zstd_custom_alloc;
  76         zp->custom_mem.customFree = zstd_custom_free;
  77
  78         prm = zstd_get_cparams(params->level, PAGE_SIZE,
  79                                params->dict_sz);
  80
  81         zp->cdict = zstd_create_cdict_byreference(params->dict,
  82                                                   params->dict_sz,
  83                                                   prm,
  84                                                   zp->custom_mem);
  85         if (!zp->cdict)
  86                 goto error;
  87
  88         zp->ddict = zstd_create_ddict_byreference(params->dict,
  89                                                   params->dict_sz,
  90                                                   zp->custom_mem);
  91         if (!zp->ddict)
  92                 goto error;
  93
  94         return 0;
  95
  96 error:
  97         zstd_release_params(params);
  98         return -EINVAL;
  99 }
 100
 101 static void zstd_destroy(struct zcomp_ctx *ctx)
 102 {
 103         struct zstd_ctx *zctx = ctx->context;
 104
 105         if (!zctx)
 106                 return;
 107
 108         /*
 109          * If ->cctx_mem and ->dctx_mem were allocated then we didn't use
 110          * C/D dictionary and ->cctx / ->dctx were "embedded" into these
 111          * buffers.
 112          *
 113          * If otherwise then we need to explicitly release ->cctx / ->dctx.
 114          */
 115         if (zctx->cctx_mem)
 116                 vfree(zctx->cctx_mem);
 117         else
 118                 zstd_free_cctx(zctx->cctx);
 119
 120         if (zctx->dctx_mem)
 121                 vfree(zctx->dctx_mem);
 122         else
 123                 zstd_free_dctx(zctx->dctx);
 124
 125         kfree(zctx);
 126 }
 127
 128 static int zstd_create(struct zcomp_params *params, struct zcomp_ctx *ctx)
 129 {
 130         struct zstd_ctx *zctx;
 131         zstd_parameters prm;
 132         size_t sz;
 133
 134         zctx = kzalloc(sizeof(*zctx), GFP_KERNEL);
 135         if (!zctx)
 136                 return -ENOMEM;
 137
 138         ctx->context = zctx;
 139         if (params->dict_sz == 0) {
 140                 prm = zstd_get_params(params->level, PAGE_SIZE);
 141                 sz = zstd_cctx_workspace_bound(&prm.cParams);
 142                 zctx->cctx_mem = vzalloc(sz);
 143                 if (!zctx->cctx_mem)
 144                         goto error;
 145
 146                 zctx->cctx = zstd_init_cctx(zctx->cctx_mem, sz);
 147                 if (!zctx->cctx)
 148                         goto error;
 149
 150                 sz = zstd_dctx_workspace_bound();
 151                 zctx->dctx_mem = vzalloc(sz);
 152                 if (!zctx->dctx_mem)
 153                         goto error;
 154
 155                 zctx->dctx = zstd_init_dctx(zctx->dctx_mem, sz);
 156                 if (!zctx->dctx)
 157                         goto error;
 158         } else {
 159                 struct zstd_params *zp = params->drv_data;
 160
 161                 zctx->cctx = zstd_create_cctx_advanced(zp->custom_mem);
 162                 if (!zctx->cctx)
 163                         goto error;
 164
 165                 zctx->dctx = zstd_create_dctx_advanced(zp->custom_mem);
 166                 if (!zctx->dctx)
 167                         goto error;
 168         }
 169
 170         return 0;
 171
 172 error:
 173         zstd_release_params(params);
 174         zstd_destroy(ctx);
 175         return -EINVAL;
 176 }
 177
 178 static int zstd_compress(struct zcomp_params *params, struct zcomp_ctx *ctx,
 179                          struct zcomp_req *req)
 180 {
 181         struct zstd_params *zp = params->drv_data;
 182         struct zstd_ctx *zctx = ctx->context;
 183         size_t ret;
 184
 185         if (params->dict_sz == 0)
 186                 ret = zstd_compress_cctx(zctx->cctx, req->dst, req->dst_len,
 187                                          req->src, req->src_len, &zp->cprm);
 188         else
 189                 ret = zstd_compress_using_cdict(zctx->cctx, req->dst,
 190                                                 req->dst_len, req->src,
 191                                                 req->src_len,
 192                                                 zp->cdict);
 193         if (zstd_is_error(ret))
 194                 return -EINVAL;
 195         req->dst_len = ret;
 196         return 0;
 197 }
 198
 199 static int zstd_decompress(struct zcomp_params *params, struct zcomp_ctx *ctx,
 200                            struct zcomp_req *req)
 201 {
 202         struct zstd_params *zp = params->drv_data;
 203         struct zstd_ctx *zctx = ctx->context;
 204         size_t ret;
 205
 206         if (params->dict_sz == 0)
 207                 ret = zstd_decompress_dctx(zctx->dctx, req->dst, req->dst_len,
 208                                            req->src, req->src_len);
 209         else
 210                 ret = zstd_decompress_using_ddict(zctx->dctx, req->dst,
 211                                                   req->dst_len, req->src,
 212                                                   req->src_len, zp->ddict);
 213         if (zstd_is_error(ret))
 214                 return -EINVAL;
 215         return 0;
 216 }
 217
 218 const struct zcomp_ops backend_zstd = {
 219         .compress       = zstd_compress,
 220         .decompress     = zstd_decompress,
 221         .create_ctx     = zstd_create,
 222         .destroy_ctx    = zstd_destroy,
 223         .setup_params   = zstd_setup_params,
 224         .release_params = zstd_release_params,
 225         .name           = "zstd",
 226 };