fs/erofs/decompressor_zstd.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 #include <linux/zstd.h>
   3 #include "compress.h"
   4
   5 struct z_erofs_zstd {
   6         struct z_erofs_zstd *next;
   7         u8 bounce[PAGE_SIZE];
   8         void *wksp;
   9         unsigned int wkspsz;
  10 };
  11
  12 static DEFINE_SPINLOCK(z_erofs_zstd_lock);
  13 static unsigned int z_erofs_zstd_max_dictsize;
  14 static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms;
  15 static struct z_erofs_zstd *z_erofs_zstd_head;
  16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq);
  17
  18 module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444);
  19
  20 static struct z_erofs_zstd *z_erofs_isolate_strms(bool all)
  21 {
  22         struct z_erofs_zstd *strm;
  23
  24 again:
  25         spin_lock(&z_erofs_zstd_lock);
  26         strm = z_erofs_zstd_head;
  27         if (!strm) {
  28                 spin_unlock(&z_erofs_zstd_lock);
  29                 wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head));
  30                 goto again;
  31         }
  32         z_erofs_zstd_head = all ? NULL : strm->next;
  33         spin_unlock(&z_erofs_zstd_lock);
  34         return strm;
  35 }
  36
  37 static void z_erofs_zstd_exit(void)
  38 {
  39         while (z_erofs_zstd_avail_strms) {
  40                 struct z_erofs_zstd *strm, *n;
  41
  42                 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
  43                         n = strm->next;
  44
  45                         kvfree(strm->wksp);
  46                         kfree(strm);
  47                         --z_erofs_zstd_avail_strms;
  48                 }
  49         }
  50 }
  51
  52 static int __init z_erofs_zstd_init(void)
  53 {
  54         /* by default, use # of possible CPUs instead */
  55         if (!z_erofs_zstd_nstrms)
  56                 z_erofs_zstd_nstrms = num_possible_cpus();
  57
  58         for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms;
  59              ++z_erofs_zstd_avail_strms) {
  60                 struct z_erofs_zstd *strm;
  61
  62                 strm = kzalloc(sizeof(*strm), GFP_KERNEL);
  63                 if (!strm) {
  64                         z_erofs_zstd_exit();
  65                         return -ENOMEM;
  66                 }
  67                 spin_lock(&z_erofs_zstd_lock);
  68                 strm->next = z_erofs_zstd_head;
  69                 z_erofs_zstd_head = strm;
  70                 spin_unlock(&z_erofs_zstd_lock);
  71         }
  72         return 0;
  73 }
  74
  75 static int z_erofs_load_zstd_config(struct super_block *sb,
  76                         struct erofs_super_block *dsb, void *data, int size)
  77 {
  78         static DEFINE_MUTEX(zstd_resize_mutex);
  79         struct z_erofs_zstd_cfgs *zstd = data;
  80         unsigned int dict_size, wkspsz;
  81         struct z_erofs_zstd *strm, *head = NULL;
  82         void *wksp;
  83
  84         if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) {
  85                 erofs_err(sb, "unsupported zstd format, size=%u", size);
  86                 return -EINVAL;
  87         }
  88
  89         if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) {
  90                 erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog);
  91                 return -EINVAL;
  92         }
  93         dict_size = 1U << (zstd->windowlog + 10);
  94
  95         /* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */
  96         mutex_lock(&zstd_resize_mutex);
  97         if (z_erofs_zstd_max_dictsize >= dict_size) {
  98                 mutex_unlock(&zstd_resize_mutex);
  99                 return 0;
 100         }
 101
 102         /* 1. collect/isolate all streams for the following check */
 103         while (z_erofs_zstd_avail_strms) {
 104                 struct z_erofs_zstd *n;
 105
 106                 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
 107                         n = strm->next;
 108                         strm->next = head;
 109                         head = strm;
 110                         --z_erofs_zstd_avail_strms;
 111                 }
 112         }
 113
 114         /* 2. walk each isolated stream and grow max dict_size if needed */
 115         wkspsz = zstd_dstream_workspace_bound(dict_size);
 116         for (strm = head; strm; strm = strm->next) {
 117                 wksp = kvmalloc(wkspsz, GFP_KERNEL);
 118                 if (!wksp)
 119                         break;
 120                 kvfree(strm->wksp);
 121                 strm->wksp = wksp;
 122                 strm->wkspsz = wkspsz;
 123         }
 124
 125         /* 3. push back all to the global list and update max dict_size */
 126         spin_lock(&z_erofs_zstd_lock);
 127         DBG_BUGON(z_erofs_zstd_head);
 128         z_erofs_zstd_head = head;
 129         spin_unlock(&z_erofs_zstd_lock);
 130         z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms;
 131         wake_up_all(&z_erofs_zstd_wq);
 132         if (!strm)
 133                 z_erofs_zstd_max_dictsize = dict_size;
 134         mutex_unlock(&zstd_resize_mutex);
 135         return strm ? -ENOMEM : 0;
 136 }
 137
 138 static int z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq,
 139                                    struct page **pgpl)
 140 {
 141         struct super_block *sb = rq->sb;
 142         struct z_erofs_stream_dctx dctx = {
 143                 .rq = rq,
 144                 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,
 145                 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)
 146                                 >> PAGE_SHIFT,
 147                 .no = -1, .ni = 0,
 148         };
 149         zstd_in_buffer in_buf = { NULL, 0, 0 };
 150         zstd_out_buffer out_buf = { NULL, 0, 0 };
 151         struct z_erofs_zstd *strm;
 152         zstd_dstream *stream;
 153         int zerr, err;
 154
 155         /* 1. get the exact compressed size */
 156         dctx.kin = kmap_local_page(*rq->in);
 157         err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
 158                         min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
 159         if (err) {
 160                 kunmap_local(dctx.kin);
 161                 return err;
 162         }
 163
 164         /* 2. get an available ZSTD context */
 165         strm = z_erofs_isolate_strms(false);
 166
 167         /* 3. multi-call decompress */
 168         stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz);
 169         if (!stream) {
 170                 err = -EIO;
 171                 goto failed_zinit;
 172         }
 173
 174         rq->fillgaps = true;    /* ZSTD doesn't support NULL output buffer */
 175         in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in);
 176         rq->inputsize -= in_buf.size;
 177         in_buf.src = dctx.kin + rq->pageofs_in;
 178         dctx.bounce = strm->bounce;
 179
 180         do {
 181                 dctx.avail_out = out_buf.size - out_buf.pos;
 182                 dctx.inbuf_sz = in_buf.size;
 183                 dctx.inbuf_pos = in_buf.pos;
 184                 err = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst,
 185                                                  (void **)&in_buf.src, pgpl);
 186                 if (err)
 187                         break;
 188
 189                 if (out_buf.size == out_buf.pos) {
 190                         out_buf.size = dctx.avail_out;
 191                         out_buf.pos = 0;
 192                 }
 193                 in_buf.size = dctx.inbuf_sz;
 194                 in_buf.pos = dctx.inbuf_pos;
 195
 196                 zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);
 197                 if (zstd_is_error(zerr) || (!zerr && rq->outputsize)) {
 198                         erofs_err(sb, "failed to decompress in[%u] out[%u]: %s",
 199                                   rq->inputsize, rq->outputsize,
 200                                   zerr ? zstd_get_error_name(zerr) : "unexpected end of stream");
 201                         err = -EFSCORRUPTED;
 202                         break;
 203                 }
 204         } while (rq->outputsize || out_buf.pos < out_buf.size);
 205
 206         if (dctx.kout)
 207                 kunmap_local(dctx.kout);
 208 failed_zinit:
 209         kunmap_local(dctx.kin);
 210         /* 4. push back ZSTD stream context to the global list */
 211         spin_lock(&z_erofs_zstd_lock);
 212         strm->next = z_erofs_zstd_head;
 213         z_erofs_zstd_head = strm;
 214         spin_unlock(&z_erofs_zstd_lock);
 215         wake_up(&z_erofs_zstd_wq);
 216         return err;
 217 }
 218
 219 const struct z_erofs_decompressor z_erofs_zstd_decomp = {
 220         .config = z_erofs_load_zstd_config,
 221         .decompress = z_erofs_zstd_decompress,
 222         .init = z_erofs_zstd_init,
 223         .exit = z_erofs_zstd_exit,
 224         .name = "zstd",
 225 };