fs/erofs/decompressor_lzma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 #include <linux/xz.h>
   3 #include "compress.h"
   4
   5 struct z_erofs_lzma {
   6         struct z_erofs_lzma *next;
   7         struct xz_dec_microlzma *state;
   8         u8 bounce[PAGE_SIZE];
   9 };
  10
  11 /* considering the LZMA performance, no need to use a lockless list for now */
  12 static DEFINE_SPINLOCK(z_erofs_lzma_lock);
  13 static unsigned int z_erofs_lzma_max_dictsize;
  14 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
  15 static struct z_erofs_lzma *z_erofs_lzma_head;
  16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
  17
  18 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
  19
  20 static void z_erofs_lzma_exit(void)
  21 {
  22         /* there should be no running fs instance */
  23         while (z_erofs_lzma_avail_strms) {
  24                 struct z_erofs_lzma *strm;
  25
  26                 spin_lock(&z_erofs_lzma_lock);
  27                 strm = z_erofs_lzma_head;
  28                 if (!strm) {
  29                         spin_unlock(&z_erofs_lzma_lock);
  30                         DBG_BUGON(1);
  31                         return;
  32                 }
  33                 z_erofs_lzma_head = NULL;
  34                 spin_unlock(&z_erofs_lzma_lock);
  35
  36                 while (strm) {
  37                         struct z_erofs_lzma *n = strm->next;
  38
  39                         if (strm->state)
  40                                 xz_dec_microlzma_end(strm->state);
  41                         kfree(strm);
  42                         --z_erofs_lzma_avail_strms;
  43                         strm = n;
  44                 }
  45         }
  46 }
  47
  48 static int __init z_erofs_lzma_init(void)
  49 {
  50         unsigned int i;
  51
  52         /* by default, use # of possible CPUs instead */
  53         if (!z_erofs_lzma_nstrms)
  54                 z_erofs_lzma_nstrms = num_possible_cpus();
  55
  56         for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
  57                 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
  58
  59                 if (!strm) {
  60                         z_erofs_lzma_exit();
  61                         return -ENOMEM;
  62                 }
  63                 spin_lock(&z_erofs_lzma_lock);
  64                 strm->next = z_erofs_lzma_head;
  65                 z_erofs_lzma_head = strm;
  66                 spin_unlock(&z_erofs_lzma_lock);
  67                 ++z_erofs_lzma_avail_strms;
  68         }
  69         return 0;
  70 }
  71
  72 static int z_erofs_load_lzma_config(struct super_block *sb,
  73                         struct erofs_super_block *dsb, void *data, int size)
  74 {
  75         static DEFINE_MUTEX(lzma_resize_mutex);
  76         struct z_erofs_lzma_cfgs *lzma = data;
  77         unsigned int dict_size, i;
  78         struct z_erofs_lzma *strm, *head = NULL;
  79         int err;
  80
  81         if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
  82                 erofs_err(sb, "invalid lzma cfgs, size=%u", size);
  83                 return -EINVAL;
  84         }
  85         if (lzma->format) {
  86                 erofs_err(sb, "unidentified lzma format %x, please check kernel version",
  87                           le16_to_cpu(lzma->format));
  88                 return -EINVAL;
  89         }
  90         dict_size = le32_to_cpu(lzma->dict_size);
  91         if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
  92                 erofs_err(sb, "unsupported lzma dictionary size %u",
  93                           dict_size);
  94                 return -EINVAL;
  95         }
  96
  97         /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
  98         mutex_lock(&lzma_resize_mutex);
  99
 100         if (z_erofs_lzma_max_dictsize >= dict_size) {
 101                 mutex_unlock(&lzma_resize_mutex);
 102                 return 0;
 103         }
 104
 105         /* 1. collect/isolate all streams for the following check */
 106         for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
 107                 struct z_erofs_lzma *last;
 108
 109 again:
 110                 spin_lock(&z_erofs_lzma_lock);
 111                 strm = z_erofs_lzma_head;
 112                 if (!strm) {
 113                         spin_unlock(&z_erofs_lzma_lock);
 114                         wait_event(z_erofs_lzma_wq,
 115                                    READ_ONCE(z_erofs_lzma_head));
 116                         goto again;
 117                 }
 118                 z_erofs_lzma_head = NULL;
 119                 spin_unlock(&z_erofs_lzma_lock);
 120
 121                 for (last = strm; last->next; last = last->next)
 122                         ++i;
 123                 last->next = head;
 124                 head = strm;
 125         }
 126
 127         err = 0;
 128         /* 2. walk each isolated stream and grow max dict_size if needed */
 129         for (strm = head; strm; strm = strm->next) {
 130                 if (strm->state)
 131                         xz_dec_microlzma_end(strm->state);
 132                 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
 133                 if (!strm->state)
 134                         err = -ENOMEM;
 135         }
 136
 137         /* 3. push back all to the global list and update max dict_size */
 138         spin_lock(&z_erofs_lzma_lock);
 139         DBG_BUGON(z_erofs_lzma_head);
 140         z_erofs_lzma_head = head;
 141         spin_unlock(&z_erofs_lzma_lock);
 142         wake_up_all(&z_erofs_lzma_wq);
 143
 144         z_erofs_lzma_max_dictsize = dict_size;
 145         mutex_unlock(&lzma_resize_mutex);
 146         return err;
 147 }
 148
 149 static int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
 150                                    struct page **pgpl)
 151 {
 152         struct super_block *sb = rq->sb;
 153         struct z_erofs_stream_dctx dctx = {
 154                 .rq = rq,
 155                 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,
 156                 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)
 157                                 >> PAGE_SHIFT,
 158                 .no = -1, .ni = 0,
 159         };
 160         struct xz_buf buf = {};
 161         struct z_erofs_lzma *strm;
 162         enum xz_ret xz_err;
 163         int err;
 164
 165         /* 1. get the exact LZMA compressed size */
 166         dctx.kin = kmap_local_page(*rq->in);
 167         err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
 168                         min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
 169         if (err) {
 170                 kunmap_local(dctx.kin);
 171                 return err;
 172         }
 173
 174         /* 2. get an available lzma context */
 175 again:
 176         spin_lock(&z_erofs_lzma_lock);
 177         strm = z_erofs_lzma_head;
 178         if (!strm) {
 179                 spin_unlock(&z_erofs_lzma_lock);
 180                 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
 181                 goto again;
 182         }
 183         z_erofs_lzma_head = strm->next;
 184         spin_unlock(&z_erofs_lzma_lock);
 185
 186         /* 3. multi-call decompress */
 187         xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize,
 188                                !rq->partial_decoding);
 189         buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);
 190         rq->inputsize -= buf.in_size;
 191         buf.in = dctx.kin + rq->pageofs_in;
 192         dctx.bounce = strm->bounce;
 193         do {
 194                 dctx.avail_out = buf.out_size - buf.out_pos;
 195                 dctx.inbuf_sz = buf.in_size;
 196                 dctx.inbuf_pos = buf.in_pos;
 197                 err = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out,
 198                                                  (void **)&buf.in, pgpl);
 199                 if (err)
 200                         break;
 201
 202                 if (buf.out_size == buf.out_pos) {
 203                         buf.out_size = dctx.avail_out;
 204                         buf.out_pos = 0;
 205                 }
 206                 buf.in_size = dctx.inbuf_sz;
 207                 buf.in_pos = dctx.inbuf_pos;
 208
 209                 xz_err = xz_dec_microlzma_run(strm->state, &buf);
 210                 DBG_BUGON(buf.out_pos > buf.out_size);
 211                 DBG_BUGON(buf.in_pos > buf.in_size);
 212
 213                 if (xz_err != XZ_OK) {
 214                         if (xz_err == XZ_STREAM_END && !rq->outputsize)
 215                                 break;
 216                         erofs_err(sb, "failed to decompress %d in[%u] out[%u]",
 217                                   xz_err, rq->inputsize, rq->outputsize);
 218                         err = -EFSCORRUPTED;
 219                         break;
 220                 }
 221         } while (1);
 222
 223         if (dctx.kout)
 224                 kunmap_local(dctx.kout);
 225         kunmap_local(dctx.kin);
 226         /* 4. push back LZMA stream context to the global list */
 227         spin_lock(&z_erofs_lzma_lock);
 228         strm->next = z_erofs_lzma_head;
 229         z_erofs_lzma_head = strm;
 230         spin_unlock(&z_erofs_lzma_lock);
 231         wake_up(&z_erofs_lzma_wq);
 232         return err;
 233 }
 234
 235 const struct z_erofs_decompressor z_erofs_lzma_decomp = {
 236         .config = z_erofs_load_lzma_config,
 237         .decompress = z_erofs_lzma_decompress,
 238         .init = z_erofs_lzma_init,
 239         .exit = z_erofs_lzma_exit,
 240         .name = "lzma"
 241 };