?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Revert "tty: hvc: Fix data abort due to race in hvc_open"
[linux/fpc-iii.git] / fs / f2fs / compress.c
bloba5b2e72174bb1f6add00e6f684e3855e95561efa
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * f2fs compress support
4 *
5 * Copyright (c) 2019 Chao Yu <chao@kernel.org>
6 */
7
8 #include <linux/fs.h>
9 #include <linux/f2fs_fs.h>
10 #include <linux/writeback.h>
11 #include <linux/backing-dev.h>
12 #include <linux/lzo.h>
13 #include <linux/lz4.h>
14 #include <linux/zstd.h>
15
16 #include "f2fs.h"
17 #include "node.h"
18 #include <trace/events/f2fs.h>
19
20 struct f2fs_compress_ops {
21 int (*init_compress_ctx)(struct compress_ctx *cc);
22 void (*destroy_compress_ctx)(struct compress_ctx *cc);
23 int (*compress_pages)(struct compress_ctx *cc);
24 int (*init_decompress_ctx)(struct decompress_io_ctx *dic);
25 void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic);
26 int (*decompress_pages)(struct decompress_io_ctx *dic);
27 };
28
29 static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index)
30 {
31 return index & (cc->cluster_size - 1);
32 }
33
34 static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index)
35 {
36 return index >> cc->log_cluster_size;
37 }
38
39 static pgoff_t start_idx_of_cluster(struct compress_ctx *cc)
40 {
41 return cc->cluster_idx << cc->log_cluster_size;
42 }
43
44 bool f2fs_is_compressed_page(struct page *page)
45 {
46 if (!PagePrivate(page))
47 return false;
48 if (!page_private(page))
49 return false;
50 if (IS_ATOMIC_WRITTEN_PAGE(page) || IS_DUMMY_WRITTEN_PAGE(page))
51 return false;
52 f2fs_bug_on(F2FS_M_SB(page->mapping),
53 *((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC);
54 return true;
55 }
56
57 static void f2fs_set_compressed_page(struct page *page,
58 struct inode *inode, pgoff_t index, void *data)
59 {
60 SetPagePrivate(page);
61 set_page_private(page, (unsigned long)data);
62
63 /* i_crypto_info and iv index */
64 page->index = index;
65 page->mapping = inode->i_mapping;
66 }
67
68 static void f2fs_put_compressed_page(struct page *page)
69 {
70 set_page_private(page, (unsigned long)NULL);
71 ClearPagePrivate(page);
72 page->mapping = NULL;
73 unlock_page(page);
74 put_page(page);
75 }
76
77 static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock)
78 {
79 int i;
80
81 for (i = 0; i < len; i++) {
82 if (!cc->rpages[i])
83 continue;
84 if (unlock)
85 unlock_page(cc->rpages[i]);
86 else
87 put_page(cc->rpages[i]);
88 }
89 }
90
91 static void f2fs_put_rpages(struct compress_ctx *cc)
92 {
93 f2fs_drop_rpages(cc, cc->cluster_size, false);
94 }
95
96 static void f2fs_unlock_rpages(struct compress_ctx *cc, int len)
97 {
98 f2fs_drop_rpages(cc, len, true);
99 }
100
101 static void f2fs_put_rpages_mapping(struct compress_ctx *cc,
102 struct address_space *mapping,
103 pgoff_t start, int len)
104 {
105 int i;
106
107 for (i = 0; i < len; i++) {
108 struct page *page = find_get_page(mapping, start + i);
109
110 put_page(page);
111 put_page(page);
112 }
113 }
114
115 static void f2fs_put_rpages_wbc(struct compress_ctx *cc,
116 struct writeback_control *wbc, bool redirty, int unlock)
117 {
118 unsigned int i;
119
120 for (i = 0; i < cc->cluster_size; i++) {
121 if (!cc->rpages[i])
122 continue;
123 if (redirty)
124 redirty_page_for_writepage(wbc, cc->rpages[i]);
125 f2fs_put_page(cc->rpages[i], unlock);
126 }
127 }
128
129 struct page *f2fs_compress_control_page(struct page *page)
130 {
131 return ((struct compress_io_ctx *)page_private(page))->rpages[0];
132 }
133
134 int f2fs_init_compress_ctx(struct compress_ctx *cc)
135 {
136 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
137
138 if (cc->nr_rpages)
139 return 0;
140
141 cc->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
142 cc->log_cluster_size, GFP_NOFS);
143 return cc->rpages ? 0 : -ENOMEM;
144 }
145
146 void f2fs_destroy_compress_ctx(struct compress_ctx *cc)
147 {
148 kfree(cc->rpages);
149 cc->rpages = NULL;
150 cc->nr_rpages = 0;
151 cc->nr_cpages = 0;
152 cc->cluster_idx = NULL_CLUSTER;
153 }
154
155 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page)
156 {
157 unsigned int cluster_ofs;
158
159 if (!f2fs_cluster_can_merge_page(cc, page->index))
160 f2fs_bug_on(F2FS_I_SB(cc->inode), 1);
161
162 cluster_ofs = offset_in_cluster(cc, page->index);
163 cc->rpages[cluster_ofs] = page;
164 cc->nr_rpages++;
165 cc->cluster_idx = cluster_idx(cc, page->index);
166 }
167
168 #ifdef CONFIG_F2FS_FS_LZO
169 static int lzo_init_compress_ctx(struct compress_ctx *cc)
170 {
171 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
172 LZO1X_MEM_COMPRESS, GFP_NOFS);
173 if (!cc->private)
174 return -ENOMEM;
175
176 cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size);
177 return 0;
178 }
179
180 static void lzo_destroy_compress_ctx(struct compress_ctx *cc)
181 {
182 kvfree(cc->private);
183 cc->private = NULL;
184 }
185
186 static int lzo_compress_pages(struct compress_ctx *cc)
187 {
188 int ret;
189
190 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
191 &cc->clen, cc->private);
192 if (ret != LZO_E_OK) {
193 printk_ratelimited("%sF2FS-fs (%s): lzo compress failed, ret:%d\n",
194 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret);
195 return -EIO;
196 }
197 return 0;
198 }
199
200 static int lzo_decompress_pages(struct decompress_io_ctx *dic)
201 {
202 int ret;
203
204 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen,
205 dic->rbuf, &dic->rlen);
206 if (ret != LZO_E_OK) {
207 printk_ratelimited("%sF2FS-fs (%s): lzo decompress failed, ret:%d\n",
208 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
209 return -EIO;
210 }
211
212 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) {
213 printk_ratelimited("%sF2FS-fs (%s): lzo invalid rlen:%zu, "
214 "expected:%lu\n", KERN_ERR,
215 F2FS_I_SB(dic->inode)->sb->s_id,
216 dic->rlen,
217 PAGE_SIZE << dic->log_cluster_size);
218 return -EIO;
219 }
220 return 0;
221 }
222
223 static const struct f2fs_compress_ops f2fs_lzo_ops = {
224 .init_compress_ctx = lzo_init_compress_ctx,
225 .destroy_compress_ctx = lzo_destroy_compress_ctx,
226 .compress_pages = lzo_compress_pages,
227 .decompress_pages = lzo_decompress_pages,
228 };
229 #endif
230
231 #ifdef CONFIG_F2FS_FS_LZ4
232 static int lz4_init_compress_ctx(struct compress_ctx *cc)
233 {
234 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
235 LZ4_MEM_COMPRESS, GFP_NOFS);
236 if (!cc->private)
237 return -ENOMEM;
238
239 /*
240 * we do not change cc->clen to LZ4_compressBound(inputsize) to
241 * adapt worst compress case, because lz4 compressor can handle
242 * output budget properly.
243 */
244 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
245 return 0;
246 }
247
248 static void lz4_destroy_compress_ctx(struct compress_ctx *cc)
249 {
250 kvfree(cc->private);
251 cc->private = NULL;
252 }
253
254 static int lz4_compress_pages(struct compress_ctx *cc)
255 {
256 int len;
257
258 len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen,
259 cc->clen, cc->private);
260 if (!len)
261 return -EAGAIN;
262
263 cc->clen = len;
264 return 0;
265 }
266
267 static int lz4_decompress_pages(struct decompress_io_ctx *dic)
268 {
269 int ret;
270
271 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf,
272 dic->clen, dic->rlen);
273 if (ret < 0) {
274 printk_ratelimited("%sF2FS-fs (%s): lz4 decompress failed, ret:%d\n",
275 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
276 return -EIO;
277 }
278
279 if (ret != PAGE_SIZE << dic->log_cluster_size) {
280 printk_ratelimited("%sF2FS-fs (%s): lz4 invalid rlen:%zu, "
281 "expected:%lu\n", KERN_ERR,
282 F2FS_I_SB(dic->inode)->sb->s_id,
283 dic->rlen,
284 PAGE_SIZE << dic->log_cluster_size);
285 return -EIO;
286 }
287 return 0;
288 }
289
290 static const struct f2fs_compress_ops f2fs_lz4_ops = {
291 .init_compress_ctx = lz4_init_compress_ctx,
292 .destroy_compress_ctx = lz4_destroy_compress_ctx,
293 .compress_pages = lz4_compress_pages,
294 .decompress_pages = lz4_decompress_pages,
295 };
296 #endif
297
298 #ifdef CONFIG_F2FS_FS_ZSTD
299 #define F2FS_ZSTD_DEFAULT_CLEVEL 1
300
301 static int zstd_init_compress_ctx(struct compress_ctx *cc)
302 {
303 ZSTD_parameters params;
304 ZSTD_CStream *stream;
305 void *workspace;
306 unsigned int workspace_size;
307
308 params = ZSTD_getParams(F2FS_ZSTD_DEFAULT_CLEVEL, cc->rlen, 0);
309 workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams);
310
311 workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
312 workspace_size, GFP_NOFS);
313 if (!workspace)
314 return -ENOMEM;
315
316 stream = ZSTD_initCStream(params, 0, workspace, workspace_size);
317 if (!stream) {
318 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initCStream failed\n",
319 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
320 __func__);
321 kvfree(workspace);
322 return -EIO;
323 }
324
325 cc->private = workspace;
326 cc->private2 = stream;
327
328 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
329 return 0;
330 }
331
332 static void zstd_destroy_compress_ctx(struct compress_ctx *cc)
333 {
334 kvfree(cc->private);
335 cc->private = NULL;
336 cc->private2 = NULL;
337 }
338
339 static int zstd_compress_pages(struct compress_ctx *cc)
340 {
341 ZSTD_CStream *stream = cc->private2;
342 ZSTD_inBuffer inbuf;
343 ZSTD_outBuffer outbuf;
344 int src_size = cc->rlen;
345 int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE;
346 int ret;
347
348 inbuf.pos = 0;
349 inbuf.src = cc->rbuf;
350 inbuf.size = src_size;
351
352 outbuf.pos = 0;
353 outbuf.dst = cc->cbuf->cdata;
354 outbuf.size = dst_size;
355
356 ret = ZSTD_compressStream(stream, &outbuf, &inbuf);
357 if (ZSTD_isError(ret)) {
358 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n",
359 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
360 __func__, ZSTD_getErrorCode(ret));
361 return -EIO;
362 }
363
364 ret = ZSTD_endStream(stream, &outbuf);
365 if (ZSTD_isError(ret)) {
366 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_endStream returned %d\n",
367 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
368 __func__, ZSTD_getErrorCode(ret));
369 return -EIO;
370 }
371
372 /*
373 * there is compressed data remained in intermediate buffer due to
374 * no more space in cbuf.cdata
375 */
376 if (ret)
377 return -EAGAIN;
378
379 cc->clen = outbuf.pos;
380 return 0;
381 }
382
383 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic)
384 {
385 ZSTD_DStream *stream;
386 void *workspace;
387 unsigned int workspace_size;
388
389 workspace_size = ZSTD_DStreamWorkspaceBound(MAX_COMPRESS_WINDOW_SIZE);
390
391 workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode),
392 workspace_size, GFP_NOFS);
393 if (!workspace)
394 return -ENOMEM;
395
396 stream = ZSTD_initDStream(MAX_COMPRESS_WINDOW_SIZE,
397 workspace, workspace_size);
398 if (!stream) {
399 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initDStream failed\n",
400 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,
401 __func__);
402 kvfree(workspace);
403 return -EIO;
404 }
405
406 dic->private = workspace;
407 dic->private2 = stream;
408
409 return 0;
410 }
411
412 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic)
413 {
414 kvfree(dic->private);
415 dic->private = NULL;
416 dic->private2 = NULL;
417 }
418
419 static int zstd_decompress_pages(struct decompress_io_ctx *dic)
420 {
421 ZSTD_DStream *stream = dic->private2;
422 ZSTD_inBuffer inbuf;
423 ZSTD_outBuffer outbuf;
424 int ret;
425
426 inbuf.pos = 0;
427 inbuf.src = dic->cbuf->cdata;
428 inbuf.size = dic->clen;
429
430 outbuf.pos = 0;
431 outbuf.dst = dic->rbuf;
432 outbuf.size = dic->rlen;
433
434 ret = ZSTD_decompressStream(stream, &outbuf, &inbuf);
435 if (ZSTD_isError(ret)) {
436 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n",
437 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,
438 __func__, ZSTD_getErrorCode(ret));
439 return -EIO;
440 }
441
442 if (dic->rlen != outbuf.pos) {
443 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD invalid rlen:%zu, "
444 "expected:%lu\n", KERN_ERR,
445 F2FS_I_SB(dic->inode)->sb->s_id,
446 __func__, dic->rlen,
447 PAGE_SIZE << dic->log_cluster_size);
448 return -EIO;
449 }
450
451 return 0;
452 }
453
454 static const struct f2fs_compress_ops f2fs_zstd_ops = {
455 .init_compress_ctx = zstd_init_compress_ctx,
456 .destroy_compress_ctx = zstd_destroy_compress_ctx,
457 .compress_pages = zstd_compress_pages,
458 .init_decompress_ctx = zstd_init_decompress_ctx,
459 .destroy_decompress_ctx = zstd_destroy_decompress_ctx,
460 .decompress_pages = zstd_decompress_pages,
461 };
462 #endif
463
464 static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = {
465 #ifdef CONFIG_F2FS_FS_LZO
466 &f2fs_lzo_ops,
467 #else
468 NULL,
469 #endif
470 #ifdef CONFIG_F2FS_FS_LZ4
471 &f2fs_lz4_ops,
472 #else
473 NULL,
474 #endif
475 #ifdef CONFIG_F2FS_FS_ZSTD
476 &f2fs_zstd_ops,
477 #else
478 NULL,
479 #endif
480 };
481
482 bool f2fs_is_compress_backend_ready(struct inode *inode)
483 {
484 if (!f2fs_compressed_file(inode))
485 return true;
486 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm];
487 }
488
489 static struct page *f2fs_grab_page(void)
490 {
491 struct page *page;
492
493 page = alloc_page(GFP_NOFS);
494 if (!page)
495 return NULL;
496 lock_page(page);
497 return page;
498 }
499
500 static int f2fs_compress_pages(struct compress_ctx *cc)
501 {
502 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
503 struct f2fs_inode_info *fi = F2FS_I(cc->inode);
504 const struct f2fs_compress_ops *cops =
505 f2fs_cops[fi->i_compress_algorithm];
506 unsigned int max_len, nr_cpages;
507 int i, ret;
508
509 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx,
510 cc->cluster_size, fi->i_compress_algorithm);
511
512 if (cops->init_compress_ctx) {
513 ret = cops->init_compress_ctx(cc);
514 if (ret)
515 goto out;
516 }
517
518 max_len = COMPRESS_HEADER_SIZE + cc->clen;
519 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE);
520
521 cc->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
522 cc->nr_cpages, GFP_NOFS);
523 if (!cc->cpages) {
524 ret = -ENOMEM;
525 goto destroy_compress_ctx;
526 }
527
528 for (i = 0; i < cc->nr_cpages; i++) {
529 cc->cpages[i] = f2fs_grab_page();
530 if (!cc->cpages[i]) {
531 ret = -ENOMEM;
532 goto out_free_cpages;
533 }
534 }
535
536 cc->rbuf = vmap(cc->rpages, cc->cluster_size, VM_MAP, PAGE_KERNEL_RO);
537 if (!cc->rbuf) {
538 ret = -ENOMEM;
539 goto out_free_cpages;
540 }
541
542 cc->cbuf = vmap(cc->cpages, cc->nr_cpages, VM_MAP, PAGE_KERNEL);
543 if (!cc->cbuf) {
544 ret = -ENOMEM;
545 goto out_vunmap_rbuf;
546 }
547
548 ret = cops->compress_pages(cc);
549 if (ret)
550 goto out_vunmap_cbuf;
551
552 max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE;
553
554 if (cc->clen > max_len) {
555 ret = -EAGAIN;
556 goto out_vunmap_cbuf;
557 }
558
559 cc->cbuf->clen = cpu_to_le32(cc->clen);
560
561 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++)
562 cc->cbuf->reserved[i] = cpu_to_le32(0);
563
564 nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE);
565
566 /* zero out any unused part of the last page */
567 memset(&cc->cbuf->cdata[cc->clen], 0,
568 (nr_cpages * PAGE_SIZE) - (cc->clen + COMPRESS_HEADER_SIZE));
569
570 vunmap(cc->cbuf);
571 vunmap(cc->rbuf);
572
573 for (i = nr_cpages; i < cc->nr_cpages; i++) {
574 f2fs_put_compressed_page(cc->cpages[i]);
575 cc->cpages[i] = NULL;
576 }
577
578 if (cops->destroy_compress_ctx)
579 cops->destroy_compress_ctx(cc);
580
581 cc->nr_cpages = nr_cpages;
582
583 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
584 cc->clen, ret);
585 return 0;
586
587 out_vunmap_cbuf:
588 vunmap(cc->cbuf);
589 out_vunmap_rbuf:
590 vunmap(cc->rbuf);
591 out_free_cpages:
592 for (i = 0; i < cc->nr_cpages; i++) {
593 if (cc->cpages[i])
594 f2fs_put_compressed_page(cc->cpages[i]);
595 }
596 kfree(cc->cpages);
597 cc->cpages = NULL;
598 destroy_compress_ctx:
599 if (cops->destroy_compress_ctx)
600 cops->destroy_compress_ctx(cc);
601 out:
602 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
603 cc->clen, ret);
604 return ret;
605 }
606
607 void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity)
608 {
609 struct decompress_io_ctx *dic =
610 (struct decompress_io_ctx *)page_private(page);
611 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);
612 struct f2fs_inode_info *fi= F2FS_I(dic->inode);
613 const struct f2fs_compress_ops *cops =
614 f2fs_cops[fi->i_compress_algorithm];
615 int ret;
616
617 dec_page_count(sbi, F2FS_RD_DATA);
618
619 if (bio->bi_status || PageError(page))
620 dic->failed = true;
621
622 if (refcount_dec_not_one(&dic->ref))
623 return;
624
625 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx,
626 dic->cluster_size, fi->i_compress_algorithm);
627
628 /* submit partial compressed pages */
629 if (dic->failed) {
630 ret = -EIO;
631 goto out_free_dic;
632 }
633
634 if (cops->init_decompress_ctx) {
635 ret = cops->init_decompress_ctx(dic);
636 if (ret)
637 goto out_free_dic;
638 }
639
640 dic->rbuf = vmap(dic->tpages, dic->cluster_size, VM_MAP, PAGE_KERNEL);
641 if (!dic->rbuf) {
642 ret = -ENOMEM;
643 goto destroy_decompress_ctx;
644 }
645
646 dic->cbuf = vmap(dic->cpages, dic->nr_cpages, VM_MAP, PAGE_KERNEL_RO);
647 if (!dic->cbuf) {
648 ret = -ENOMEM;
649 goto out_vunmap_rbuf;
650 }
651
652 dic->clen = le32_to_cpu(dic->cbuf->clen);
653 dic->rlen = PAGE_SIZE << dic->log_cluster_size;
654
655 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {
656 ret = -EFSCORRUPTED;
657 goto out_vunmap_cbuf;
658 }
659
660 ret = cops->decompress_pages(dic);
661
662 out_vunmap_cbuf:
663 vunmap(dic->cbuf);
664 out_vunmap_rbuf:
665 vunmap(dic->rbuf);
666 destroy_decompress_ctx:
667 if (cops->destroy_decompress_ctx)
668 cops->destroy_decompress_ctx(dic);
669 out_free_dic:
670 if (verity)
671 refcount_set(&dic->ref, dic->nr_cpages);
672 if (!verity)
673 f2fs_decompress_end_io(dic->rpages, dic->cluster_size,
674 ret, false);
675
676 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx,
677 dic->clen, ret);
678 if (!verity)
679 f2fs_free_dic(dic);
680 }
681
682 static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index)
683 {
684 if (cc->cluster_idx == NULL_CLUSTER)
685 return true;
686 return cc->cluster_idx == cluster_idx(cc, index);
687 }
688
689 bool f2fs_cluster_is_empty(struct compress_ctx *cc)
690 {
691 return cc->nr_rpages == 0;
692 }
693
694 static bool f2fs_cluster_is_full(struct compress_ctx *cc)
695 {
696 return cc->cluster_size == cc->nr_rpages;
697 }
698
699 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index)
700 {
701 if (f2fs_cluster_is_empty(cc))
702 return true;
703 return is_page_in_cluster(cc, index);
704 }
705
706 static bool __cluster_may_compress(struct compress_ctx *cc)
707 {
708 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
709 loff_t i_size = i_size_read(cc->inode);
710 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE);
711 int i;
712
713 for (i = 0; i < cc->cluster_size; i++) {
714 struct page *page = cc->rpages[i];
715
716 f2fs_bug_on(sbi, !page);
717
718 if (unlikely(f2fs_cp_error(sbi)))
719 return false;
720 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
721 return false;
722
723 /* beyond EOF */
724 if (page->index >= nr_pages)
725 return false;
726 }
727 return true;
728 }
729
730 static int __f2fs_cluster_blocks(struct compress_ctx *cc, bool compr)
731 {
732 struct dnode_of_data dn;
733 int ret;
734
735 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
736 ret = f2fs_get_dnode_of_data(&dn, start_idx_of_cluster(cc),
737 LOOKUP_NODE);
738 if (ret) {
739 if (ret == -ENOENT)
740 ret = 0;
741 goto fail;
742 }
743
744 if (dn.data_blkaddr == COMPRESS_ADDR) {
745 int i;
746
747 ret = 1;
748 for (i = 1; i < cc->cluster_size; i++) {
749 block_t blkaddr;
750
751 blkaddr = data_blkaddr(dn.inode,
752 dn.node_page, dn.ofs_in_node + i);
753 if (compr) {
754 if (__is_valid_data_blkaddr(blkaddr))
755 ret++;
756 } else {
757 if (blkaddr != NULL_ADDR)
758 ret++;
759 }
760 }
761 }
762 fail:
763 f2fs_put_dnode(&dn);
764 return ret;
765 }
766
767 /* return # of compressed blocks in compressed cluster */
768 static int f2fs_compressed_blocks(struct compress_ctx *cc)
769 {
770 return __f2fs_cluster_blocks(cc, true);
771 }
772
773 /* return # of valid blocks in compressed cluster */
774 static int f2fs_cluster_blocks(struct compress_ctx *cc, bool compr)
775 {
776 return __f2fs_cluster_blocks(cc, false);
777 }
778
779 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index)
780 {
781 struct compress_ctx cc = {
782 .inode = inode,
783 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
784 .cluster_size = F2FS_I(inode)->i_cluster_size,
785 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
786 };
787
788 return f2fs_cluster_blocks(&cc, false);
789 }
790
791 static bool cluster_may_compress(struct compress_ctx *cc)
792 {
793 if (!f2fs_compressed_file(cc->inode))
794 return false;
795 if (f2fs_is_atomic_file(cc->inode))
796 return false;
797 if (f2fs_is_mmap_file(cc->inode))
798 return false;
799 if (!f2fs_cluster_is_full(cc))
800 return false;
801 return __cluster_may_compress(cc);
802 }
803
804 static void set_cluster_writeback(struct compress_ctx *cc)
805 {
806 int i;
807
808 for (i = 0; i < cc->cluster_size; i++) {
809 if (cc->rpages[i])
810 set_page_writeback(cc->rpages[i]);
811 }
812 }
813
814 static void set_cluster_dirty(struct compress_ctx *cc)
815 {
816 int i;
817
818 for (i = 0; i < cc->cluster_size; i++)
819 if (cc->rpages[i])
820 set_page_dirty(cc->rpages[i]);
821 }
822
823 static int prepare_compress_overwrite(struct compress_ctx *cc,
824 struct page **pagep, pgoff_t index, void **fsdata)
825 {
826 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
827 struct address_space *mapping = cc->inode->i_mapping;
828 struct page *page;
829 struct dnode_of_data dn;
830 sector_t last_block_in_bio;
831 unsigned fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT;
832 pgoff_t start_idx = start_idx_of_cluster(cc);
833 int i, ret;
834 bool prealloc;
835
836 retry:
837 ret = f2fs_cluster_blocks(cc, false);
838 if (ret <= 0)
839 return ret;
840
841 /* compressed case */
842 prealloc = (ret < cc->cluster_size);
843
844 ret = f2fs_init_compress_ctx(cc);
845 if (ret)
846 return ret;
847
848 /* keep page reference to avoid page reclaim */
849 for (i = 0; i < cc->cluster_size; i++) {
850 page = f2fs_pagecache_get_page(mapping, start_idx + i,
851 fgp_flag, GFP_NOFS);
852 if (!page) {
853 ret = -ENOMEM;
854 goto unlock_pages;
855 }
856
857 if (PageUptodate(page))
858 unlock_page(page);
859 else
860 f2fs_compress_ctx_add_page(cc, page);
861 }
862
863 if (!f2fs_cluster_is_empty(cc)) {
864 struct bio *bio = NULL;
865
866 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size,
867 &last_block_in_bio, false, true);
868 f2fs_destroy_compress_ctx(cc);
869 if (ret)
870 goto release_pages;
871 if (bio)
872 f2fs_submit_bio(sbi, bio, DATA);
873
874 ret = f2fs_init_compress_ctx(cc);
875 if (ret)
876 goto release_pages;
877 }
878
879 for (i = 0; i < cc->cluster_size; i++) {
880 f2fs_bug_on(sbi, cc->rpages[i]);
881
882 page = find_lock_page(mapping, start_idx + i);
883 f2fs_bug_on(sbi, !page);
884
885 f2fs_wait_on_page_writeback(page, DATA, true, true);
886
887 f2fs_compress_ctx_add_page(cc, page);
888 f2fs_put_page(page, 0);
889
890 if (!PageUptodate(page)) {
891 f2fs_unlock_rpages(cc, i + 1);
892 f2fs_put_rpages_mapping(cc, mapping, start_idx,
893 cc->cluster_size);
894 f2fs_destroy_compress_ctx(cc);
895 goto retry;
896 }
897 }
898
899 if (prealloc) {
900 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
901
902 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
903
904 for (i = cc->cluster_size - 1; i > 0; i--) {
905 ret = f2fs_get_block(&dn, start_idx + i);
906 if (ret) {
907 i = cc->cluster_size;
908 break;
909 }
910
911 if (dn.data_blkaddr != NEW_ADDR)
912 break;
913 }
914
915 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
916 }
917
918 if (likely(!ret)) {
919 *fsdata = cc->rpages;
920 *pagep = cc->rpages[offset_in_cluster(cc, index)];
921 return cc->cluster_size;
922 }
923
924 unlock_pages:
925 f2fs_unlock_rpages(cc, i);
926 release_pages:
927 f2fs_put_rpages_mapping(cc, mapping, start_idx, i);
928 f2fs_destroy_compress_ctx(cc);
929 return ret;
930 }
931
932 int f2fs_prepare_compress_overwrite(struct inode *inode,
933 struct page **pagep, pgoff_t index, void **fsdata)
934 {
935 struct compress_ctx cc = {
936 .inode = inode,
937 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
938 .cluster_size = F2FS_I(inode)->i_cluster_size,
939 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
940 .rpages = NULL,
941 .nr_rpages = 0,
942 };
943
944 return prepare_compress_overwrite(&cc, pagep, index, fsdata);
945 }
946
947 bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
948 pgoff_t index, unsigned copied)
949
950 {
951 struct compress_ctx cc = {
952 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
953 .cluster_size = F2FS_I(inode)->i_cluster_size,
954 .rpages = fsdata,
955 };
956 bool first_index = (index == cc.rpages[0]->index);
957
958 if (copied)
959 set_cluster_dirty(&cc);
960
961 f2fs_put_rpages_wbc(&cc, NULL, false, 1);
962 f2fs_destroy_compress_ctx(&cc);
963
964 return first_index;
965 }
966
967 static int f2fs_write_compressed_pages(struct compress_ctx *cc,
968 int *submitted,
969 struct writeback_control *wbc,
970 enum iostat_type io_type)
971 {
972 struct inode *inode = cc->inode;
973 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
974 struct f2fs_inode_info *fi = F2FS_I(inode);
975 struct f2fs_io_info fio = {
976 .sbi = sbi,
977 .ino = cc->inode->i_ino,
978 .type = DATA,
979 .op = REQ_OP_WRITE,
980 .op_flags = wbc_to_write_flags(wbc),
981 .old_blkaddr = NEW_ADDR,
982 .page = NULL,
983 .encrypted_page = NULL,
984 .compressed_page = NULL,
985 .submitted = false,
986 .io_type = io_type,
987 .io_wbc = wbc,
988 .encrypted = f2fs_encrypted_file(cc->inode),
989 };
990 struct dnode_of_data dn;
991 struct node_info ni;
992 struct compress_io_ctx *cic;
993 pgoff_t start_idx = start_idx_of_cluster(cc);
994 unsigned int last_index = cc->cluster_size - 1;
995 loff_t psize;
996 int i, err;
997
998 if (!f2fs_trylock_op(sbi))
999 return -EAGAIN;
1000
1001 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
1002
1003 err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
1004 if (err)
1005 goto out_unlock_op;
1006
1007 for (i = 0; i < cc->cluster_size; i++) {
1008 if (data_blkaddr(dn.inode, dn.node_page,
1009 dn.ofs_in_node + i) == NULL_ADDR)
1010 goto out_put_dnode;
1011 }
1012
1013 psize = (loff_t)(cc->rpages[last_index]->index + 1) << PAGE_SHIFT;
1014
1015 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni);
1016 if (err)
1017 goto out_put_dnode;
1018
1019 fio.version = ni.version;
1020
1021 cic = f2fs_kzalloc(sbi, sizeof(struct compress_io_ctx), GFP_NOFS);
1022 if (!cic)
1023 goto out_put_dnode;
1024
1025 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1026 cic->inode = inode;
1027 refcount_set(&cic->ref, cc->nr_cpages);
1028 cic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
1029 cc->log_cluster_size, GFP_NOFS);
1030 if (!cic->rpages)
1031 goto out_put_cic;
1032
1033 cic->nr_rpages = cc->cluster_size;
1034
1035 for (i = 0; i < cc->nr_cpages; i++) {
1036 f2fs_set_compressed_page(cc->cpages[i], inode,
1037 cc->rpages[i + 1]->index, cic);
1038 fio.compressed_page = cc->cpages[i];
1039 if (fio.encrypted) {
1040 fio.page = cc->rpages[i + 1];
1041 err = f2fs_encrypt_one_page(&fio);
1042 if (err)
1043 goto out_destroy_crypt;
1044 cc->cpages[i] = fio.encrypted_page;
1045 }
1046 }
1047
1048 set_cluster_writeback(cc);
1049
1050 for (i = 0; i < cc->cluster_size; i++)
1051 cic->rpages[i] = cc->rpages[i];
1052
1053 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) {
1054 block_t blkaddr;
1055
1056 blkaddr = f2fs_data_blkaddr(&dn);
1057 fio.page = cc->rpages[i];
1058 fio.old_blkaddr = blkaddr;
1059
1060 /* cluster header */
1061 if (i == 0) {
1062 if (blkaddr == COMPRESS_ADDR)
1063 fio.compr_blocks++;
1064 if (__is_valid_data_blkaddr(blkaddr))
1065 f2fs_invalidate_blocks(sbi, blkaddr);
1066 f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR);
1067 goto unlock_continue;
1068 }
1069
1070 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr))
1071 fio.compr_blocks++;
1072
1073 if (i > cc->nr_cpages) {
1074 if (__is_valid_data_blkaddr(blkaddr)) {
1075 f2fs_invalidate_blocks(sbi, blkaddr);
1076 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
1077 }
1078 goto unlock_continue;
1079 }
1080
1081 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR);
1082
1083 if (fio.encrypted)
1084 fio.encrypted_page = cc->cpages[i - 1];
1085 else
1086 fio.compressed_page = cc->cpages[i - 1];
1087
1088 cc->cpages[i - 1] = NULL;
1089 f2fs_outplace_write_data(&dn, &fio);
1090 (*submitted)++;
1091 unlock_continue:
1092 inode_dec_dirty_pages(cc->inode);
1093 unlock_page(fio.page);
1094 }
1095
1096 if (fio.compr_blocks)
1097 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false);
1098 f2fs_i_compr_blocks_update(inode, cc->nr_cpages, true);
1099
1100 set_inode_flag(cc->inode, FI_APPEND_WRITE);
1101 if (cc->cluster_idx == 0)
1102 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1103
1104 f2fs_put_dnode(&dn);
1105 f2fs_unlock_op(sbi);
1106
1107 spin_lock(&fi->i_size_lock);
1108 if (fi->last_disk_size < psize)
1109 fi->last_disk_size = psize;
1110 spin_unlock(&fi->i_size_lock);
1111
1112 f2fs_put_rpages(cc);
1113 f2fs_destroy_compress_ctx(cc);
1114 return 0;
1115
1116 out_destroy_crypt:
1117 kfree(cic->rpages);
1118
1119 for (--i; i >= 0; i--)
1120 fscrypt_finalize_bounce_page(&cc->cpages[i]);
1121 for (i = 0; i < cc->nr_cpages; i++) {
1122 if (!cc->cpages[i])
1123 continue;
1124 f2fs_put_page(cc->cpages[i], 1);
1125 }
1126 out_put_cic:
1127 kfree(cic);
1128 out_put_dnode:
1129 f2fs_put_dnode(&dn);
1130 out_unlock_op:
1131 f2fs_unlock_op(sbi);
1132 return -EAGAIN;
1133 }
1134
1135 void f2fs_compress_write_end_io(struct bio *bio, struct page *page)
1136 {
1137 struct f2fs_sb_info *sbi = bio->bi_private;
1138 struct compress_io_ctx *cic =
1139 (struct compress_io_ctx *)page_private(page);
1140 int i;
1141
1142 if (unlikely(bio->bi_status))
1143 mapping_set_error(cic->inode->i_mapping, -EIO);
1144
1145 f2fs_put_compressed_page(page);
1146
1147 dec_page_count(sbi, F2FS_WB_DATA);
1148
1149 if (refcount_dec_not_one(&cic->ref))
1150 return;
1151
1152 for (i = 0; i < cic->nr_rpages; i++) {
1153 WARN_ON(!cic->rpages[i]);
1154 clear_cold_data(cic->rpages[i]);
1155 end_page_writeback(cic->rpages[i]);
1156 }
1157
1158 kfree(cic->rpages);
1159 kfree(cic);
1160 }
1161
1162 static int f2fs_write_raw_pages(struct compress_ctx *cc,
1163 int *submitted,
1164 struct writeback_control *wbc,
1165 enum iostat_type io_type)
1166 {
1167 struct address_space *mapping = cc->inode->i_mapping;
1168 int _submitted, compr_blocks, ret;
1169 int i = -1, err = 0;
1170
1171 compr_blocks = f2fs_compressed_blocks(cc);
1172 if (compr_blocks < 0) {
1173 err = compr_blocks;
1174 goto out_err;
1175 }
1176
1177 for (i = 0; i < cc->cluster_size; i++) {
1178 if (!cc->rpages[i])
1179 continue;
1180 retry_write:
1181 if (cc->rpages[i]->mapping != mapping) {
1182 unlock_page(cc->rpages[i]);
1183 continue;
1184 }
1185
1186 BUG_ON(!PageLocked(cc->rpages[i]));
1187
1188 ret = f2fs_write_single_data_page(cc->rpages[i], &_submitted,
1189 NULL, NULL, wbc, io_type,
1190 compr_blocks);
1191 if (ret) {
1192 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1193 unlock_page(cc->rpages[i]);
1194 ret = 0;
1195 } else if (ret == -EAGAIN) {
1196 /*
1197 * for quota file, just redirty left pages to
1198 * avoid deadlock caused by cluster update race
1199 * from foreground operation.
1200 */
1201 if (IS_NOQUOTA(cc->inode)) {
1202 err = 0;
1203 goto out_err;
1204 }
1205 ret = 0;
1206 cond_resched();
1207 congestion_wait(BLK_RW_ASYNC,
1208 DEFAULT_IO_TIMEOUT);
1209 lock_page(cc->rpages[i]);
1210 clear_page_dirty_for_io(cc->rpages[i]);
1211 goto retry_write;
1212 }
1213 err = ret;
1214 goto out_err;
1215 }
1216
1217 *submitted += _submitted;
1218 }
1219 return 0;
1220 out_err:
1221 for (++i; i < cc->cluster_size; i++) {
1222 if (!cc->rpages[i])
1223 continue;
1224 redirty_page_for_writepage(wbc, cc->rpages[i]);
1225 unlock_page(cc->rpages[i]);
1226 }
1227 return err;
1228 }
1229
1230 int f2fs_write_multi_pages(struct compress_ctx *cc,
1231 int *submitted,
1232 struct writeback_control *wbc,
1233 enum iostat_type io_type)
1234 {
1235 struct f2fs_inode_info *fi = F2FS_I(cc->inode);
1236 const struct f2fs_compress_ops *cops =
1237 f2fs_cops[fi->i_compress_algorithm];
1238 int err;
1239
1240 *submitted = 0;
1241 if (cluster_may_compress(cc)) {
1242 err = f2fs_compress_pages(cc);
1243 if (err == -EAGAIN) {
1244 goto write;
1245 } else if (err) {
1246 f2fs_put_rpages_wbc(cc, wbc, true, 1);
1247 goto destroy_out;
1248 }
1249
1250 err = f2fs_write_compressed_pages(cc, submitted,
1251 wbc, io_type);
1252 cops->destroy_compress_ctx(cc);
1253 if (!err)
1254 return 0;
1255 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN);
1256 }
1257 write:
1258 f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted);
1259
1260 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type);
1261 f2fs_put_rpages_wbc(cc, wbc, false, 0);
1262 destroy_out:
1263 f2fs_destroy_compress_ctx(cc);
1264 return err;
1265 }
1266
1267 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc)
1268 {
1269 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
1270 struct decompress_io_ctx *dic;
1271 pgoff_t start_idx = start_idx_of_cluster(cc);
1272 int i;
1273
1274 dic = f2fs_kzalloc(sbi, sizeof(struct decompress_io_ctx), GFP_NOFS);
1275 if (!dic)
1276 return ERR_PTR(-ENOMEM);
1277
1278 dic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
1279 cc->log_cluster_size, GFP_NOFS);
1280 if (!dic->rpages) {
1281 kfree(dic);
1282 return ERR_PTR(-ENOMEM);
1283 }
1284
1285 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1286 dic->inode = cc->inode;
1287 refcount_set(&dic->ref, cc->nr_cpages);
1288 dic->cluster_idx = cc->cluster_idx;
1289 dic->cluster_size = cc->cluster_size;
1290 dic->log_cluster_size = cc->log_cluster_size;
1291 dic->nr_cpages = cc->nr_cpages;
1292 dic->failed = false;
1293
1294 for (i = 0; i < dic->cluster_size; i++)
1295 dic->rpages[i] = cc->rpages[i];
1296 dic->nr_rpages = cc->cluster_size;
1297
1298 dic->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1299 dic->nr_cpages, GFP_NOFS);
1300 if (!dic->cpages)
1301 goto out_free;
1302
1303 for (i = 0; i < dic->nr_cpages; i++) {
1304 struct page *page;
1305
1306 page = f2fs_grab_page();
1307 if (!page)
1308 goto out_free;
1309
1310 f2fs_set_compressed_page(page, cc->inode,
1311 start_idx + i + 1, dic);
1312 dic->cpages[i] = page;
1313 }
1314
1315 dic->tpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1316 dic->cluster_size, GFP_NOFS);
1317 if (!dic->tpages)
1318 goto out_free;
1319
1320 for (i = 0; i < dic->cluster_size; i++) {
1321 if (cc->rpages[i]) {
1322 dic->tpages[i] = cc->rpages[i];
1323 continue;
1324 }
1325
1326 dic->tpages[i] = f2fs_grab_page();
1327 if (!dic->tpages[i])
1328 goto out_free;
1329 }
1330
1331 return dic;
1332
1333 out_free:
1334 f2fs_free_dic(dic);
1335 return ERR_PTR(-ENOMEM);
1336 }
1337
1338 void f2fs_free_dic(struct decompress_io_ctx *dic)
1339 {
1340 int i;
1341
1342 if (dic->tpages) {
1343 for (i = 0; i < dic->cluster_size; i++) {
1344 if (dic->rpages[i])
1345 continue;
1346 if (!dic->tpages[i])
1347 continue;
1348 unlock_page(dic->tpages[i]);
1349 put_page(dic->tpages[i]);
1350 }
1351 kfree(dic->tpages);
1352 }
1353
1354 if (dic->cpages) {
1355 for (i = 0; i < dic->nr_cpages; i++) {
1356 if (!dic->cpages[i])
1357 continue;
1358 f2fs_put_compressed_page(dic->cpages[i]);
1359 }
1360 kfree(dic->cpages);
1361 }
1362
1363 kfree(dic->rpages);
1364 kfree(dic);
1365 }
1366
1367 void f2fs_decompress_end_io(struct page **rpages,
1368 unsigned int cluster_size, bool err, bool verity)
1369 {
1370 int i;
1371
1372 for (i = 0; i < cluster_size; i++) {
1373 struct page *rpage = rpages[i];
1374
1375 if (!rpage)
1376 continue;
1377
1378 if (err || PageError(rpage))
1379 goto clear_uptodate;
1380
1381 if (!verity || fsverity_verify_page(rpage)) {
1382 SetPageUptodate(rpage);
1383 goto unlock;
1384 }
1385 clear_uptodate:
1386 ClearPageUptodate(rpage);
1387 ClearPageError(rpage);
1388 unlock:
1389 unlock_page(rpage);
1390 }
1391 }
Linux with added FPC-III support