Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2016-present, Facebook, Inc.
4 : * All rights reserved.
5 : *
6 : */
7 :
8 : #include <linux/bio.h>
9 : #include <linux/bitmap.h>
10 : #include <linux/err.h>
11 : #include <linux/init.h>
12 : #include <linux/kernel.h>
13 : #include <linux/mm.h>
14 : #include <linux/sched/mm.h>
15 : #include <linux/pagemap.h>
16 : #include <linux/refcount.h>
17 : #include <linux/sched.h>
18 : #include <linux/slab.h>
19 : #include <linux/zstd.h>
20 : #include "misc.h"
21 : #include "compression.h"
22 : #include "ctree.h"
23 :
24 : #define ZSTD_BTRFS_MAX_WINDOWLOG 17
25 : #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
26 : #define ZSTD_BTRFS_DEFAULT_LEVEL 3
27 : #define ZSTD_BTRFS_MAX_LEVEL 15
28 : /* 307s to avoid pathologically clashing with transaction commit */
29 : #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ)
30 :
31 8377 : static zstd_parameters zstd_get_btrfs_parameters(unsigned int level,
32 : size_t src_len)
33 : {
34 8377 : zstd_parameters params = zstd_get_params(level, src_len);
35 :
36 8371 : if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
37 : params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
38 8371 : WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
39 8371 : return params;
40 : }
41 :
42 : struct workspace {
43 : void *mem;
44 : size_t size;
45 : char *buf;
46 : unsigned int level;
47 : unsigned int req_level;
48 : unsigned long last_used; /* jiffies */
49 : struct list_head list;
50 : struct list_head lru_list;
51 : zstd_in_buffer in_buf;
52 : zstd_out_buffer out_buf;
53 : };
54 :
55 : /*
56 : * Zstd Workspace Management
57 : *
58 : * Zstd workspaces have different memory requirements depending on the level.
59 : * The zstd workspaces are managed by having individual lists for each level
60 : * and a global lru. Forward progress is maintained by protecting a max level
61 : * workspace.
62 : *
63 : * Getting a workspace is done by using the bitmap to identify the levels that
64 : * have available workspaces and scans up. This lets us recycle higher level
65 : * workspaces because of the monotonic memory guarantee. A workspace's
66 : * last_used is only updated if it is being used by the corresponding memory
67 : * level. Putting a workspace involves adding it back to the appropriate places
68 : * and adding it back to the lru if necessary.
69 : *
70 : * A timer is used to reclaim workspaces if they have not been used for
71 : * ZSTD_BTRFS_RECLAIM_JIFFIES. This helps keep only active workspaces around.
72 : * The upper bound is provided by the workqueue limit which is 2 (percpu limit).
73 : */
74 :
75 : struct zstd_workspace_manager {
76 : const struct btrfs_compress_op *ops;
77 : spinlock_t lock;
78 : struct list_head lru_list;
79 : struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL];
80 : unsigned long active_map;
81 : wait_queue_head_t wait;
82 : struct timer_list timer;
83 : };
84 :
85 : static struct zstd_workspace_manager wsm;
86 :
87 : static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL];
88 :
89 : static inline struct workspace *list_to_workspace(struct list_head *list)
90 : {
91 : return container_of(list, struct workspace, list);
92 : }
93 :
94 : void zstd_free_workspace(struct list_head *ws);
95 : struct list_head *zstd_alloc_workspace(unsigned int level);
96 :
97 : /*
98 : * Timer callback to free unused workspaces.
99 : *
100 : * @t: timer
101 : *
102 : * This scans the lru_list and attempts to reclaim any workspace that hasn't
103 : * been used for ZSTD_BTRFS_RECLAIM_JIFFIES.
104 : *
105 : * The context is softirq and does not need the _bh locking primitives.
106 : */
107 3 : static void zstd_reclaim_timer_fn(struct timer_list *timer)
108 : {
109 3 : unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES;
110 3 : struct list_head *pos, *next;
111 :
112 3 : spin_lock(&wsm.lock);
113 :
114 3 : if (list_empty(&wsm.lru_list)) {
115 0 : spin_unlock(&wsm.lock);
116 0 : return;
117 : }
118 :
119 8 : list_for_each_prev_safe(pos, next, &wsm.lru_list) {
120 6 : struct workspace *victim = container_of(pos, struct workspace,
121 : lru_list);
122 6 : unsigned int level;
123 :
124 6 : if (time_after(victim->last_used, reclaim_threshold))
125 : break;
126 :
127 : /* workspace is in use */
128 5 : if (victim->req_level)
129 0 : continue;
130 :
131 5 : level = victim->level;
132 5 : list_del(&victim->lru_list);
133 5 : list_del(&victim->list);
134 5 : zstd_free_workspace(&victim->list);
135 :
136 5 : if (list_empty(&wsm.idle_ws[level - 1]))
137 1 : clear_bit(level - 1, &wsm.active_map);
138 :
139 : }
140 :
141 3 : if (!list_empty(&wsm.lru_list))
142 1 : mod_timer(&wsm.timer, jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
143 :
144 3 : spin_unlock(&wsm.lock);
145 : }
146 :
147 : /*
148 : * zstd_calc_ws_mem_sizes - calculate monotonic memory bounds
149 : *
150 : * It is possible based on the level configurations that a higher level
151 : * workspace uses less memory than a lower level workspace. In order to reuse
152 : * workspaces, this must be made a monotonic relationship. This precomputes
153 : * the required memory for each level and enforces the monotonicity between
154 : * level and memory required.
155 : */
156 11 : static void zstd_calc_ws_mem_sizes(void)
157 : {
158 11 : size_t max_size = 0;
159 11 : unsigned int level;
160 :
161 176 : for (level = 1; level <= ZSTD_BTRFS_MAX_LEVEL; level++) {
162 165 : zstd_parameters params =
163 165 : zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT);
164 165 : size_t level_size =
165 165 : max_t(size_t,
166 : zstd_cstream_workspace_bound(¶ms.cParams),
167 : zstd_dstream_workspace_bound(ZSTD_BTRFS_MAX_INPUT));
168 :
169 165 : max_size = max_t(size_t, max_size, level_size);
170 165 : zstd_ws_mem_sizes[level - 1] = max_size;
171 : }
172 11 : }
173 :
174 11 : void zstd_init_workspace_manager(void)
175 : {
176 11 : struct list_head *ws;
177 11 : int i;
178 :
179 11 : zstd_calc_ws_mem_sizes();
180 :
181 11 : wsm.ops = &btrfs_zstd_compress;
182 11 : spin_lock_init(&wsm.lock);
183 11 : init_waitqueue_head(&wsm.wait);
184 11 : timer_setup(&wsm.timer, zstd_reclaim_timer_fn, 0);
185 :
186 11 : INIT_LIST_HEAD(&wsm.lru_list);
187 176 : for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++)
188 165 : INIT_LIST_HEAD(&wsm.idle_ws[i]);
189 :
190 11 : ws = zstd_alloc_workspace(ZSTD_BTRFS_MAX_LEVEL);
191 11 : if (IS_ERR(ws)) {
192 0 : pr_warn(
193 : "BTRFS: cannot preallocate zstd compression workspace\n");
194 : } else {
195 11 : set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, &wsm.active_map);
196 11 : list_add(ws, &wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]);
197 : }
198 11 : }
199 :
200 0 : void zstd_cleanup_workspace_manager(void)
201 : {
202 0 : struct workspace *workspace;
203 0 : int i;
204 :
205 0 : spin_lock_bh(&wsm.lock);
206 0 : for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) {
207 0 : while (!list_empty(&wsm.idle_ws[i])) {
208 0 : workspace = container_of(wsm.idle_ws[i].next,
209 : struct workspace, list);
210 0 : list_del(&workspace->list);
211 0 : list_del(&workspace->lru_list);
212 0 : zstd_free_workspace(&workspace->list);
213 : }
214 : }
215 0 : spin_unlock_bh(&wsm.lock);
216 :
217 0 : del_timer_sync(&wsm.timer);
218 0 : }
219 :
220 : /*
221 : * zstd_find_workspace - find workspace
222 : * @level: compression level
223 : *
224 : * This iterates over the set bits in the active_map beginning at the requested
225 : * compression level. This lets us utilize already allocated workspaces before
226 : * allocating a new one. If the workspace is of a larger size, it is used, but
227 : * the place in the lru_list and last_used times are not updated. This is to
228 : * offer the opportunity to reclaim the workspace in favor of allocating an
229 : * appropriately sized one in the future.
230 : */
231 8215 : static struct list_head *zstd_find_workspace(unsigned int level)
232 : {
233 8215 : struct list_head *ws;
234 8215 : struct workspace *workspace;
235 8215 : int i = level - 1;
236 :
237 8215 : spin_lock_bh(&wsm.lock);
238 8218 : for_each_set_bit_from(i, &wsm.active_map, ZSTD_BTRFS_MAX_LEVEL) {
239 8213 : if (!list_empty(&wsm.idle_ws[i])) {
240 8213 : ws = wsm.idle_ws[i].next;
241 8213 : workspace = list_to_workspace(ws);
242 8213 : list_del_init(ws);
243 : /* keep its place if it's a lower level using this */
244 8213 : workspace->req_level = level;
245 8213 : if (level == workspace->level)
246 7143 : list_del(&workspace->lru_list);
247 8213 : if (list_empty(&wsm.idle_ws[i]))
248 7089 : clear_bit(i, &wsm.active_map);
249 8213 : spin_unlock_bh(&wsm.lock);
250 8213 : return ws;
251 : }
252 : }
253 5 : spin_unlock_bh(&wsm.lock);
254 :
255 5 : return NULL;
256 : }
257 :
258 : /*
259 : * zstd_get_workspace - zstd's get_workspace
260 : * @level: compression level
261 : *
262 : * If @level is 0, then any compression level can be used. Therefore, we begin
263 : * scanning from 1. We first scan through possible workspaces and then after
264 : * attempt to allocate a new workspace. If we fail to allocate one due to
265 : * memory pressure, go to sleep waiting for the max level workspace to free up.
266 : */
267 8217 : struct list_head *zstd_get_workspace(unsigned int level)
268 : {
269 8217 : struct list_head *ws;
270 8217 : unsigned int nofs_flag;
271 :
272 : /* level == 0 means we can use any workspace */
273 8217 : if (!level)
274 : level = 1;
275 :
276 : again:
277 8217 : ws = zstd_find_workspace(level);
278 8218 : if (ws)
279 8213 : return ws;
280 :
281 5 : nofs_flag = memalloc_nofs_save();
282 5 : ws = zstd_alloc_workspace(level);
283 5 : memalloc_nofs_restore(nofs_flag);
284 :
285 5 : if (IS_ERR(ws)) {
286 0 : DEFINE_WAIT(wait);
287 :
288 0 : prepare_to_wait(&wsm.wait, &wait, TASK_UNINTERRUPTIBLE);
289 0 : schedule();
290 0 : finish_wait(&wsm.wait, &wait);
291 :
292 0 : goto again;
293 : }
294 :
295 : return ws;
296 : }
297 :
298 : /*
299 : * zstd_put_workspace - zstd put_workspace
300 : * @ws: list_head for the workspace
301 : *
302 : * When putting back a workspace, we only need to update the LRU if we are of
303 : * the requested compression level. Here is where we continue to protect the
304 : * max level workspace or update last_used accordingly. If the reclaim timer
305 : * isn't set, it is also set here. Only the max level workspace tries and wakes
306 : * up waiting workspaces.
307 : */
308 8214 : void zstd_put_workspace(struct list_head *ws)
309 : {
310 8214 : struct workspace *workspace = list_to_workspace(ws);
311 :
312 8214 : spin_lock_bh(&wsm.lock);
313 :
314 : /* A node is only taken off the lru if we are the corresponding level */
315 8218 : if (workspace->req_level == workspace->level) {
316 : /* Hide a max level workspace from reclaim */
317 7148 : if (list_empty(&wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) {
318 5590 : INIT_LIST_HEAD(&workspace->lru_list);
319 : } else {
320 1558 : workspace->last_used = jiffies;
321 1558 : list_add(&workspace->lru_list, &wsm.lru_list);
322 1558 : if (!timer_pending(&wsm.timer))
323 2 : mod_timer(&wsm.timer,
324 : jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
325 : }
326 : }
327 :
328 8218 : set_bit(workspace->level - 1, &wsm.active_map);
329 8218 : list_add(&workspace->list, &wsm.idle_ws[workspace->level - 1]);
330 8218 : workspace->req_level = 0;
331 :
332 8218 : spin_unlock_bh(&wsm.lock);
333 :
334 8218 : if (workspace->level == ZSTD_BTRFS_MAX_LEVEL)
335 1066 : cond_wake_up(&wsm.wait);
336 8218 : }
337 :
338 5 : void zstd_free_workspace(struct list_head *ws)
339 : {
340 5 : struct workspace *workspace = list_entry(ws, struct workspace, list);
341 :
342 5 : kvfree(workspace->mem);
343 5 : kfree(workspace->buf);
344 5 : kfree(workspace);
345 5 : }
346 :
347 16 : struct list_head *zstd_alloc_workspace(unsigned int level)
348 : {
349 16 : struct workspace *workspace;
350 :
351 16 : workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
352 16 : if (!workspace)
353 : return ERR_PTR(-ENOMEM);
354 :
355 16 : workspace->size = zstd_ws_mem_sizes[level - 1];
356 16 : workspace->level = level;
357 16 : workspace->req_level = level;
358 16 : workspace->last_used = jiffies;
359 16 : workspace->mem = kvmalloc(workspace->size, GFP_KERNEL | __GFP_NOWARN);
360 16 : workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
361 16 : if (!workspace->mem || !workspace->buf)
362 0 : goto fail;
363 :
364 16 : INIT_LIST_HEAD(&workspace->list);
365 16 : INIT_LIST_HEAD(&workspace->lru_list);
366 :
367 16 : return &workspace->list;
368 : fail:
369 0 : zstd_free_workspace(&workspace->list);
370 0 : return ERR_PTR(-ENOMEM);
371 : }
372 :
373 8214 : int zstd_compress_pages(struct list_head *ws, struct address_space *mapping,
374 : u64 start, struct page **pages, unsigned long *out_pages,
375 : unsigned long *total_in, unsigned long *total_out)
376 : {
377 8214 : struct workspace *workspace = list_entry(ws, struct workspace, list);
378 8214 : zstd_cstream *stream;
379 8214 : int ret = 0;
380 8214 : int nr_pages = 0;
381 8214 : struct page *in_page = NULL; /* The current page to read */
382 8214 : struct page *out_page = NULL; /* The current page to write to */
383 8214 : unsigned long tot_in = 0;
384 8214 : unsigned long tot_out = 0;
385 8214 : unsigned long len = *total_out;
386 8214 : const unsigned long nr_dest_pages = *out_pages;
387 8214 : unsigned long max_out = nr_dest_pages * PAGE_SIZE;
388 8214 : zstd_parameters params = zstd_get_btrfs_parameters(workspace->req_level,
389 : len);
390 :
391 8208 : *out_pages = 0;
392 8208 : *total_out = 0;
393 8208 : *total_in = 0;
394 :
395 : /* Initialize the stream */
396 8208 : stream = zstd_init_cstream(¶ms, len, workspace->mem,
397 : workspace->size);
398 8198 : if (!stream) {
399 0 : pr_warn("BTRFS: zstd_init_cstream failed\n");
400 0 : ret = -EIO;
401 0 : goto out;
402 : }
403 :
404 : /* map in the first page of input data */
405 8198 : in_page = find_get_page(mapping, start >> PAGE_SHIFT);
406 8212 : workspace->in_buf.src = kmap_local_page(in_page);
407 8212 : workspace->in_buf.pos = 0;
408 8212 : workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
409 :
410 :
411 : /* Allocate and map in the output buffer */
412 8212 : out_page = alloc_page(GFP_NOFS);
413 8197 : if (out_page == NULL) {
414 0 : ret = -ENOMEM;
415 0 : goto out;
416 : }
417 8197 : pages[nr_pages++] = out_page;
418 8197 : workspace->out_buf.dst = page_address(out_page);
419 8197 : workspace->out_buf.pos = 0;
420 8197 : workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
421 :
422 262315 : while (1) {
423 262315 : size_t ret2;
424 :
425 262315 : ret2 = zstd_compress_stream(stream, &workspace->out_buf,
426 : &workspace->in_buf);
427 262321 : if (zstd_is_error(ret2)) {
428 0 : pr_debug("BTRFS: zstd_compress_stream returned %d\n",
429 : zstd_get_error_code(ret2));
430 0 : ret = -EIO;
431 0 : goto out;
432 : }
433 :
434 : /* Check to see if we are making it bigger */
435 262325 : if (tot_in + workspace->in_buf.pos > 8192 &&
436 : tot_in + workspace->in_buf.pos <
437 245912 : tot_out + workspace->out_buf.pos) {
438 0 : ret = -E2BIG;
439 0 : goto out;
440 : }
441 :
442 : /* We've reached the end of our output range */
443 262325 : if (workspace->out_buf.pos >= max_out) {
444 0 : tot_out += workspace->out_buf.pos;
445 0 : ret = -E2BIG;
446 0 : goto out;
447 : }
448 :
449 : /* Check if we need more output space */
450 262325 : if (workspace->out_buf.pos == workspace->out_buf.size) {
451 0 : tot_out += PAGE_SIZE;
452 0 : max_out -= PAGE_SIZE;
453 0 : if (nr_pages == nr_dest_pages) {
454 0 : ret = -E2BIG;
455 0 : goto out;
456 : }
457 0 : out_page = alloc_page(GFP_NOFS);
458 0 : if (out_page == NULL) {
459 0 : ret = -ENOMEM;
460 0 : goto out;
461 : }
462 0 : pages[nr_pages++] = out_page;
463 0 : workspace->out_buf.dst = page_address(out_page);
464 0 : workspace->out_buf.pos = 0;
465 0 : workspace->out_buf.size = min_t(size_t, max_out,
466 : PAGE_SIZE);
467 : }
468 :
469 : /* We've reached the end of the input */
470 262325 : if (workspace->in_buf.pos >= len) {
471 8210 : tot_in += workspace->in_buf.pos;
472 8210 : break;
473 : }
474 :
475 : /* Check if we need more input */
476 254115 : if (workspace->in_buf.pos == workspace->in_buf.size) {
477 254117 : tot_in += PAGE_SIZE;
478 254117 : kunmap_local(workspace->in_buf.src);
479 254117 : put_page(in_page);
480 254138 : start += PAGE_SIZE;
481 254138 : len -= PAGE_SIZE;
482 254138 : in_page = find_get_page(mapping, start >> PAGE_SHIFT);
483 254120 : workspace->in_buf.src = kmap_local_page(in_page);
484 254120 : workspace->in_buf.pos = 0;
485 254120 : workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
486 : }
487 : }
488 221066 : while (1) {
489 114638 : size_t ret2;
490 :
491 114638 : ret2 = zstd_end_stream(stream, &workspace->out_buf);
492 114676 : if (zstd_is_error(ret2)) {
493 0 : pr_debug("BTRFS: zstd_end_stream returned %d\n",
494 : zstd_get_error_code(ret2));
495 0 : ret = -EIO;
496 0 : goto out;
497 : }
498 114668 : if (ret2 == 0) {
499 8209 : tot_out += workspace->out_buf.pos;
500 8209 : break;
501 : }
502 106459 : if (workspace->out_buf.pos >= max_out) {
503 1 : tot_out += workspace->out_buf.pos;
504 1 : ret = -E2BIG;
505 1 : goto out;
506 : }
507 :
508 106458 : tot_out += PAGE_SIZE;
509 106458 : max_out -= PAGE_SIZE;
510 106458 : if (nr_pages == nr_dest_pages) {
511 0 : ret = -E2BIG;
512 0 : goto out;
513 : }
514 106458 : out_page = alloc_page(GFP_NOFS);
515 106428 : if (out_page == NULL) {
516 0 : ret = -ENOMEM;
517 0 : goto out;
518 : }
519 106428 : pages[nr_pages++] = out_page;
520 106428 : workspace->out_buf.dst = page_address(out_page);
521 106428 : workspace->out_buf.pos = 0;
522 106428 : workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
523 : }
524 :
525 8209 : if (tot_out >= tot_in) {
526 0 : ret = -E2BIG;
527 0 : goto out;
528 : }
529 :
530 8209 : ret = 0;
531 8209 : *total_in = tot_in;
532 8209 : *total_out = tot_out;
533 8210 : out:
534 8210 : *out_pages = nr_pages;
535 8210 : if (workspace->in_buf.src) {
536 8211 : kunmap_local(workspace->in_buf.src);
537 8211 : put_page(in_page);
538 : }
539 8212 : return ret;
540 : }
541 :
542 4 : int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
543 : {
544 4 : struct workspace *workspace = list_entry(ws, struct workspace, list);
545 4 : struct page **pages_in = cb->compressed_pages;
546 4 : size_t srclen = cb->compressed_len;
547 4 : zstd_dstream *stream;
548 4 : int ret = 0;
549 4 : unsigned long page_in_index = 0;
550 4 : unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
551 4 : unsigned long buf_start;
552 4 : unsigned long total_out = 0;
553 :
554 4 : stream = zstd_init_dstream(
555 : ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
556 4 : if (!stream) {
557 0 : pr_debug("BTRFS: zstd_init_dstream failed\n");
558 0 : ret = -EIO;
559 0 : goto done;
560 : }
561 :
562 4 : workspace->in_buf.src = kmap_local_page(pages_in[page_in_index]);
563 4 : workspace->in_buf.pos = 0;
564 4 : workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
565 :
566 4 : workspace->out_buf.dst = workspace->buf;
567 4 : workspace->out_buf.pos = 0;
568 4 : workspace->out_buf.size = PAGE_SIZE;
569 :
570 180 : while (1) {
571 180 : size_t ret2;
572 :
573 180 : ret2 = zstd_decompress_stream(stream, &workspace->out_buf,
574 : &workspace->in_buf);
575 180 : if (zstd_is_error(ret2)) {
576 0 : pr_debug("BTRFS: zstd_decompress_stream returned %d\n",
577 : zstd_get_error_code(ret2));
578 0 : ret = -EIO;
579 0 : goto done;
580 : }
581 180 : buf_start = total_out;
582 180 : total_out += workspace->out_buf.pos;
583 180 : workspace->out_buf.pos = 0;
584 :
585 180 : ret = btrfs_decompress_buf2page(workspace->out_buf.dst,
586 : total_out - buf_start, cb, buf_start);
587 180 : if (ret == 0)
588 : break;
589 :
590 176 : if (workspace->in_buf.pos >= srclen)
591 : break;
592 :
593 : /* Check if we've hit the end of a frame */
594 176 : if (ret2 == 0)
595 : break;
596 :
597 176 : if (workspace->in_buf.pos == workspace->in_buf.size) {
598 52 : kunmap_local(workspace->in_buf.src);
599 52 : page_in_index++;
600 52 : if (page_in_index >= total_pages_in) {
601 0 : workspace->in_buf.src = NULL;
602 0 : ret = -EIO;
603 0 : goto done;
604 : }
605 52 : srclen -= PAGE_SIZE;
606 52 : workspace->in_buf.src = kmap_local_page(pages_in[page_in_index]);
607 52 : workspace->in_buf.pos = 0;
608 52 : workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
609 : }
610 : }
611 : ret = 0;
612 4 : done:
613 4 : if (workspace->in_buf.src)
614 4 : kunmap_local(workspace->in_buf.src);
615 4 : return ret;
616 : }
617 :
618 0 : int zstd_decompress(struct list_head *ws, const u8 *data_in,
619 : struct page *dest_page, unsigned long start_byte, size_t srclen,
620 : size_t destlen)
621 : {
622 0 : struct workspace *workspace = list_entry(ws, struct workspace, list);
623 0 : zstd_dstream *stream;
624 0 : int ret = 0;
625 0 : size_t ret2;
626 0 : unsigned long total_out = 0;
627 0 : unsigned long pg_offset = 0;
628 :
629 0 : stream = zstd_init_dstream(
630 : ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
631 0 : if (!stream) {
632 0 : pr_warn("BTRFS: zstd_init_dstream failed\n");
633 0 : ret = -EIO;
634 0 : goto finish;
635 : }
636 :
637 0 : destlen = min_t(size_t, destlen, PAGE_SIZE);
638 :
639 0 : workspace->in_buf.src = data_in;
640 0 : workspace->in_buf.pos = 0;
641 0 : workspace->in_buf.size = srclen;
642 :
643 0 : workspace->out_buf.dst = workspace->buf;
644 0 : workspace->out_buf.pos = 0;
645 0 : workspace->out_buf.size = PAGE_SIZE;
646 :
647 0 : ret2 = 1;
648 0 : while (pg_offset < destlen
649 0 : && workspace->in_buf.pos < workspace->in_buf.size) {
650 0 : unsigned long buf_start;
651 0 : unsigned long buf_offset;
652 0 : unsigned long bytes;
653 :
654 : /* Check if the frame is over and we still need more input */
655 0 : if (ret2 == 0) {
656 0 : pr_debug("BTRFS: zstd_decompress_stream ended early\n");
657 0 : ret = -EIO;
658 0 : goto finish;
659 : }
660 0 : ret2 = zstd_decompress_stream(stream, &workspace->out_buf,
661 : &workspace->in_buf);
662 0 : if (zstd_is_error(ret2)) {
663 0 : pr_debug("BTRFS: zstd_decompress_stream returned %d\n",
664 : zstd_get_error_code(ret2));
665 0 : ret = -EIO;
666 0 : goto finish;
667 : }
668 :
669 0 : buf_start = total_out;
670 0 : total_out += workspace->out_buf.pos;
671 0 : workspace->out_buf.pos = 0;
672 :
673 0 : if (total_out <= start_byte)
674 0 : continue;
675 :
676 0 : if (total_out > start_byte && buf_start < start_byte)
677 0 : buf_offset = start_byte - buf_start;
678 : else
679 : buf_offset = 0;
680 :
681 0 : bytes = min_t(unsigned long, destlen - pg_offset,
682 : workspace->out_buf.size - buf_offset);
683 :
684 0 : memcpy_to_page(dest_page, pg_offset,
685 0 : workspace->out_buf.dst + buf_offset, bytes);
686 :
687 0 : pg_offset += bytes;
688 : }
689 : ret = 0;
690 0 : finish:
691 0 : if (pg_offset < destlen) {
692 0 : memzero_page(dest_page, pg_offset, destlen - pg_offset);
693 : }
694 0 : return ret;
695 : }
696 :
697 : const struct btrfs_compress_op btrfs_zstd_compress = {
698 : /* ZSTD uses own workspace manager */
699 : .workspace_manager = NULL,
700 : .max_level = ZSTD_BTRFS_MAX_LEVEL,
701 : .default_level = ZSTD_BTRFS_DEFAULT_LEVEL,
702 : };
|
