Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 :
3 : #include <linux/bitops.h>
4 : #include <linux/slab.h>
5 : #include <linux/bio.h>
6 : #include <linux/mm.h>
7 : #include <linux/pagemap.h>
8 : #include <linux/page-flags.h>
9 : #include <linux/sched/mm.h>
10 : #include <linux/spinlock.h>
11 : #include <linux/blkdev.h>
12 : #include <linux/swap.h>
13 : #include <linux/writeback.h>
14 : #include <linux/pagevec.h>
15 : #include <linux/prefetch.h>
16 : #include <linux/fsverity.h>
17 : #include "misc.h"
18 : #include "extent_io.h"
19 : #include "extent-io-tree.h"
20 : #include "extent_map.h"
21 : #include "ctree.h"
22 : #include "btrfs_inode.h"
23 : #include "bio.h"
24 : #include "check-integrity.h"
25 : #include "locking.h"
26 : #include "rcu-string.h"
27 : #include "backref.h"
28 : #include "disk-io.h"
29 : #include "subpage.h"
30 : #include "zoned.h"
31 : #include "block-group.h"
32 : #include "compression.h"
33 : #include "fs.h"
34 : #include "accessors.h"
35 : #include "file-item.h"
36 : #include "file.h"
37 : #include "dev-replace.h"
38 : #include "super.h"
39 : #include "transaction.h"
40 :
41 : static struct kmem_cache *extent_buffer_cache;
42 :
43 : #ifdef CONFIG_BTRFS_DEBUG
44 : static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
45 : {
46 : struct btrfs_fs_info *fs_info = eb->fs_info;
47 : unsigned long flags;
48 :
49 : spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
50 : list_add(&eb->leak_list, &fs_info->allocated_ebs);
51 : spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
52 : }
53 :
54 : static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
55 : {
56 : struct btrfs_fs_info *fs_info = eb->fs_info;
57 : unsigned long flags;
58 :
59 : spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
60 : list_del(&eb->leak_list);
61 : spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
62 : }
63 :
64 : void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
65 : {
66 : struct extent_buffer *eb;
67 : unsigned long flags;
68 :
69 : /*
70 : * If we didn't get into open_ctree our allocated_ebs will not be
71 : * initialized, so just skip this.
72 : */
73 : if (!fs_info->allocated_ebs.next)
74 : return;
75 :
76 : WARN_ON(!list_empty(&fs_info->allocated_ebs));
77 : spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
78 : while (!list_empty(&fs_info->allocated_ebs)) {
79 : eb = list_first_entry(&fs_info->allocated_ebs,
80 : struct extent_buffer, leak_list);
81 : pr_err(
82 : "BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n",
83 : eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
84 : btrfs_header_owner(eb));
85 : list_del(&eb->leak_list);
86 : kmem_cache_free(extent_buffer_cache, eb);
87 : }
88 : spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
89 : }
90 : #else
91 : #define btrfs_leak_debug_add_eb(eb) do {} while (0)
92 : #define btrfs_leak_debug_del_eb(eb) do {} while (0)
93 : #endif
94 :
95 : /*
96 : * Structure to record info about the bio being assembled, and other info like
97 : * how many bytes are there before stripe/ordered extent boundary.
98 : */
99 : struct btrfs_bio_ctrl {
100 : struct btrfs_bio *bbio;
101 : enum btrfs_compression_type compress_type;
102 : u32 len_to_oe_boundary;
103 : blk_opf_t opf;
104 : btrfs_bio_end_io_t end_io_func;
105 : struct writeback_control *wbc;
106 : };
107 :
108 18447708 : static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
109 : {
110 18447708 : struct btrfs_bio *bbio = bio_ctrl->bbio;
111 :
112 18447708 : if (!bbio)
113 : return;
114 :
115 : /* Caller should ensure the bio has at least some range added */
116 4362720 : ASSERT(bbio->bio.bi_iter.bi_size);
117 :
118 4362720 : if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ &&
119 1735593 : bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
120 1162 : btrfs_submit_compressed_read(bbio);
121 : else
122 4360926 : btrfs_submit_bio(bbio, 0);
123 :
124 : /* The bbio is owned by the end_io handler now */
125 4363667 : bio_ctrl->bbio = NULL;
126 : }
127 :
128 : /*
129 : * Submit or fail the current bio in the bio_ctrl structure.
130 : */
131 6480192 : static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
132 : {
133 6480192 : struct btrfs_bio *bbio = bio_ctrl->bbio;
134 :
135 6480192 : if (!bbio)
136 : return;
137 :
138 20886 : if (ret) {
139 0 : ASSERT(ret < 0);
140 0 : btrfs_bio_end_io(bbio, errno_to_blk_status(ret));
141 : /* The bio is owned by the end_io handler now */
142 0 : bio_ctrl->bbio = NULL;
143 : } else {
144 20886 : submit_one_bio(bio_ctrl);
145 : }
146 : }
147 :
148 11 : int __init extent_buffer_init_cachep(void)
149 : {
150 11 : extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
151 : sizeof(struct extent_buffer), 0,
152 : SLAB_MEM_SPREAD, NULL);
153 11 : if (!extent_buffer_cache)
154 0 : return -ENOMEM;
155 :
156 : return 0;
157 : }
158 :
159 0 : void __cold extent_buffer_free_cachep(void)
160 : {
161 : /*
162 : * Make sure all delayed rcu free are flushed before we
163 : * destroy caches.
164 : */
165 0 : rcu_barrier();
166 0 : kmem_cache_destroy(extent_buffer_cache);
167 0 : }
168 :
169 39762 : void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
170 : {
171 39762 : unsigned long index = start >> PAGE_SHIFT;
172 39762 : unsigned long end_index = end >> PAGE_SHIFT;
173 39762 : struct page *page;
174 :
175 5096971 : while (index <= end_index) {
176 5057203 : page = find_get_page(inode->i_mapping, index);
177 5057250 : BUG_ON(!page); /* Pages should be in the extent_io_tree */
178 5057250 : clear_page_dirty_for_io(page);
179 5055964 : put_page(page);
180 5057209 : index++;
181 : }
182 39768 : }
183 :
184 4 : void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
185 : {
186 4 : struct address_space *mapping = inode->i_mapping;
187 4 : unsigned long index = start >> PAGE_SHIFT;
188 4 : unsigned long end_index = end >> PAGE_SHIFT;
189 4 : struct folio *folio;
190 :
191 8 : while (index <= end_index) {
192 4 : folio = filemap_get_folio(mapping, index);
193 4 : filemap_dirty_folio(mapping, folio);
194 4 : folio_account_redirty(folio);
195 4 : index += folio_nr_pages(folio);
196 4 : folio_put(folio);
197 : }
198 4 : }
199 :
200 : /*
201 : * Process one page for __process_pages_contig().
202 : *
203 : * Return >0 if we hit @page == @locked_page.
204 : * Return 0 if we updated the page status.
205 : * Return -EGAIN if the we need to try again.
206 : * (For PAGE_LOCK case but got dirty page or page not belong to mapping)
207 : */
208 121483876 : static int process_one_page(struct btrfs_fs_info *fs_info,
209 : struct address_space *mapping,
210 : struct page *page, struct page *locked_page,
211 : unsigned long page_ops, u64 start, u64 end)
212 : {
213 121483876 : u32 len;
214 :
215 121483876 : ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
216 121483876 : len = end + 1 - start;
217 :
218 121483876 : if (page_ops & PAGE_SET_ORDERED)
219 55858032 : btrfs_page_clamp_set_ordered(fs_info, page, start, len);
220 121484978 : if (page_ops & PAGE_START_WRITEBACK) {
221 6155757 : btrfs_page_clamp_clear_dirty(fs_info, page, start, len);
222 6155840 : btrfs_page_clamp_set_writeback(fs_info, page, start, len);
223 : }
224 121485028 : if (page_ops & PAGE_END_WRITEBACK)
225 1094602 : btrfs_page_clamp_clear_writeback(fs_info, page, start, len);
226 :
227 121485077 : if (page == locked_page)
228 : return 1;
229 :
230 116882971 : if (page_ops & PAGE_LOCK) {
231 58444963 : int ret;
232 :
233 58444963 : ret = btrfs_page_start_writer_lock(fs_info, page, start, len);
234 58445315 : if (ret)
235 : return ret;
236 58445242 : if (!PageDirty(page) || page->mapping != mapping) {
237 39 : btrfs_page_end_writer_lock(fs_info, page, start, len);
238 39 : return -EAGAIN;
239 : }
240 : }
241 116883230 : if (page_ops & PAGE_UNLOCK)
242 58449252 : btrfs_page_end_writer_lock(fs_info, page, start, len);
243 : return 0;
244 : }
245 :
246 4853829 : static int __process_pages_contig(struct address_space *mapping,
247 : struct page *locked_page,
248 : u64 start, u64 end, unsigned long page_ops,
249 : u64 *processed_end)
250 : {
251 4853829 : struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb);
252 4853829 : pgoff_t start_index = start >> PAGE_SHIFT;
253 4853829 : pgoff_t end_index = end >> PAGE_SHIFT;
254 4853829 : pgoff_t index = start_index;
255 4853829 : unsigned long pages_processed = 0;
256 4853829 : struct folio_batch fbatch;
257 4853829 : int err = 0;
258 4853829 : int i;
259 :
260 4853829 : if (page_ops & PAGE_LOCK) {
261 : ASSERT(page_ops == PAGE_LOCK);
262 : ASSERT(processed_end && *processed_end == start);
263 : }
264 :
265 4853829 : folio_batch_init(&fbatch);
266 16869680 : while (index <= end_index) {
267 12015851 : int found_folios;
268 :
269 12015851 : found_folios = filemap_get_folios_contig(mapping, &index,
270 : end_index, &fbatch);
271 :
272 12015664 : if (found_folios == 0) {
273 : /*
274 : * Only if we're going to lock these pages, we can find
275 : * nothing at @index.
276 : */
277 1 : ASSERT(page_ops & PAGE_LOCK);
278 1 : err = -EAGAIN;
279 1 : goto out;
280 : }
281 :
282 133501622 : for (i = 0; i < found_folios; i++) {
283 121485848 : int process_ret;
284 121485848 : struct folio *folio = fbatch.folios[i];
285 121485848 : process_ret = process_one_page(fs_info, mapping,
286 : &folio->page, locked_page, page_ops,
287 : start, end);
288 121485998 : if (process_ret < 0) {
289 39 : err = -EAGAIN;
290 39 : folio_batch_release(&fbatch);
291 39 : goto out;
292 : }
293 121485959 : pages_processed += folio_nr_pages(folio);
294 : }
295 12015774 : folio_batch_release(&fbatch);
296 12015854 : cond_resched();
297 : }
298 4853829 : out:
299 4853869 : if (err && processed_end) {
300 : /*
301 : * Update @processed_end. I know this is awful since it has
302 : * two different return value patterns (inclusive vs exclusive).
303 : *
304 : * But the exclusive pattern is necessary if @start is 0, or we
305 : * underflow and check against processed_end won't work as
306 : * expected.
307 : */
308 40 : if (pages_processed)
309 40 : *processed_end = min(end,
310 : ((u64)(start_index + pages_processed) << PAGE_SHIFT) - 1);
311 : else
312 0 : *processed_end = start;
313 : }
314 4853869 : return err;
315 : }
316 :
317 40 : static noinline void __unlock_for_delalloc(struct inode *inode,
318 : struct page *locked_page,
319 : u64 start, u64 end)
320 : {
321 40 : unsigned long index = start >> PAGE_SHIFT;
322 40 : unsigned long end_index = end >> PAGE_SHIFT;
323 :
324 40 : ASSERT(locked_page);
325 40 : if (index == locked_page->index && end_index == index)
326 : return;
327 :
328 2 : __process_pages_contig(inode->i_mapping, locked_page, start, end,
329 : PAGE_UNLOCK, NULL);
330 : }
331 :
332 3566355 : static noinline int lock_delalloc_pages(struct inode *inode,
333 : struct page *locked_page,
334 : u64 delalloc_start,
335 : u64 delalloc_end)
336 : {
337 3566355 : unsigned long index = delalloc_start >> PAGE_SHIFT;
338 3566355 : unsigned long end_index = delalloc_end >> PAGE_SHIFT;
339 3566355 : u64 processed_end = delalloc_start;
340 3566355 : int ret;
341 :
342 3566355 : ASSERT(locked_page);
343 3566355 : if (index == locked_page->index && index == end_index)
344 : return 0;
345 :
346 1036116 : ret = __process_pages_contig(inode->i_mapping, locked_page, delalloc_start,
347 : delalloc_end, PAGE_LOCK, &processed_end);
348 1036082 : if (ret == -EAGAIN && processed_end > delalloc_start)
349 40 : __unlock_for_delalloc(inode, locked_page, delalloc_start,
350 : processed_end);
351 : return ret;
352 : }
353 :
354 : /*
355 : * Find and lock a contiguous range of bytes in the file marked as delalloc, no
356 : * more than @max_bytes.
357 : *
358 : * @start: The original start bytenr to search.
359 : * Will store the extent range start bytenr.
360 : * @end: The original end bytenr of the search range
361 : * Will store the extent range end bytenr.
362 : *
363 : * Return true if we find a delalloc range which starts inside the original
364 : * range, and @start/@end will store the delalloc range start/end.
365 : *
366 : * Return false if we can't find any delalloc range which starts inside the
367 : * original range, and @start/@end will be the non-delalloc range start/end.
368 : */
369 : EXPORT_FOR_TESTS
370 56933276 : noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
371 : struct page *locked_page, u64 *start,
372 : u64 *end)
373 : {
374 56933276 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
375 56933276 : struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
376 56933276 : const u64 orig_start = *start;
377 56933276 : const u64 orig_end = *end;
378 : /* The sanity tests may not set a valid fs_info. */
379 56933276 : u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
380 56933276 : u64 delalloc_start;
381 56933276 : u64 delalloc_end;
382 56933276 : bool found;
383 56933276 : struct extent_state *cached_state = NULL;
384 56933276 : int ret;
385 56933276 : int loops = 0;
386 :
387 : /* Caller should pass a valid @end to indicate the search range end */
388 56933276 : ASSERT(orig_end > orig_start);
389 :
390 : /* The range should at least cover part of the page */
391 56933276 : ASSERT(!(orig_start >= page_offset(locked_page) + PAGE_SIZE ||
392 : orig_end <= page_offset(locked_page)));
393 : again:
394 : /* step one, find a bunch of delalloc bytes starting at start */
395 56933316 : delalloc_start = *start;
396 56933316 : delalloc_end = 0;
397 56933316 : found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
398 : max_bytes, &cached_state);
399 56933830 : if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
400 53367389 : *start = delalloc_start;
401 :
402 : /* @delalloc_end can be -1, never go beyond @orig_end */
403 53367389 : *end = min(delalloc_end, orig_end);
404 53367389 : free_extent_state(cached_state);
405 53367389 : return false;
406 : }
407 :
408 : /*
409 : * start comes from the offset of locked_page. We have to lock
410 : * pages in order, so we can't process delalloc bytes before
411 : * locked_page
412 : */
413 3566441 : if (delalloc_start < *start)
414 20847 : delalloc_start = *start;
415 :
416 : /*
417 : * make sure to limit the number of pages we try to lock down
418 : */
419 3566441 : if (delalloc_end + 1 - delalloc_start > max_bytes)
420 676 : delalloc_end = delalloc_start + max_bytes - 1;
421 :
422 : /* step two, lock all the pages after the page that has start */
423 3566441 : ret = lock_delalloc_pages(inode, locked_page,
424 : delalloc_start, delalloc_end);
425 3566238 : ASSERT(!ret || ret == -EAGAIN);
426 3566238 : if (ret == -EAGAIN) {
427 : /* some of the pages are gone, lets avoid looping by
428 : * shortening the size of the delalloc range we're searching
429 : */
430 40 : free_extent_state(cached_state);
431 40 : cached_state = NULL;
432 40 : if (!loops) {
433 40 : max_bytes = PAGE_SIZE;
434 40 : loops = 1;
435 40 : goto again;
436 : } else {
437 0 : found = false;
438 0 : goto out_failed;
439 : }
440 : }
441 :
442 : /* step three, lock the state bits for the whole range */
443 3566198 : lock_extent(tree, delalloc_start, delalloc_end, &cached_state);
444 :
445 : /* then test to make sure it is all still delalloc */
446 3566452 : ret = test_range_bit(tree, delalloc_start, delalloc_end,
447 : EXTENT_DELALLOC, 1, cached_state);
448 3566188 : if (!ret) {
449 0 : unlock_extent(tree, delalloc_start, delalloc_end,
450 : &cached_state);
451 0 : __unlock_for_delalloc(inode, locked_page,
452 : delalloc_start, delalloc_end);
453 0 : cond_resched();
454 0 : goto again;
455 : }
456 3566188 : free_extent_state(cached_state);
457 3566497 : *start = delalloc_start;
458 3566497 : *end = delalloc_end;
459 : out_failed:
460 : return found;
461 : }
462 :
463 3817867 : void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
464 : struct page *locked_page,
465 : u32 clear_bits, unsigned long page_ops)
466 : {
467 3817867 : clear_extent_bit(&inode->io_tree, start, end, clear_bits, NULL);
468 :
469 3817859 : __process_pages_contig(inode->vfs_inode.i_mapping, locked_page,
470 : start, end, page_ops, NULL);
471 3817822 : }
472 :
473 : static bool btrfs_verify_page(struct page *page, u64 start)
474 : {
475 : if (!fsverity_active(page->mapping->host) ||
476 : PageUptodate(page) ||
477 : start >= i_size_read(page->mapping->host))
478 : return true;
479 : return fsverity_verify_page(page);
480 : }
481 :
482 59450849 : static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len)
483 : {
484 59450849 : struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
485 :
486 59450849 : ASSERT(page_offset(page) <= start &&
487 : start + len <= page_offset(page) + PAGE_SIZE);
488 :
489 59450849 : if (uptodate && btrfs_verify_page(page, start))
490 59450847 : btrfs_page_set_uptodate(fs_info, page, start, len);
491 : else
492 2 : btrfs_page_clear_uptodate(fs_info, page, start, len);
493 :
494 59459035 : if (!btrfs_is_subpage(fs_info, page))
495 59456671 : unlock_page(page);
496 : else
497 0 : btrfs_subpage_end_reader(fs_info, page, start, len);
498 59456187 : }
499 :
500 : /* lots and lots of room for performance fixes in the end_bio funcs */
501 :
502 0 : void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
503 : {
504 0 : struct btrfs_inode *inode;
505 0 : const bool uptodate = (err == 0);
506 0 : int ret = 0;
507 :
508 0 : ASSERT(page && page->mapping);
509 0 : inode = BTRFS_I(page->mapping->host);
510 0 : btrfs_writepage_endio_finish_ordered(inode, page, start, end, uptodate);
511 :
512 0 : if (!uptodate) {
513 0 : const struct btrfs_fs_info *fs_info = inode->root->fs_info;
514 0 : u32 len;
515 :
516 0 : ASSERT(end + 1 - start <= U32_MAX);
517 0 : len = end + 1 - start;
518 :
519 0 : btrfs_page_clear_uptodate(fs_info, page, start, len);
520 0 : ret = err < 0 ? err : -EIO;
521 0 : mapping_set_error(page->mapping, ret);
522 : }
523 0 : }
524 :
525 : /*
526 : * after a writepage IO is done, we need to:
527 : * clear the uptodate bits on error
528 : * clear the writeback bits in the extent tree for this IO
529 : * end_page_writeback if the page has no more pending IO
530 : *
531 : * Scheduling is not allowed, so the extent state tree is expected
532 : * to have one and only one object corresponding to this IO.
533 : */
534 2626778 : static void end_bio_extent_writepage(struct btrfs_bio *bbio)
535 : {
536 2626778 : struct bio *bio = &bbio->bio;
537 2626778 : int error = blk_status_to_errno(bio->bi_status);
538 2626778 : struct bio_vec *bvec;
539 2626778 : struct bvec_iter_all iter_all;
540 :
541 2626778 : ASSERT(!bio_flagged(bio, BIO_CLONED));
542 58469906 : bio_for_each_segment_all(bvec, bio, iter_all) {
543 55843128 : struct page *page = bvec->bv_page;
544 55843128 : struct inode *inode = page->mapping->host;
545 55843128 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
546 55843128 : const u32 sectorsize = fs_info->sectorsize;
547 55843128 : u64 start = page_offset(page) + bvec->bv_offset;
548 55843128 : u32 len = bvec->bv_len;
549 :
550 : /* Our read/write should always be sector aligned. */
551 55843128 : if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
552 0 : btrfs_err(fs_info,
553 : "partial page write in btrfs with offset %u and length %u",
554 : bvec->bv_offset, bvec->bv_len);
555 55843128 : else if (!IS_ALIGNED(bvec->bv_len, sectorsize))
556 0 : btrfs_info(fs_info,
557 : "incomplete page write with offset %u and length %u",
558 : bvec->bv_offset, bvec->bv_len);
559 :
560 55843128 : btrfs_finish_ordered_extent(bbio->ordered, page, start, len, !error);
561 55843128 : if (error) {
562 129535 : btrfs_page_clear_uptodate(fs_info, page, start, len);
563 129535 : mapping_set_error(page->mapping, error);
564 : }
565 55843128 : btrfs_page_clear_writeback(fs_info, page, start, len);
566 : }
567 :
568 2626778 : bio_put(bio);
569 2626778 : }
570 :
571 : /*
572 : * Record previously processed extent range
573 : *
574 : * For endio_readpage_release_extent() to handle a full extent range, reducing
575 : * the extent io operations.
576 : */
577 : struct processed_extent {
578 : struct btrfs_inode *inode;
579 : /* Start of the range in @inode */
580 : u64 start;
581 : /* End of the range in @inode */
582 : u64 end;
583 : bool uptodate;
584 : };
585 :
586 : /*
587 : * Try to release processed extent range
588 : *
589 : * May not release the extent range right now if the current range is
590 : * contiguous to processed extent.
591 : *
592 : * Will release processed extent when any of @inode, @uptodate, the range is
593 : * no longer contiguous to the processed range.
594 : *
595 : * Passing @inode == NULL will force processed extent to be released.
596 : */
597 21020401 : static void endio_readpage_release_extent(struct processed_extent *processed,
598 : struct btrfs_inode *inode, u64 start, u64 end,
599 : bool uptodate)
600 : {
601 21020401 : struct extent_state *cached = NULL;
602 21020401 : struct extent_io_tree *tree;
603 :
604 : /* The first extent, initialize @processed */
605 21020401 : if (!processed->inode)
606 1737214 : goto update;
607 :
608 : /*
609 : * Contiguous to processed extent, just uptodate the end.
610 : *
611 : * Several things to notice:
612 : *
613 : * - bio can be merged as long as on-disk bytenr is contiguous
614 : * This means we can have page belonging to other inodes, thus need to
615 : * check if the inode still matches.
616 : * - bvec can contain range beyond current page for multi-page bvec
617 : * Thus we need to do processed->end + 1 >= start check
618 : */
619 19283187 : if (processed->inode == inode && processed->uptodate == uptodate &&
620 17546583 : processed->end + 1 >= start && end >= processed->end) {
621 17546198 : processed->end = end;
622 17546198 : return;
623 : }
624 :
625 1736989 : tree = &processed->inode->io_tree;
626 : /*
627 : * Now we don't have range contiguous to the processed range, release
628 : * the processed range now.
629 : */
630 1736989 : unlock_extent(tree, processed->start, processed->end, &cached);
631 :
632 3474337 : update:
633 : /* Update processed to current range */
634 3474337 : processed->inode = inode;
635 3474337 : processed->start = start;
636 3474337 : processed->end = end;
637 3474337 : processed->uptodate = uptodate;
638 : }
639 :
640 59438219 : static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page)
641 : {
642 59438219 : ASSERT(PageLocked(page));
643 59422239 : if (!btrfs_is_subpage(fs_info, page))
644 : return;
645 :
646 0 : ASSERT(PagePrivate(page));
647 0 : btrfs_subpage_start_reader(fs_info, page, page_offset(page), PAGE_SIZE);
648 : }
649 :
650 : /*
651 : * after a readpage IO is done, we need to:
652 : * clear the uptodate bits on error
653 : * set the uptodate bits if things worked
654 : * set the page up to date if all extents in the tree are uptodate
655 : * clear the lock bit in the extent tree
656 : * unlock the page if there are no other extents locked for it
657 : *
658 : * Scheduling is not allowed, so the extent state tree is expected
659 : * to have one and only one object corresponding to this IO.
660 : */
661 1737200 : static void end_bio_extent_readpage(struct btrfs_bio *bbio)
662 : {
663 1737200 : struct bio *bio = &bbio->bio;
664 1737200 : struct bio_vec *bvec;
665 1737200 : struct processed_extent processed = { 0 };
666 : /*
667 : * The offset to the beginning of a bio, since one bio can never be
668 : * larger than UINT_MAX, u32 here is enough.
669 : */
670 1737200 : u32 bio_offset = 0;
671 1737200 : struct bvec_iter_all iter_all;
672 :
673 1737200 : ASSERT(!bio_flagged(bio, BIO_CLONED));
674 21019286 : bio_for_each_segment_all(bvec, bio, iter_all) {
675 19277956 : bool uptodate = !bio->bi_status;
676 19277956 : struct page *page = bvec->bv_page;
677 19277956 : struct inode *inode = page->mapping->host;
678 19277956 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
679 19277956 : const u32 sectorsize = fs_info->sectorsize;
680 19277956 : u64 start;
681 19277956 : u64 end;
682 19277956 : u32 len;
683 :
684 19277956 : btrfs_debug(fs_info,
685 : "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
686 : bio->bi_iter.bi_sector, bio->bi_status,
687 : bbio->mirror_num);
688 :
689 : /*
690 : * We always issue full-sector reads, but if some block in a
691 : * page fails to read, blk_update_request() will advance
692 : * bv_offset and adjust bv_len to compensate. Print a warning
693 : * for unaligned offsets, and an error if they don't add up to
694 : * a full sector.
695 : */
696 19277956 : if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
697 0 : btrfs_err(fs_info,
698 : "partial page read in btrfs with offset %u and length %u",
699 : bvec->bv_offset, bvec->bv_len);
700 19277956 : else if (!IS_ALIGNED(bvec->bv_offset + bvec->bv_len,
701 : sectorsize))
702 0 : btrfs_info(fs_info,
703 : "incomplete page read with offset %u and length %u",
704 : bvec->bv_offset, bvec->bv_len);
705 :
706 19277956 : start = page_offset(page) + bvec->bv_offset;
707 19277956 : end = start + bvec->bv_len - 1;
708 19277956 : len = bvec->bv_len;
709 :
710 19277956 : if (likely(uptodate)) {
711 19277954 : loff_t i_size = i_size_read(inode);
712 19277954 : pgoff_t end_index = i_size >> PAGE_SHIFT;
713 :
714 : /*
715 : * Zero out the remaining part if this range straddles
716 : * i_size.
717 : *
718 : * Here we should only zero the range inside the bvec,
719 : * not touch anything else.
720 : *
721 : * NOTE: i_size is exclusive while end is inclusive.
722 : */
723 19277954 : if (page->index == end_index && i_size <= end) {
724 22584 : u32 zero_start = max(offset_in_page(i_size),
725 : offset_in_page(start));
726 :
727 22584 : zero_user_segment(page, zero_start,
728 22584 : offset_in_page(end) + 1);
729 : }
730 : }
731 :
732 : /* Update page status and unlock. */
733 19277955 : end_page_read(page, uptodate, start, len);
734 19282724 : endio_readpage_release_extent(&processed, BTRFS_I(inode),
735 : start, end, uptodate);
736 :
737 19282724 : ASSERT(bio_offset + len > bio_offset);
738 19282724 : bio_offset += len;
739 :
740 : }
741 : /* Release the last extent */
742 1737222 : endio_readpage_release_extent(&processed, NULL, 0, 0, false);
743 1737174 : bio_put(bio);
744 1737221 : }
745 :
746 : /*
747 : * Populate every free slot in a provided array with pages.
748 : *
749 : * @nr_pages: number of pages to allocate
750 : * @page_array: the array to fill with pages; any existing non-null entries in
751 : * the array will be skipped
752 : *
753 : * Return: 0 if all pages were able to be allocated;
754 : * -ENOMEM otherwise, and the caller is responsible for freeing all
755 : * non-null page pointers in the array.
756 : */
757 19353997 : int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array)
758 : {
759 19353997 : unsigned int allocated;
760 :
761 19353714 : for (allocated = 0; allocated < nr_pages;) {
762 19353714 : unsigned int last = allocated;
763 :
764 19353714 : allocated = alloc_pages_bulk_array(GFP_NOFS, nr_pages, page_array);
765 :
766 19354070 : if (allocated == nr_pages)
767 : return 0;
768 :
769 : /*
770 : * During this iteration, no page could be allocated, even
771 : * though alloc_pages_bulk_array() falls back to alloc_page()
772 : * if it could not bulk-allocate. So we must be out of memory.
773 : */
774 34 : if (allocated == last)
775 : return -ENOMEM;
776 :
777 34 : memalloc_retry_wait(GFP_NOFS);
778 : }
779 : return 0;
780 : }
781 :
782 71041943 : static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
783 : struct page *page, u64 disk_bytenr,
784 : unsigned int pg_offset)
785 : {
786 71041943 : struct bio *bio = &bio_ctrl->bbio->bio;
787 71041943 : struct bio_vec *bvec = bio_last_bvec_all(bio);
788 71041943 : const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
789 :
790 71041943 : if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
791 : /*
792 : * For compression, all IO should have its logical bytenr set
793 : * to the starting bytenr of the compressed extent.
794 : */
795 33114 : return bio->bi_iter.bi_sector == sector;
796 : }
797 :
798 : /*
799 : * The contig check requires the following conditions to be met:
800 : *
801 : * 1) The pages are belonging to the same inode
802 : * This is implied by the call chain.
803 : *
804 : * 2) The range has adjacent logical bytenr
805 : *
806 : * 3) The range has adjacent file offset
807 : * This is required for the usage of btrfs_bio->file_offset.
808 : */
809 141790593 : return bio_end_sector(bio) == sector &&
810 70781764 : page_offset(bvec->bv_page) + bvec->bv_offset + bvec->bv_len ==
811 70781764 : page_offset(page) + pg_offset;
812 : }
813 :
814 4362042 : static void alloc_new_bio(struct btrfs_inode *inode,
815 : struct btrfs_bio_ctrl *bio_ctrl,
816 : u64 disk_bytenr, u64 file_offset)
817 : {
818 4362042 : struct btrfs_fs_info *fs_info = inode->root->fs_info;
819 4362042 : struct btrfs_bio *bbio;
820 :
821 4362042 : bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, fs_info,
822 : bio_ctrl->end_io_func, NULL);
823 4363522 : bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
824 4363522 : bbio->inode = inode;
825 4363522 : bbio->file_offset = file_offset;
826 4363522 : bio_ctrl->bbio = bbio;
827 4363522 : bio_ctrl->len_to_oe_boundary = U32_MAX;
828 :
829 : /* Limit data write bios to the ordered boundary. */
830 4363522 : if (bio_ctrl->wbc) {
831 2626710 : struct btrfs_ordered_extent *ordered;
832 :
833 2626710 : ordered = btrfs_lookup_ordered_extent(inode, file_offset);
834 2626746 : if (ordered) {
835 2626746 : bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
836 : ordered->file_offset +
837 : ordered->disk_num_bytes - file_offset);
838 2626746 : bbio->ordered = ordered;
839 : }
840 :
841 : /*
842 : * Pick the last added device to support cgroup writeback. For
843 : * multi-device file systems this means blk-cgroup policies have
844 : * to always be set on the last added/replaced device.
845 : * This is a bit odd but has been like that for a long time.
846 : */
847 2626746 : bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
848 2626655 : wbc_init_bio(bio_ctrl->wbc, &bbio->bio);
849 : }
850 4363501 : }
851 :
852 : /*
853 : * @disk_bytenr: logical bytenr where the write will be
854 : * @page: page to add to the bio
855 : * @size: portion of page that we want to write to
856 : * @pg_offset: offset of the new bio or to check whether we are adding
857 : * a contiguous page to the previous one
858 : *
859 : * The will either add the page into the existing @bio_ctrl->bbio, or allocate a
860 : * new one in @bio_ctrl->bbio.
861 : * The mirror number for this IO should already be initizlied in
862 : * @bio_ctrl->mirror_num.
863 : */
864 75119992 : static void submit_extent_page(struct btrfs_bio_ctrl *bio_ctrl,
865 : u64 disk_bytenr, struct page *page,
866 : size_t size, unsigned long pg_offset)
867 : {
868 75119992 : struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
869 :
870 75119992 : ASSERT(pg_offset + size <= PAGE_SIZE);
871 75119992 : ASSERT(bio_ctrl->end_io_func);
872 :
873 146163528 : if (bio_ctrl->bbio &&
874 71042700 : !btrfs_bio_is_contig(bio_ctrl, page, disk_bytenr, pg_offset))
875 240664 : submit_one_bio(bio_ctrl);
876 :
877 75165069 : do {
878 75165069 : u32 len = size;
879 :
880 : /* Allocate new bio if needed */
881 75165069 : if (!bio_ctrl->bbio) {
882 4361973 : alloc_new_bio(inode, bio_ctrl, disk_bytenr,
883 4361973 : page_offset(page) + pg_offset);
884 : }
885 :
886 : /* Cap to the current ordered extent boundary if there is one. */
887 75166624 : if (len > bio_ctrl->len_to_oe_boundary) {
888 : ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
889 : ASSERT(is_data_inode(&inode->vfs_inode));
890 : len = bio_ctrl->len_to_oe_boundary;
891 : }
892 :
893 75166624 : if (bio_add_page(&bio_ctrl->bbio->bio, page, len, pg_offset) != len) {
894 : /* bio full: move on to a new one */
895 44664 : submit_one_bio(bio_ctrl);
896 44664 : continue;
897 : }
898 :
899 75123668 : if (bio_ctrl->wbc)
900 55836377 : wbc_account_cgroup_owner(bio_ctrl->wbc, page, len);
901 :
902 75114754 : size -= len;
903 75114754 : pg_offset += len;
904 75114754 : disk_bytenr += len;
905 75114754 : bio_ctrl->len_to_oe_boundary -= len;
906 :
907 : /* Ordered extent boundary: move on to a new bio. */
908 75114754 : if (bio_ctrl->len_to_oe_boundary == 0)
909 2564428 : submit_one_bio(bio_ctrl);
910 75159607 : } while (size);
911 75115366 : }
912 :
913 122397510 : static int attach_extent_buffer_page(struct extent_buffer *eb,
914 : struct page *page,
915 : struct btrfs_subpage *prealloc)
916 : {
917 122397510 : struct btrfs_fs_info *fs_info = eb->fs_info;
918 122397510 : int ret = 0;
919 :
920 : /*
921 : * If the page is mapped to btree inode, we should hold the private
922 : * lock to prevent race.
923 : * For cloned or dummy extent buffers, their pages are not mapped and
924 : * will not race with any other ebs.
925 : */
926 122397510 : if (page->mapping)
927 122397510 : lockdep_assert_held(&page->mapping->private_lock);
928 :
929 122397510 : if (fs_info->nodesize >= PAGE_SIZE) {
930 122397510 : if (!PagePrivate(page))
931 122397510 : attach_page_private(page, eb);
932 : else
933 0 : WARN_ON(page->private != (unsigned long)eb);
934 122401551 : return 0;
935 : }
936 :
937 : /* Already mapped, just free prealloc */
938 0 : if (PagePrivate(page)) {
939 0 : btrfs_free_subpage(prealloc);
940 0 : return 0;
941 : }
942 :
943 0 : if (prealloc)
944 : /* Has preallocated memory for subpage */
945 0 : attach_page_private(page, prealloc);
946 : else
947 : /* Do new allocation to attach subpage */
948 0 : ret = btrfs_attach_subpage(fs_info, page,
949 : BTRFS_SUBPAGE_METADATA);
950 : return ret;
951 : }
952 :
953 203506173 : int set_page_extent_mapped(struct page *page)
954 : {
955 203506173 : struct btrfs_fs_info *fs_info;
956 :
957 203506173 : ASSERT(page->mapping);
958 :
959 203506173 : if (PagePrivate(page))
960 : return 0;
961 :
962 109062025 : fs_info = btrfs_sb(page->mapping->host->i_sb);
963 :
964 109062025 : if (btrfs_is_subpage(fs_info, page))
965 0 : return btrfs_attach_subpage(fs_info, page, BTRFS_SUBPAGE_DATA);
966 :
967 109049634 : attach_page_private(page, (void *)EXTENT_PAGE_PRIVATE);
968 109049634 : return 0;
969 : }
970 :
971 142298184 : void clear_page_extent_mapped(struct page *page)
972 : {
973 142298184 : struct btrfs_fs_info *fs_info;
974 :
975 142298184 : ASSERT(page->mapping);
976 :
977 142298184 : if (!PagePrivate(page))
978 : return;
979 :
980 109111984 : fs_info = btrfs_sb(page->mapping->host->i_sb);
981 109111984 : if (btrfs_is_subpage(fs_info, page))
982 0 : return btrfs_detach_subpage(fs_info, page);
983 :
984 109112177 : detach_page_private(page);
985 : }
986 :
987 : static struct extent_map *
988 56755990 : __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
989 : u64 start, u64 len, struct extent_map **em_cached)
990 : {
991 56755990 : struct extent_map *em;
992 :
993 56755990 : if (em_cached && *em_cached) {
994 43875052 : em = *em_cached;
995 87749678 : if (extent_map_in_tree(em) && start >= em->start &&
996 : start < extent_map_end(em)) {
997 38195830 : refcount_inc(&em->refs);
998 38195830 : return em;
999 : }
1000 :
1001 5679222 : free_extent_map(em);
1002 5679433 : *em_cached = NULL;
1003 : }
1004 :
1005 18560371 : em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, start, len);
1006 18560396 : if (em_cached && !IS_ERR(em)) {
1007 7140648 : BUG_ON(*em_cached);
1008 7140648 : refcount_inc(&em->refs);
1009 7140695 : *em_cached = em;
1010 : }
1011 : return em;
1012 : }
1013 : /*
1014 : * basic readpage implementation. Locked extent state structs are inserted
1015 : * into the tree that are removed when the IO is done (by the end_io
1016 : * handlers)
1017 : * XXX JDM: This needs looking at to ensure proper page locking
1018 : * return 0 on success, otherwise return error
1019 : */
1020 59449283 : static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
1021 : struct btrfs_bio_ctrl *bio_ctrl, u64 *prev_em_start)
1022 : {
1023 59449283 : struct inode *inode = page->mapping->host;
1024 59449283 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1025 59449283 : u64 start = page_offset(page);
1026 59449283 : const u64 end = start + PAGE_SIZE - 1;
1027 59449283 : u64 cur = start;
1028 59449283 : u64 extent_offset;
1029 59449283 : u64 last_byte = i_size_read(inode);
1030 59449283 : u64 block_start;
1031 59449283 : struct extent_map *em;
1032 59449283 : int ret = 0;
1033 59449283 : size_t pg_offset = 0;
1034 59449283 : size_t iosize;
1035 59449283 : size_t blocksize = inode->i_sb->s_blocksize;
1036 59449283 : struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
1037 :
1038 59449283 : ret = set_page_extent_mapped(page);
1039 59441895 : if (ret < 0) {
1040 0 : unlock_extent(tree, start, end, NULL);
1041 0 : unlock_page(page);
1042 0 : return ret;
1043 : }
1044 :
1045 59441895 : if (page->index == last_byte >> PAGE_SHIFT) {
1046 1380165 : size_t zero_offset = offset_in_page(last_byte);
1047 :
1048 1380165 : if (zero_offset) {
1049 185288 : iosize = PAGE_SIZE - zero_offset;
1050 185288 : memzero_page(page, zero_offset, iosize);
1051 : }
1052 : }
1053 59441897 : bio_ctrl->end_io_func = end_bio_extent_readpage;
1054 59441897 : begin_page_read(fs_info, page);
1055 116206181 : while (cur <= end) {
1056 59427721 : enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
1057 59427721 : bool force_bio_submit = false;
1058 59427721 : u64 disk_bytenr;
1059 :
1060 59427721 : ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1061 59427721 : if (cur >= last_byte) {
1062 2672756 : iosize = PAGE_SIZE - pg_offset;
1063 2672756 : memzero_page(page, pg_offset, iosize);
1064 2696128 : unlock_extent(tree, cur, cur + iosize - 1, NULL);
1065 2695622 : end_page_read(page, true, cur, iosize);
1066 2695622 : break;
1067 : }
1068 56754965 : em = __get_extent_map(inode, page, pg_offset, cur,
1069 56754965 : end - cur + 1, em_cached);
1070 56761491 : if (IS_ERR(em)) {
1071 0 : unlock_extent(tree, cur, end, NULL);
1072 0 : end_page_read(page, false, cur, end + 1 - cur);
1073 0 : return PTR_ERR(em);
1074 : }
1075 56761491 : extent_offset = cur - em->start;
1076 56761491 : BUG_ON(extent_map_end(em) <= cur);
1077 56761491 : BUG_ON(end < cur);
1078 :
1079 113522982 : if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
1080 34292 : compress_type = em->compress_type;
1081 :
1082 56761491 : iosize = min(extent_map_end(em) - cur, end - cur + 1);
1083 56761491 : iosize = ALIGN(iosize, blocksize);
1084 56761491 : if (compress_type != BTRFS_COMPRESS_NONE)
1085 34292 : disk_bytenr = em->block_start;
1086 : else
1087 56727199 : disk_bytenr = em->block_start + extent_offset;
1088 56761491 : block_start = em->block_start;
1089 113522982 : if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
1090 5050613 : block_start = EXTENT_MAP_HOLE;
1091 :
1092 : /*
1093 : * If we have a file range that points to a compressed extent
1094 : * and it's followed by a consecutive file range that points
1095 : * to the same compressed extent (possibly with a different
1096 : * offset and/or length, so it either points to the whole extent
1097 : * or only part of it), we must make sure we do not submit a
1098 : * single bio to populate the pages for the 2 ranges because
1099 : * this makes the compressed extent read zero out the pages
1100 : * belonging to the 2nd range. Imagine the following scenario:
1101 : *
1102 : * File layout
1103 : * [0 - 8K] [8K - 24K]
1104 : * | |
1105 : * | |
1106 : * points to extent X, points to extent X,
1107 : * offset 4K, length of 8K offset 0, length 16K
1108 : *
1109 : * [extent X, compressed length = 4K uncompressed length = 16K]
1110 : *
1111 : * If the bio to read the compressed extent covers both ranges,
1112 : * it will decompress extent X into the pages belonging to the
1113 : * first range and then it will stop, zeroing out the remaining
1114 : * pages that belong to the other range that points to extent X.
1115 : * So here we make sure we submit 2 bios, one for the first
1116 : * range and another one for the third range. Both will target
1117 : * the same physical extent from disk, but we can't currently
1118 : * make the compressed bio endio callback populate the pages
1119 : * for both ranges because each compressed bio is tightly
1120 : * coupled with a single extent map, and each range can have
1121 : * an extent map with a different offset value relative to the
1122 : * uncompressed data of our extent and different lengths. This
1123 : * is a corner case so we prioritize correctness over
1124 : * non-optimal behavior (submitting 2 bios for the same extent).
1125 : */
1126 113522982 : if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
1127 34289 : prev_em_start && *prev_em_start != (u64)-1 &&
1128 : *prev_em_start != em->start)
1129 1090 : force_bio_submit = true;
1130 :
1131 56761491 : if (prev_em_start)
1132 45341363 : *prev_em_start = em->start;
1133 :
1134 56761491 : free_extent_map(em);
1135 56766290 : em = NULL;
1136 :
1137 : /* we've found a hole, just zero and go on */
1138 56766290 : if (block_start == EXTENT_MAP_HOLE) {
1139 37451197 : memzero_page(page, pg_offset, iosize);
1140 :
1141 37454071 : unlock_extent(tree, cur, cur + iosize - 1, NULL);
1142 37452971 : end_page_read(page, true, cur, iosize);
1143 37450800 : cur = cur + iosize;
1144 37450800 : pg_offset += iosize;
1145 37450800 : continue;
1146 : }
1147 : /* the get_extent function already copied into the page */
1148 19315093 : if (block_start == EXTENT_MAP_INLINE) {
1149 26205 : unlock_extent(tree, cur, cur + iosize - 1, NULL);
1150 26205 : end_page_read(page, true, cur, iosize);
1151 26204 : cur = cur + iosize;
1152 26204 : pg_offset += iosize;
1153 26204 : continue;
1154 : }
1155 :
1156 19288888 : if (bio_ctrl->compress_type != compress_type) {
1157 148 : submit_one_bio(bio_ctrl);
1158 148 : bio_ctrl->compress_type = compress_type;
1159 : }
1160 :
1161 19288888 : if (force_bio_submit)
1162 1090 : submit_one_bio(bio_ctrl);
1163 19288888 : submit_extent_page(bio_ctrl, disk_bytenr, page, iosize,
1164 : pg_offset);
1165 19287280 : cur = cur + iosize;
1166 19287280 : pg_offset += iosize;
1167 : }
1168 :
1169 : return 0;
1170 : }
1171 :
1172 14112595 : int btrfs_read_folio(struct file *file, struct folio *folio)
1173 : {
1174 14112595 : struct page *page = &folio->page;
1175 14112595 : struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
1176 14112595 : u64 start = page_offset(page);
1177 14112595 : u64 end = start + PAGE_SIZE - 1;
1178 14112595 : struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ };
1179 14112595 : int ret;
1180 :
1181 14112595 : btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1182 :
1183 14113229 : ret = btrfs_do_readpage(page, NULL, &bio_ctrl, NULL);
1184 : /*
1185 : * If btrfs_do_readpage() failed we will want to submit the assembled
1186 : * bio to do the cleanup.
1187 : */
1188 14116384 : submit_one_bio(&bio_ctrl);
1189 14114292 : return ret;
1190 : }
1191 :
1192 3382074 : static inline void contiguous_readpages(struct page *pages[], int nr_pages,
1193 : u64 start, u64 end,
1194 : struct extent_map **em_cached,
1195 : struct btrfs_bio_ctrl *bio_ctrl,
1196 : u64 *prev_em_start)
1197 : {
1198 3382074 : struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host);
1199 3382074 : int index;
1200 :
1201 3382074 : btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1202 :
1203 52109071 : for (index = 0; index < nr_pages; index++) {
1204 45344893 : btrfs_do_readpage(pages[index], em_cached, bio_ctrl,
1205 : prev_em_start);
1206 45343790 : put_page(pages[index]);
1207 : }
1208 3382104 : }
1209 :
1210 : /*
1211 : * helper for __extent_writepage, doing all of the delayed allocation setup.
1212 : *
1213 : * This returns 1 if btrfs_run_delalloc_range function did all the work required
1214 : * to write the page (copy into inline extent). In this case the IO has
1215 : * been started and the page is already unlocked.
1216 : *
1217 : * This returns 0 if all went well (page still locked)
1218 : * This returns < 0 if there were errors (page still locked)
1219 : */
1220 56928536 : static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1221 : struct page *page, struct writeback_control *wbc)
1222 : {
1223 56928536 : const u64 page_end = page_offset(page) + PAGE_SIZE - 1;
1224 56928536 : u64 delalloc_start = page_offset(page);
1225 56928536 : u64 delalloc_to_write = 0;
1226 : /* How many pages are started by btrfs_run_delalloc_range() */
1227 56928536 : unsigned long nr_written = 0;
1228 56928536 : int ret;
1229 56928536 : int page_started = 0;
1230 :
1231 113856340 : while (delalloc_start < page_end) {
1232 56928465 : u64 delalloc_end = page_end;
1233 56928465 : bool found;
1234 :
1235 56928465 : found = find_lock_delalloc_range(&inode->vfs_inode, page,
1236 : &delalloc_start,
1237 : &delalloc_end);
1238 56927630 : if (!found) {
1239 53361274 : delalloc_start = delalloc_end + 1;
1240 53361274 : continue;
1241 : }
1242 3566356 : ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
1243 : delalloc_end, &page_started, &nr_written, wbc);
1244 3566530 : if (ret)
1245 0 : return ret;
1246 :
1247 : /*
1248 : * delalloc_end is already one less than the total length, so
1249 : * we don't subtract one from PAGE_SIZE
1250 : */
1251 3566530 : delalloc_to_write += (delalloc_end - delalloc_start +
1252 3566530 : PAGE_SIZE) >> PAGE_SHIFT;
1253 3566530 : delalloc_start = delalloc_end + 1;
1254 : }
1255 56927875 : if (wbc->nr_to_write < delalloc_to_write) {
1256 4761 : int thresh = 8192;
1257 :
1258 4761 : if (delalloc_to_write < thresh * 2)
1259 4378 : thresh = delalloc_to_write;
1260 4761 : wbc->nr_to_write = min_t(u64, delalloc_to_write,
1261 : thresh);
1262 : }
1263 :
1264 : /* Did btrfs_run_dealloc_range() already unlock and start the IO? */
1265 56927875 : if (page_started) {
1266 : /*
1267 : * We've unlocked the page, so we can't update the mapping's
1268 : * writeback index, just update nr_to_write.
1269 : */
1270 1095303 : wbc->nr_to_write -= nr_written;
1271 1095303 : return 1;
1272 : }
1273 :
1274 : return 0;
1275 : }
1276 :
1277 : /*
1278 : * Find the first byte we need to write.
1279 : *
1280 : * For subpage, one page can contain several sectors, and
1281 : * __extent_writepage_io() will just grab all extent maps in the page
1282 : * range and try to submit all non-inline/non-compressed extents.
1283 : *
1284 : * This is a big problem for subpage, we shouldn't re-submit already written
1285 : * data at all.
1286 : * This function will lookup subpage dirty bit to find which range we really
1287 : * need to submit.
1288 : *
1289 : * Return the next dirty range in [@start, @end).
1290 : * If no dirty range is found, @start will be page_offset(page) + PAGE_SIZE.
1291 : */
1292 55828295 : static void find_next_dirty_byte(struct btrfs_fs_info *fs_info,
1293 : struct page *page, u64 *start, u64 *end)
1294 : {
1295 55828295 : struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
1296 55828295 : struct btrfs_subpage_info *spi = fs_info->subpage_info;
1297 55828295 : u64 orig_start = *start;
1298 : /* Declare as unsigned long so we can use bitmap ops */
1299 55828295 : unsigned long flags;
1300 55828295 : int range_start_bit;
1301 55828295 : int range_end_bit;
1302 :
1303 : /*
1304 : * For regular sector size == page size case, since one page only
1305 : * contains one sector, we return the page offset directly.
1306 : */
1307 55828295 : if (!btrfs_is_subpage(fs_info, page)) {
1308 55826833 : *start = page_offset(page);
1309 55826833 : *end = page_offset(page) + PAGE_SIZE;
1310 55826833 : return;
1311 : }
1312 :
1313 0 : range_start_bit = spi->dirty_offset +
1314 0 : (offset_in_page(orig_start) >> fs_info->sectorsize_bits);
1315 :
1316 : /* We should have the page locked, but just in case */
1317 0 : spin_lock_irqsave(&subpage->lock, flags);
1318 0 : bitmap_next_set_region(subpage->bitmaps, &range_start_bit, &range_end_bit,
1319 0 : spi->dirty_offset + spi->bitmap_nr_bits);
1320 0 : spin_unlock_irqrestore(&subpage->lock, flags);
1321 :
1322 0 : range_start_bit -= spi->dirty_offset;
1323 0 : range_end_bit -= spi->dirty_offset;
1324 :
1325 0 : *start = page_offset(page) + range_start_bit * fs_info->sectorsize;
1326 0 : *end = page_offset(page) + range_end_bit * fs_info->sectorsize;
1327 : }
1328 :
1329 : /*
1330 : * helper for __extent_writepage. This calls the writepage start hooks,
1331 : * and does the loop to map the page into extents and bios.
1332 : *
1333 : * We return 1 if the IO is started and the page is unlocked,
1334 : * 0 if all went well (page still locked)
1335 : * < 0 if there were errors (page still locked)
1336 : */
1337 55833609 : static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode,
1338 : struct page *page,
1339 : struct btrfs_bio_ctrl *bio_ctrl,
1340 : loff_t i_size,
1341 : int *nr_ret)
1342 : {
1343 55833609 : struct btrfs_fs_info *fs_info = inode->root->fs_info;
1344 55833609 : u64 cur = page_offset(page);
1345 55833609 : u64 end = cur + PAGE_SIZE - 1;
1346 55833609 : u64 extent_offset;
1347 55833609 : u64 block_start;
1348 55833609 : struct extent_map *em;
1349 55833609 : int ret = 0;
1350 55833609 : int nr = 0;
1351 :
1352 55833609 : ret = btrfs_writepage_cow_fixup(page);
1353 55830094 : if (ret) {
1354 : /* Fixup worker will requeue */
1355 0 : redirty_page_for_writepage(bio_ctrl->wbc, page);
1356 0 : unlock_page(page);
1357 0 : return 1;
1358 : }
1359 :
1360 55830094 : bio_ctrl->end_io_func = end_bio_extent_writepage;
1361 111659129 : while (cur <= end) {
1362 55828997 : u64 disk_bytenr;
1363 55828997 : u64 em_end;
1364 55828997 : u64 dirty_range_start = cur;
1365 55828997 : u64 dirty_range_end;
1366 55828997 : u32 iosize;
1367 :
1368 55828997 : if (cur >= i_size) {
1369 0 : btrfs_writepage_endio_finish_ordered(inode, page, cur,
1370 : end, true);
1371 : /*
1372 : * This range is beyond i_size, thus we don't need to
1373 : * bother writing back.
1374 : * But we still need to clear the dirty subpage bit, or
1375 : * the next time the page gets dirtied, we will try to
1376 : * writeback the sectors with subpage dirty bits,
1377 : * causing writeback without ordered extent.
1378 : */
1379 0 : btrfs_page_clear_dirty(fs_info, page, cur, end + 1 - cur);
1380 0 : break;
1381 : }
1382 :
1383 55828997 : find_next_dirty_byte(fs_info, page, &dirty_range_start,
1384 : &dirty_range_end);
1385 55827383 : if (cur < dirty_range_start) {
1386 0 : cur = dirty_range_start;
1387 0 : continue;
1388 : }
1389 :
1390 55827383 : em = btrfs_get_extent(inode, NULL, 0, cur, end - cur + 1);
1391 55833204 : if (IS_ERR(em)) {
1392 0 : ret = PTR_ERR_OR_ZERO(em);
1393 0 : goto out_error;
1394 : }
1395 :
1396 55833204 : extent_offset = cur - em->start;
1397 55833204 : em_end = extent_map_end(em);
1398 55833204 : ASSERT(cur <= em_end);
1399 55833204 : ASSERT(cur < end);
1400 55833204 : ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize));
1401 55833204 : ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize));
1402 :
1403 55833204 : block_start = em->block_start;
1404 55833204 : disk_bytenr = em->block_start + extent_offset;
1405 :
1406 55833204 : ASSERT(!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags));
1407 55833204 : ASSERT(block_start != EXTENT_MAP_HOLE);
1408 55833204 : ASSERT(block_start != EXTENT_MAP_INLINE);
1409 :
1410 : /*
1411 : * Note that em_end from extent_map_end() and dirty_range_end from
1412 : * find_next_dirty_byte() are all exclusive
1413 : */
1414 55833204 : iosize = min(min(em_end, end + 1), dirty_range_end) - cur;
1415 55833204 : free_extent_map(em);
1416 55837165 : em = NULL;
1417 :
1418 55837165 : btrfs_set_range_writeback(inode, cur, cur + iosize - 1);
1419 55837186 : if (!PageWriteback(page)) {
1420 0 : btrfs_err(inode->root->fs_info,
1421 : "page %lu not writeback, cur %llu end %llu",
1422 : page->index, cur, end);
1423 : }
1424 :
1425 : /*
1426 : * Although the PageDirty bit is cleared before entering this
1427 : * function, subpage dirty bit is not cleared.
1428 : * So clear subpage dirty bit here so next time we won't submit
1429 : * page for range already written to disk.
1430 : */
1431 55835587 : btrfs_page_clear_dirty(fs_info, page, cur, iosize);
1432 :
1433 55833818 : submit_extent_page(bio_ctrl, disk_bytenr, page, iosize,
1434 55833818 : cur - page_offset(page));
1435 55829035 : cur += iosize;
1436 55829035 : nr++;
1437 : }
1438 :
1439 55830132 : btrfs_page_assert_not_dirty(fs_info, page);
1440 55830169 : *nr_ret = nr;
1441 55830169 : return 0;
1442 :
1443 : out_error:
1444 : /*
1445 : * If we finish without problem, we should not only clear page dirty,
1446 : * but also empty subpage dirty bits
1447 : */
1448 0 : *nr_ret = nr;
1449 0 : return ret;
1450 : }
1451 :
1452 : /*
1453 : * the writepage semantics are similar to regular writepage. extent
1454 : * records are inserted to lock ranges in the tree, and as dirty areas
1455 : * are found, they are marked writeback. Then the lock bits are removed
1456 : * and the end_io handler clears the writeback ranges
1457 : *
1458 : * Return 0 if everything goes well.
1459 : * Return <0 for error.
1460 : */
1461 56928193 : static int __extent_writepage(struct page *page, struct btrfs_bio_ctrl *bio_ctrl)
1462 : {
1463 56928193 : struct folio *folio = page_folio(page);
1464 56927881 : struct inode *inode = page->mapping->host;
1465 56927881 : const u64 page_start = page_offset(page);
1466 56927881 : const u64 page_end = page_start + PAGE_SIZE - 1;
1467 56927881 : int ret;
1468 56927881 : int nr = 0;
1469 56927881 : size_t pg_offset;
1470 56927881 : loff_t i_size = i_size_read(inode);
1471 56927881 : unsigned long end_index = i_size >> PAGE_SHIFT;
1472 :
1473 56927881 : trace___extent_writepage(page, inode, bio_ctrl->wbc);
1474 :
1475 56926488 : WARN_ON(!PageLocked(page));
1476 :
1477 56926658 : pg_offset = offset_in_page(i_size);
1478 56926658 : if (page->index > end_index ||
1479 1966087 : (page->index == end_index && !pg_offset)) {
1480 401 : folio_invalidate(folio, 0, folio_size(folio));
1481 401 : folio_unlock(folio);
1482 401 : return 0;
1483 : }
1484 :
1485 56926257 : if (page->index == end_index)
1486 1966115 : memzero_page(page, pg_offset, PAGE_SIZE - pg_offset);
1487 :
1488 56926295 : ret = set_page_extent_mapped(page);
1489 56925026 : if (ret < 0)
1490 0 : goto done;
1491 :
1492 56925026 : ret = writepage_delalloc(BTRFS_I(inode), page, bio_ctrl->wbc);
1493 56928105 : if (ret == 1)
1494 : return 0;
1495 55832789 : if (ret)
1496 0 : goto done;
1497 :
1498 55832789 : ret = __extent_writepage_io(BTRFS_I(inode), page, bio_ctrl, i_size, &nr);
1499 55830031 : if (ret == 1)
1500 : return 0;
1501 :
1502 55830031 : bio_ctrl->wbc->nr_to_write--;
1503 :
1504 55830031 : done:
1505 55830031 : if (nr == 0) {
1506 : /* make sure the mapping tag for page dirty gets cleared */
1507 0 : set_page_writeback(page);
1508 0 : end_page_writeback(page);
1509 : }
1510 55830031 : if (ret)
1511 0 : end_extent_writepage(page, ret, page_start, page_end);
1512 55830031 : unlock_page(page);
1513 55830031 : ASSERT(ret <= 0);
1514 55830031 : return ret;
1515 : }
1516 :
1517 30956 : void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
1518 : {
1519 30956 : wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
1520 : TASK_UNINTERRUPTIBLE);
1521 30956 : }
1522 :
1523 : /*
1524 : * Lock extent buffer status and pages for writeback.
1525 : *
1526 : * Return %false if the extent buffer doesn't need to be submitted (e.g. the
1527 : * extent buffer is not dirty)
1528 : * Return %true is the extent buffer is submitted to bio.
1529 : */
1530 9312487 : static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
1531 : struct writeback_control *wbc)
1532 : {
1533 9312487 : struct btrfs_fs_info *fs_info = eb->fs_info;
1534 9312487 : bool ret = false;
1535 :
1536 9312487 : btrfs_tree_lock(eb);
1537 18629149 : while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
1538 2546 : btrfs_tree_unlock(eb);
1539 2547 : if (wbc->sync_mode != WB_SYNC_ALL)
1540 : return false;
1541 2095 : wait_on_extent_buffer_writeback(eb);
1542 2095 : btrfs_tree_lock(eb);
1543 : }
1544 :
1545 : /*
1546 : * We need to do this to prevent races in people who check if the eb is
1547 : * under IO since we can end up having no IO bits set for a short period
1548 : * of time.
1549 : */
1550 9312021 : spin_lock(&eb->refs_lock);
1551 9312028 : if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
1552 9306703 : set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1553 9306705 : spin_unlock(&eb->refs_lock);
1554 9306707 : btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1555 9306704 : percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1556 9306704 : -eb->len,
1557 : fs_info->dirty_metadata_batch);
1558 9306704 : ret = true;
1559 : } else {
1560 5327 : spin_unlock(&eb->refs_lock);
1561 : }
1562 9312033 : btrfs_tree_unlock(eb);
1563 9312033 : return ret;
1564 : }
1565 :
1566 72 : static void set_btree_ioerr(struct extent_buffer *eb)
1567 : {
1568 72 : struct btrfs_fs_info *fs_info = eb->fs_info;
1569 :
1570 72 : set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1571 :
1572 : /*
1573 : * A read may stumble upon this buffer later, make sure that it gets an
1574 : * error and knows there was an error.
1575 : */
1576 72 : clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
1577 :
1578 : /*
1579 : * We need to set the mapping with the io error as well because a write
1580 : * error will flip the file system readonly, and then syncfs() will
1581 : * return a 0 because we are readonly if we don't modify the err seq for
1582 : * the superblock.
1583 : */
1584 72 : mapping_set_error(eb->fs_info->btree_inode->i_mapping, -EIO);
1585 :
1586 : /*
1587 : * If writeback for a btree extent that doesn't belong to a log tree
1588 : * failed, increment the counter transaction->eb_write_errors.
1589 : * We do this because while the transaction is running and before it's
1590 : * committing (when we call filemap_fdata[write|wait]_range against
1591 : * the btree inode), we might have
1592 : * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1593 : * returns an error or an error happens during writeback, when we're
1594 : * committing the transaction we wouldn't know about it, since the pages
1595 : * can be no longer dirty nor marked anymore for writeback (if a
1596 : * subsequent modification to the extent buffer didn't happen before the
1597 : * transaction commit), which makes filemap_fdata[write|wait]_range not
1598 : * able to find the pages tagged with SetPageError at transaction
1599 : * commit time. So if this happens we must abort the transaction,
1600 : * otherwise we commit a super block with btree roots that point to
1601 : * btree nodes/leafs whose content on disk is invalid - either garbage
1602 : * or the content of some node/leaf from a past generation that got
1603 : * cowed or deleted and is no longer valid.
1604 : *
1605 : * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1606 : * not be enough - we need to distinguish between log tree extents vs
1607 : * non-log tree extents, and the next filemap_fdatawait_range() call
1608 : * will catch and clear such errors in the mapping - and that call might
1609 : * be from a log sync and not from a transaction commit. Also, checking
1610 : * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1611 : * not done and would not be reliable - the eb might have been released
1612 : * from memory and reading it back again means that flag would not be
1613 : * set (since it's a runtime flag, not persisted on disk).
1614 : *
1615 : * Using the flags below in the btree inode also makes us achieve the
1616 : * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
1617 : * writeback for all dirty pages and before filemap_fdatawait_range()
1618 : * is called, the writeback for all dirty pages had already finished
1619 : * with errors - because we were not using AS_EIO/AS_ENOSPC,
1620 : * filemap_fdatawait_range() would return success, as it could not know
1621 : * that writeback errors happened (the pages were no longer tagged for
1622 : * writeback).
1623 : */
1624 72 : switch (eb->log_index) {
1625 68 : case -1:
1626 68 : set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags);
1627 : break;
1628 4 : case 0:
1629 4 : set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags);
1630 : break;
1631 0 : case 1:
1632 0 : set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags);
1633 : break;
1634 0 : default:
1635 0 : BUG(); /* unexpected, logic error */
1636 : }
1637 72 : }
1638 :
1639 : /*
1640 : * The endio specific version which won't touch any unsafe spinlock in endio
1641 : * context.
1642 : */
1643 597176447 : static struct extent_buffer *find_extent_buffer_nolock(
1644 : struct btrfs_fs_info *fs_info, u64 start)
1645 : {
1646 597176447 : struct extent_buffer *eb;
1647 :
1648 597176447 : rcu_read_lock();
1649 597123747 : eb = radix_tree_lookup(&fs_info->buffer_radix,
1650 597123747 : start >> fs_info->sectorsize_bits);
1651 1182046418 : if (eb && atomic_inc_not_zero(&eb->refs)) {
1652 584885795 : rcu_read_unlock();
1653 584885795 : return eb;
1654 : }
1655 13086315 : rcu_read_unlock();
1656 13086315 : return NULL;
1657 : }
1658 :
1659 9306709 : static void extent_buffer_write_end_io(struct btrfs_bio *bbio)
1660 : {
1661 9306709 : struct extent_buffer *eb = bbio->private;
1662 9306709 : struct btrfs_fs_info *fs_info = eb->fs_info;
1663 9306709 : bool uptodate = !bbio->bio.bi_status;
1664 9306709 : struct bvec_iter_all iter_all;
1665 9306709 : struct bio_vec *bvec;
1666 9306709 : u32 bio_offset = 0;
1667 :
1668 9306709 : if (!uptodate)
1669 72 : set_btree_ioerr(eb);
1670 :
1671 45504176 : bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
1672 36197467 : u64 start = eb->start + bio_offset;
1673 36197467 : struct page *page = bvec->bv_page;
1674 36197467 : u32 len = bvec->bv_len;
1675 :
1676 36197467 : if (!uptodate)
1677 288 : btrfs_page_clear_uptodate(fs_info, page, start, len);
1678 36197467 : btrfs_page_clear_writeback(fs_info, page, start, len);
1679 36197467 : bio_offset += len;
1680 : }
1681 :
1682 9306709 : clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1683 9306709 : smp_mb__after_atomic();
1684 9306709 : wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
1685 :
1686 9306709 : bio_put(&bbio->bio);
1687 9306709 : }
1688 :
1689 9306688 : static void prepare_eb_write(struct extent_buffer *eb)
1690 : {
1691 9306688 : u32 nritems;
1692 9306688 : unsigned long start;
1693 9306688 : unsigned long end;
1694 :
1695 9306688 : clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1696 :
1697 : /* Set btree blocks beyond nritems with 0 to avoid stale content */
1698 9306699 : nritems = btrfs_header_nritems(eb);
1699 9306699 : if (btrfs_header_level(eb) > 0) {
1700 880239 : end = btrfs_node_key_ptr_offset(eb, nritems);
1701 880239 : memzero_extent_buffer(eb, end, eb->len - end);
1702 : } else {
1703 : /*
1704 : * Leaf:
1705 : * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
1706 : */
1707 8426460 : start = btrfs_item_nr_offset(eb, nritems);
1708 8426460 : end = btrfs_item_nr_offset(eb, 0);
1709 8426460 : if (nritems == 0)
1710 234 : end += BTRFS_LEAF_DATA_SIZE(eb->fs_info);
1711 : else
1712 8426226 : end += btrfs_item_offset(eb, nritems - 1);
1713 8426445 : memzero_extent_buffer(eb, start, end - start);
1714 : }
1715 9306676 : }
1716 :
1717 9306694 : static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
1718 : struct writeback_control *wbc)
1719 : {
1720 9306694 : struct btrfs_fs_info *fs_info = eb->fs_info;
1721 9306694 : struct btrfs_bio *bbio;
1722 :
1723 9306694 : prepare_eb_write(eb);
1724 :
1725 9661741 : bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
1726 : REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
1727 : eb->fs_info, extent_buffer_write_end_io, eb);
1728 9306676 : bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
1729 9306676 : bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
1730 9306667 : wbc_init_bio(wbc, &bbio->bio);
1731 9306677 : bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
1732 9306677 : bbio->file_offset = eb->start;
1733 9306677 : if (fs_info->nodesize < PAGE_SIZE) {
1734 0 : struct page *p = eb->pages[0];
1735 :
1736 0 : lock_page(p);
1737 0 : btrfs_subpage_set_writeback(fs_info, p, eb->start, eb->len);
1738 0 : if (btrfs_subpage_clear_and_test_dirty(fs_info, p, eb->start,
1739 0 : eb->len)) {
1740 0 : clear_page_dirty_for_io(p);
1741 0 : wbc->nr_to_write--;
1742 : }
1743 0 : __bio_add_page(&bbio->bio, p, eb->len, eb->start - page_offset(p));
1744 0 : wbc_account_cgroup_owner(wbc, p, eb->len);
1745 0 : unlock_page(p);
1746 : } else {
1747 45504118 : for (int i = 0; i < num_extent_pages(eb); i++) {
1748 36197409 : struct page *p = eb->pages[i];
1749 :
1750 36197409 : lock_page(p);
1751 36197440 : clear_page_dirty_for_io(p);
1752 36197413 : set_page_writeback(p);
1753 36197463 : __bio_add_page(&bbio->bio, p, PAGE_SIZE, 0);
1754 36197432 : wbc_account_cgroup_owner(wbc, p, PAGE_SIZE);
1755 36197414 : wbc->nr_to_write--;
1756 36197414 : unlock_page(p);
1757 : }
1758 : }
1759 9306709 : btrfs_submit_bio(bbio, 0);
1760 9306708 : }
1761 :
1762 : /*
1763 : * Submit one subpage btree page.
1764 : *
1765 : * The main difference to submit_eb_page() is:
1766 : * - Page locking
1767 : * For subpage, we don't rely on page locking at all.
1768 : *
1769 : * - Flush write bio
1770 : * We only flush bio if we may be unable to fit current extent buffers into
1771 : * current bio.
1772 : *
1773 : * Return >=0 for the number of submitted extent buffers.
1774 : * Return <0 for fatal error.
1775 : */
1776 0 : static int submit_eb_subpage(struct page *page, struct writeback_control *wbc)
1777 : {
1778 0 : struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
1779 0 : int submitted = 0;
1780 0 : u64 page_start = page_offset(page);
1781 0 : int bit_start = 0;
1782 0 : int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits;
1783 :
1784 : /* Lock and write each dirty extent buffers in the range */
1785 0 : while (bit_start < fs_info->subpage_info->bitmap_nr_bits) {
1786 0 : struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
1787 0 : struct extent_buffer *eb;
1788 0 : unsigned long flags;
1789 0 : u64 start;
1790 :
1791 : /*
1792 : * Take private lock to ensure the subpage won't be detached
1793 : * in the meantime.
1794 : */
1795 0 : spin_lock(&page->mapping->private_lock);
1796 0 : if (!PagePrivate(page)) {
1797 0 : spin_unlock(&page->mapping->private_lock);
1798 : break;
1799 : }
1800 0 : spin_lock_irqsave(&subpage->lock, flags);
1801 0 : if (!test_bit(bit_start + fs_info->subpage_info->dirty_offset,
1802 : subpage->bitmaps)) {
1803 0 : spin_unlock_irqrestore(&subpage->lock, flags);
1804 0 : spin_unlock(&page->mapping->private_lock);
1805 0 : bit_start++;
1806 0 : continue;
1807 : }
1808 :
1809 0 : start = page_start + bit_start * fs_info->sectorsize;
1810 0 : bit_start += sectors_per_node;
1811 :
1812 : /*
1813 : * Here we just want to grab the eb without touching extra
1814 : * spin locks, so call find_extent_buffer_nolock().
1815 : */
1816 0 : eb = find_extent_buffer_nolock(fs_info, start);
1817 0 : spin_unlock_irqrestore(&subpage->lock, flags);
1818 0 : spin_unlock(&page->mapping->private_lock);
1819 :
1820 : /*
1821 : * The eb has already reached 0 refs thus find_extent_buffer()
1822 : * doesn't return it. We don't need to write back such eb
1823 : * anyway.
1824 : */
1825 0 : if (!eb)
1826 0 : continue;
1827 :
1828 0 : if (lock_extent_buffer_for_io(eb, wbc)) {
1829 0 : write_one_eb(eb, wbc);
1830 0 : submitted++;
1831 : }
1832 0 : free_extent_buffer(eb);
1833 : }
1834 0 : return submitted;
1835 : }
1836 :
1837 : /*
1838 : * Submit all page(s) of one extent buffer.
1839 : *
1840 : * @page: the page of one extent buffer
1841 : * @eb_context: to determine if we need to submit this page, if current page
1842 : * belongs to this eb, we don't need to submit
1843 : *
1844 : * The caller should pass each page in their bytenr order, and here we use
1845 : * @eb_context to determine if we have submitted pages of one extent buffer.
1846 : *
1847 : * If we have, we just skip until we hit a new page that doesn't belong to
1848 : * current @eb_context.
1849 : *
1850 : * If not, we submit all the page(s) of the extent buffer.
1851 : *
1852 : * Return >0 if we have submitted the extent buffer successfully.
1853 : * Return 0 if we don't need to submit the page, as it's already submitted by
1854 : * previous call.
1855 : * Return <0 for fatal error.
1856 : */
1857 34653517 : static int submit_eb_page(struct page *page, struct writeback_control *wbc,
1858 : struct extent_buffer **eb_context)
1859 : {
1860 34653517 : struct address_space *mapping = page->mapping;
1861 34653517 : struct btrfs_block_group *cache = NULL;
1862 34653517 : struct extent_buffer *eb;
1863 34653517 : int ret;
1864 :
1865 34653517 : if (!PagePrivate(page))
1866 : return 0;
1867 :
1868 34610990 : if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
1869 0 : return submit_eb_subpage(page, wbc);
1870 :
1871 34610990 : spin_lock(&mapping->private_lock);
1872 34611045 : if (!PagePrivate(page)) {
1873 0 : spin_unlock(&mapping->private_lock);
1874 0 : return 0;
1875 : }
1876 :
1877 34611045 : eb = (struct extent_buffer *)page->private;
1878 :
1879 : /*
1880 : * Shouldn't happen and normally this would be a BUG_ON but no point
1881 : * crashing the machine for something we can survive anyway.
1882 : */
1883 34611045 : if (WARN_ON(!eb)) {
1884 0 : spin_unlock(&mapping->private_lock);
1885 0 : return 0;
1886 : }
1887 :
1888 34611045 : if (eb == *eb_context) {
1889 25298557 : spin_unlock(&mapping->private_lock);
1890 25298557 : return 0;
1891 : }
1892 9312488 : ret = atomic_inc_not_zero(&eb->refs);
1893 9312488 : spin_unlock(&mapping->private_lock);
1894 9312488 : if (!ret)
1895 : return 0;
1896 :
1897 9312488 : if (!btrfs_check_meta_write_pointer(eb->fs_info, eb, &cache)) {
1898 : /*
1899 : * If for_sync, this hole will be filled with
1900 : * trasnsaction commit.
1901 : */
1902 : if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
1903 : ret = -EAGAIN;
1904 : else
1905 : ret = 0;
1906 : free_extent_buffer(eb);
1907 : return ret;
1908 : }
1909 :
1910 9312488 : *eb_context = eb;
1911 :
1912 9312488 : if (!lock_extent_buffer_for_io(eb, wbc)) {
1913 5779 : btrfs_revert_meta_write_pointer(cache, eb);
1914 5779 : if (cache)
1915 : btrfs_put_block_group(cache);
1916 5779 : free_extent_buffer(eb);
1917 5779 : return 0;
1918 : }
1919 9306702 : if (cache) {
1920 : /*
1921 : * Implies write in zoned mode. Mark the last eb in a block group.
1922 : */
1923 : btrfs_schedule_zone_finish_bg(cache, eb);
1924 : btrfs_put_block_group(cache);
1925 : }
1926 9306702 : write_one_eb(eb, wbc);
1927 9306708 : free_extent_buffer(eb);
1928 9306708 : return 1;
1929 : }
1930 :
1931 2320253 : int btree_write_cache_pages(struct address_space *mapping,
1932 : struct writeback_control *wbc)
1933 : {
1934 2320253 : struct extent_buffer *eb_context = NULL;
1935 2320253 : struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
1936 2320253 : int ret = 0;
1937 2320253 : int done = 0;
1938 2320253 : int nr_to_write_done = 0;
1939 2320253 : struct folio_batch fbatch;
1940 2320253 : unsigned int nr_folios;
1941 2320253 : pgoff_t index;
1942 2320253 : pgoff_t end; /* Inclusive */
1943 2320253 : int scanned = 0;
1944 2320253 : xa_mark_t tag;
1945 :
1946 2320253 : folio_batch_init(&fbatch);
1947 2320253 : if (wbc->range_cyclic) {
1948 63 : index = mapping->writeback_index; /* Start from prev offset */
1949 63 : end = -1;
1950 : /*
1951 : * Start from the beginning does not need to cycle over the
1952 : * range, mark it as scanned.
1953 : */
1954 63 : scanned = (index == 0);
1955 : } else {
1956 2320190 : index = wbc->range_start >> PAGE_SHIFT;
1957 2320190 : end = wbc->range_end >> PAGE_SHIFT;
1958 2320190 : scanned = 1;
1959 : }
1960 2320253 : if (wbc->sync_mode == WB_SYNC_ALL)
1961 : tag = PAGECACHE_TAG_TOWRITE;
1962 : else
1963 71 : tag = PAGECACHE_TAG_DIRTY;
1964 2320253 : btrfs_zoned_meta_io_lock(fs_info);
1965 2320253 : retry:
1966 2320253 : if (wbc->sync_mode == WB_SYNC_ALL)
1967 2320162 : tag_pages_for_writeback(mapping, index, end);
1968 9898337 : while (!done && !nr_to_write_done && (index <= end) &&
1969 3932538 : (nr_folios = filemap_get_folios_tag(mapping, &index, end,
1970 : tag, &fbatch))) {
1971 : unsigned i;
1972 :
1973 38299083 : for (i = 0; i < nr_folios; i++) {
1974 34653559 : struct folio *folio = fbatch.folios[i];
1975 :
1976 34653559 : ret = submit_eb_page(&folio->page, wbc, &eb_context);
1977 34653564 : if (ret == 0)
1978 25346857 : continue;
1979 9306707 : if (ret < 0) {
1980 : done = 1;
1981 : break;
1982 : }
1983 :
1984 : /*
1985 : * the filesystem may choose to bump up nr_to_write.
1986 : * We have to make sure to honor the new nr_to_write
1987 : * at any time
1988 : */
1989 9306707 : nr_to_write_done = wbc->nr_to_write <= 0;
1990 : }
1991 3645524 : folio_batch_release(&fbatch);
1992 3645525 : cond_resched();
1993 : }
1994 2320280 : if (!scanned && !done) {
1995 : /*
1996 : * We hit the last page and there is more work to be done: wrap
1997 : * back to the start of the file
1998 : */
1999 0 : scanned = 1;
2000 0 : index = 0;
2001 0 : goto retry;
2002 : }
2003 : /*
2004 : * If something went wrong, don't allow any metadata write bio to be
2005 : * submitted.
2006 : *
2007 : * This would prevent use-after-free if we had dirty pages not
2008 : * cleaned up, which can still happen by fuzzed images.
2009 : *
2010 : * - Bad extent tree
2011 : * Allowing existing tree block to be allocated for other trees.
2012 : *
2013 : * - Log tree operations
2014 : * Exiting tree blocks get allocated to log tree, bumps its
2015 : * generation, then get cleaned in tree re-balance.
2016 : * Such tree block will not be written back, since it's clean,
2017 : * thus no WRITTEN flag set.
2018 : * And after log writes back, this tree block is not traced by
2019 : * any dirty extent_io_tree.
2020 : *
2021 : * - Offending tree block gets re-dirtied from its original owner
2022 : * Since it has bumped generation, no WRITTEN flag, it can be
2023 : * reused without COWing. This tree block will not be traced
2024 : * by btrfs_transaction::dirty_pages.
2025 : *
2026 : * Now such dirty tree block will not be cleaned by any dirty
2027 : * extent io tree. Thus we don't want to submit such wild eb
2028 : * if the fs already has error.
2029 : *
2030 : * We can get ret > 0 from submit_extent_page() indicating how many ebs
2031 : * were submitted. Reset it to 0 to avoid false alerts for the caller.
2032 : */
2033 2320280 : if (ret > 0)
2034 : ret = 0;
2035 2320280 : if (!ret && BTRFS_FS_ERROR(fs_info))
2036 15 : ret = -EROFS;
2037 2320280 : btrfs_zoned_meta_io_unlock(fs_info);
2038 2320280 : return ret;
2039 : }
2040 :
2041 : /*
2042 : * Walk the list of dirty pages of the given address space and write all of them.
2043 : *
2044 : * @mapping: address space structure to write
2045 : * @wbc: subtract the number of written pages from *@wbc->nr_to_write
2046 : * @bio_ctrl: holds context for the write, namely the bio
2047 : *
2048 : * If a page is already under I/O, write_cache_pages() skips it, even
2049 : * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
2050 : * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
2051 : * and msync() need to guarantee that all the data which was dirty at the time
2052 : * the call was made get new I/O started against them. If wbc->sync_mode is
2053 : * WB_SYNC_ALL then we were called for data integrity and we must wait for
2054 : * existing IO to complete.
2055 : */
2056 5051100 : static int extent_write_cache_pages(struct address_space *mapping,
2057 : struct btrfs_bio_ctrl *bio_ctrl)
2058 : {
2059 5051100 : struct writeback_control *wbc = bio_ctrl->wbc;
2060 5051100 : struct inode *inode = mapping->host;
2061 5051100 : int ret = 0;
2062 5051100 : int done = 0;
2063 5051100 : int nr_to_write_done = 0;
2064 5051100 : struct folio_batch fbatch;
2065 5051100 : unsigned int nr_folios;
2066 5051100 : pgoff_t index;
2067 5051100 : pgoff_t end; /* Inclusive */
2068 5051100 : pgoff_t done_index;
2069 5051100 : int range_whole = 0;
2070 5051100 : int scanned = 0;
2071 5051100 : xa_mark_t tag;
2072 :
2073 : /*
2074 : * We have to hold onto the inode so that ordered extents can do their
2075 : * work when the IO finishes. The alternative to this is failing to add
2076 : * an ordered extent if the igrab() fails there and that is a huge pain
2077 : * to deal with, so instead just hold onto the inode throughout the
2078 : * writepages operation. If it fails here we are freeing up the inode
2079 : * anyway and we'd rather not waste our time writing out stuff that is
2080 : * going to be truncated anyway.
2081 : */
2082 5051100 : if (!igrab(inode))
2083 : return 0;
2084 :
2085 5051117 : folio_batch_init(&fbatch);
2086 5051117 : if (wbc->range_cyclic) {
2087 429840 : index = mapping->writeback_index; /* Start from prev offset */
2088 429840 : end = -1;
2089 : /*
2090 : * Start from the beginning does not need to cycle over the
2091 : * range, mark it as scanned.
2092 : */
2093 429840 : scanned = (index == 0);
2094 : } else {
2095 4621277 : index = wbc->range_start >> PAGE_SHIFT;
2096 4621277 : end = wbc->range_end >> PAGE_SHIFT;
2097 4621277 : if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2098 2214790 : range_whole = 1;
2099 : scanned = 1;
2100 : }
2101 :
2102 : /*
2103 : * We do the tagged writepage as long as the snapshot flush bit is set
2104 : * and we are the first one who do the filemap_flush() on this inode.
2105 : *
2106 : * The nr_to_write == LONG_MAX is needed to make sure other flushers do
2107 : * not race in and drop the bit.
2108 : */
2109 5725070 : if (range_whole && wbc->nr_to_write == LONG_MAX &&
2110 : test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
2111 673902 : &BTRFS_I(inode)->runtime_flags))
2112 20126 : wbc->tagged_writepages = 1;
2113 :
2114 5051168 : if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2115 : tag = PAGECACHE_TAG_TOWRITE;
2116 : else
2117 2270287 : tag = PAGECACHE_TAG_DIRTY;
2118 5051168 : retry:
2119 5428254 : if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2120 2780704 : tag_pages_for_writeback(mapping, index, end);
2121 5428355 : done_index = index;
2122 20903816 : while (!done && !nr_to_write_done && (index <= end) &&
2123 9015914 : (nr_folios = filemap_get_folios_tag(mapping, &index,
2124 : end, tag, &fbatch))) {
2125 : unsigned i;
2126 :
2127 65318792 : for (i = 0; i < nr_folios; i++) {
2128 58858770 : struct folio *folio = fbatch.folios[i];
2129 :
2130 58858770 : done_index = folio->index + folio_nr_pages(folio);
2131 : /*
2132 : * At this point we hold neither the i_pages lock nor
2133 : * the page lock: the page may be truncated or
2134 : * invalidated (changing page->mapping to NULL),
2135 : * or even swizzled back from swapper_space to
2136 : * tmpfs file mapping
2137 : */
2138 58858770 : if (!folio_trylock(folio)) {
2139 1040852 : submit_write_bio(bio_ctrl, 0);
2140 1040852 : folio_lock(folio);
2141 : }
2142 :
2143 58861421 : if (unlikely(folio->mapping != mapping)) {
2144 562 : folio_unlock(folio);
2145 562 : continue;
2146 : }
2147 :
2148 58860859 : if (wbc->sync_mode != WB_SYNC_NONE) {
2149 19604587 : if (folio_test_writeback(folio))
2150 10529 : submit_write_bio(bio_ctrl, 0);
2151 19604587 : folio_wait_writeback(folio);
2152 : }
2153 :
2154 115841925 : if (folio_test_writeback(folio) ||
2155 56987708 : !folio_clear_dirty_for_io(folio)) {
2156 1925472 : folio_unlock(folio);
2157 1926095 : continue;
2158 : }
2159 :
2160 56928745 : ret = __extent_writepage(&folio->page, bio_ctrl);
2161 56932466 : if (ret < 0) {
2162 : done = 1;
2163 : break;
2164 : }
2165 :
2166 : /*
2167 : * the filesystem may choose to bump up nr_to_write.
2168 : * We have to make sure to honor the new nr_to_write
2169 : * at any time
2170 : */
2171 56932466 : nr_to_write_done = wbc->nr_to_write <= 0;
2172 : }
2173 6460022 : folio_batch_release(&fbatch);
2174 6459933 : cond_resched();
2175 : }
2176 5428233 : if (!scanned && !done) {
2177 : /*
2178 : * We hit the last page and there is more work to be done: wrap
2179 : * back to the start of the file
2180 : */
2181 377261 : scanned = 1;
2182 377261 : index = 0;
2183 :
2184 : /*
2185 : * If we're looping we could run into a page that is locked by a
2186 : * writer and that writer could be waiting on writeback for a
2187 : * page in our current bio, and thus deadlock, so flush the
2188 : * write bio here.
2189 : */
2190 377261 : submit_write_bio(bio_ctrl, 0);
2191 377086 : goto retry;
2192 : }
2193 :
2194 5050972 : if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2195 2614652 : mapping->writeback_index = done_index;
2196 :
2197 5050972 : btrfs_add_delayed_iput(BTRFS_I(inode));
2198 5050972 : return ret;
2199 : }
2200 :
2201 : /*
2202 : * Submit the pages in the range to bio for call sites which delalloc range has
2203 : * already been ran (aka, ordered extent inserted) and all pages are still
2204 : * locked.
2205 : */
2206 220 : int extent_write_locked_range(struct inode *inode, u64 start, u64 end,
2207 : struct writeback_control *wbc)
2208 : {
2209 220 : bool found_error = false;
2210 220 : int first_error = 0;
2211 220 : int ret = 0;
2212 220 : struct address_space *mapping = inode->i_mapping;
2213 220 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2214 220 : const u32 sectorsize = fs_info->sectorsize;
2215 220 : loff_t i_size = i_size_read(inode);
2216 220 : u64 cur = start;
2217 220 : struct btrfs_bio_ctrl bio_ctrl = {
2218 : .wbc = wbc,
2219 220 : .opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2220 : };
2221 :
2222 220 : if (wbc->no_cgroup_owner)
2223 220 : bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
2224 :
2225 220 : ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
2226 :
2227 440 : while (cur <= end) {
2228 220 : u64 cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
2229 220 : struct page *page;
2230 220 : int nr = 0;
2231 :
2232 220 : page = find_get_page(mapping, cur >> PAGE_SHIFT);
2233 : /*
2234 : * All pages in the range are locked since
2235 : * btrfs_run_delalloc_range(), thus there is no way to clear
2236 : * the page dirty flag.
2237 : */
2238 220 : ASSERT(PageLocked(page));
2239 220 : ASSERT(PageDirty(page));
2240 220 : clear_page_dirty_for_io(page);
2241 :
2242 220 : ret = __extent_writepage_io(BTRFS_I(inode), page, &bio_ctrl,
2243 : i_size, &nr);
2244 220 : if (ret == 1)
2245 0 : goto next_page;
2246 :
2247 : /* Make sure the mapping tag for page dirty gets cleared. */
2248 220 : if (nr == 0) {
2249 0 : set_page_writeback(page);
2250 0 : end_page_writeback(page);
2251 : }
2252 220 : if (ret)
2253 0 : end_extent_writepage(page, ret, cur, cur_end);
2254 220 : btrfs_page_unlock_writer(fs_info, page, cur, cur_end + 1 - cur);
2255 220 : if (ret < 0) {
2256 0 : found_error = true;
2257 0 : first_error = ret;
2258 : }
2259 220 : next_page:
2260 220 : put_page(page);
2261 220 : cur = cur_end + 1;
2262 : }
2263 :
2264 440 : submit_write_bio(&bio_ctrl, found_error ? ret : 0);
2265 :
2266 220 : if (found_error)
2267 0 : return first_error;
2268 : return ret;
2269 : }
2270 :
2271 5051213 : int extent_writepages(struct address_space *mapping,
2272 : struct writeback_control *wbc)
2273 : {
2274 5051213 : struct inode *inode = mapping->host;
2275 5051213 : int ret = 0;
2276 5051213 : struct btrfs_bio_ctrl bio_ctrl = {
2277 : .wbc = wbc,
2278 5051213 : .opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2279 : };
2280 :
2281 : /*
2282 : * Allow only a single thread to do the reloc work in zoned mode to
2283 : * protect the write pointer updates.
2284 : */
2285 5051213 : btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
2286 5051213 : ret = extent_write_cache_pages(mapping, &bio_ctrl);
2287 5051362 : submit_write_bio(&bio_ctrl, ret);
2288 5051297 : btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
2289 5051297 : return ret;
2290 : }
2291 :
2292 1461271 : void extent_readahead(struct readahead_control *rac)
2293 : {
2294 1461271 : struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ | REQ_RAHEAD };
2295 1461271 : struct page *pagepool[16];
2296 1461271 : struct extent_map *em_cached = NULL;
2297 1461271 : u64 prev_em_start = (u64)-1;
2298 1461271 : int nr;
2299 :
2300 4843374 : while ((nr = readahead_page_batch(rac, pagepool))) {
2301 3382057 : u64 contig_start = readahead_pos(rac);
2302 3382057 : u64 contig_end = contig_start + readahead_batch_length(rac) - 1;
2303 :
2304 3382057 : contiguous_readpages(pagepool, nr, contig_start, contig_end,
2305 : &em_cached, &bio_ctrl, &prev_em_start);
2306 : }
2307 :
2308 1461185 : if (em_cached)
2309 1461185 : free_extent_map(em_cached);
2310 1461260 : submit_one_bio(&bio_ctrl);
2311 1460929 : }
2312 :
2313 : /*
2314 : * basic invalidate_folio code, this waits on any locked or writeback
2315 : * ranges corresponding to the folio, and then deletes any extent state
2316 : * records from the tree
2317 : */
2318 34689 : int extent_invalidate_folio(struct extent_io_tree *tree,
2319 : struct folio *folio, size_t offset)
2320 : {
2321 34689 : struct extent_state *cached_state = NULL;
2322 34689 : u64 start = folio_pos(folio);
2323 34689 : u64 end = start + folio_size(folio) - 1;
2324 34689 : size_t blocksize = folio->mapping->host->i_sb->s_blocksize;
2325 :
2326 : /* This function is only called for the btree inode */
2327 34689 : ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2328 :
2329 34689 : start += ALIGN(offset, blocksize);
2330 34689 : if (start > end)
2331 : return 0;
2332 :
2333 34689 : lock_extent(tree, start, end, &cached_state);
2334 34689 : folio_wait_writeback(folio);
2335 :
2336 : /*
2337 : * Currently for btree io tree, only EXTENT_LOCKED is utilized,
2338 : * so here we only need to unlock the extent range to free any
2339 : * existing extent state.
2340 : */
2341 34689 : unlock_extent(tree, start, end, &cached_state);
2342 34689 : return 0;
2343 : }
2344 :
2345 : /*
2346 : * a helper for release_folio, this tests for areas of the page that
2347 : * are locked or under IO and drops the related state bits if it is safe
2348 : * to drop the page.
2349 : */
2350 55635103 : static int try_release_extent_state(struct extent_io_tree *tree,
2351 : struct page *page, gfp_t mask)
2352 : {
2353 55635103 : u64 start = page_offset(page);
2354 55635103 : u64 end = start + PAGE_SIZE - 1;
2355 55635103 : int ret = 1;
2356 :
2357 55635103 : if (test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) {
2358 : ret = 0;
2359 : } else {
2360 55440255 : u32 clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM |
2361 : EXTENT_DELALLOC_NEW | EXTENT_CTLBITS);
2362 :
2363 : /*
2364 : * At this point we can safely clear everything except the
2365 : * locked bit, the nodatasum bit and the delalloc new bit.
2366 : * The delalloc new bit will be cleared by ordered extent
2367 : * completion.
2368 : */
2369 55440255 : ret = __clear_extent_bit(tree, start, end, clear_bits, NULL, NULL);
2370 :
2371 : /* if clear_extent_bit failed for enomem reasons,
2372 : * we can't allow the release to continue.
2373 : */
2374 55441024 : if (ret < 0)
2375 : ret = 0;
2376 : else
2377 55441024 : ret = 1;
2378 : }
2379 55636139 : return ret;
2380 : }
2381 :
2382 : /*
2383 : * a helper for release_folio. As long as there are no locked extents
2384 : * in the range corresponding to the page, both state records and extent
2385 : * map records are removed
2386 : */
2387 55636522 : int try_release_extent_mapping(struct page *page, gfp_t mask)
2388 : {
2389 55636522 : struct extent_map *em;
2390 55636522 : u64 start = page_offset(page);
2391 55636522 : u64 end = start + PAGE_SIZE - 1;
2392 55636522 : struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
2393 55636522 : struct extent_io_tree *tree = &btrfs_inode->io_tree;
2394 55636522 : struct extent_map_tree *map = &btrfs_inode->extent_tree;
2395 :
2396 55636522 : if (gfpflags_allow_blocking(mask) &&
2397 46328270 : page->mapping->host->i_size > SZ_16M) {
2398 : u64 len;
2399 7337951 : while (start <= end) {
2400 7280810 : struct btrfs_fs_info *fs_info;
2401 7280810 : u64 cur_gen;
2402 :
2403 7280810 : len = end - start + 1;
2404 7280810 : write_lock(&map->lock);
2405 7280810 : em = lookup_extent_mapping(map, start, len);
2406 7280810 : if (!em) {
2407 668344 : write_unlock(&map->lock);
2408 668344 : break;
2409 : }
2410 6612466 : if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
2411 6578105 : em->start != start) {
2412 6555325 : write_unlock(&map->lock);
2413 6555325 : free_extent_map(em);
2414 6555325 : break;
2415 : }
2416 57141 : if (test_range_bit(tree, em->start,
2417 : extent_map_end(em) - 1,
2418 : EXTENT_LOCKED, 0, NULL))
2419 15 : goto next;
2420 : /*
2421 : * If it's not in the list of modified extents, used
2422 : * by a fast fsync, we can remove it. If it's being
2423 : * logged we can safely remove it since fsync took an
2424 : * extra reference on the em.
2425 : */
2426 57126 : if (list_empty(&em->list) ||
2427 0 : test_bit(EXTENT_FLAG_LOGGING, &em->flags))
2428 35046 : goto remove_em;
2429 : /*
2430 : * If it's in the list of modified extents, remove it
2431 : * only if its generation is older then the current one,
2432 : * in which case we don't need it for a fast fsync.
2433 : * Otherwise don't remove it, we could be racing with an
2434 : * ongoing fast fsync that could miss the new extent.
2435 : */
2436 22080 : fs_info = btrfs_inode->root->fs_info;
2437 22080 : spin_lock(&fs_info->trans_lock);
2438 22080 : cur_gen = fs_info->generation;
2439 22080 : spin_unlock(&fs_info->trans_lock);
2440 22080 : if (em->generation >= cur_gen)
2441 15496 : goto next;
2442 6584 : remove_em:
2443 : /*
2444 : * We only remove extent maps that are not in the list of
2445 : * modified extents or that are in the list but with a
2446 : * generation lower then the current generation, so there
2447 : * is no need to set the full fsync flag on the inode (it
2448 : * hurts the fsync performance for workloads with a data
2449 : * size that exceeds or is close to the system's memory).
2450 : */
2451 41630 : remove_extent_mapping(map, em);
2452 : /* once for the rb tree */
2453 41630 : free_extent_map(em);
2454 57141 : next:
2455 57141 : start = extent_map_end(em);
2456 57141 : write_unlock(&map->lock);
2457 :
2458 : /* once for us */
2459 57141 : free_extent_map(em);
2460 :
2461 57141 : cond_resched(); /* Allow large-extent preemption. */
2462 : }
2463 : }
2464 55636522 : return try_release_extent_state(tree, page, mask);
2465 : }
2466 :
2467 : /*
2468 : * To cache previous fiemap extent
2469 : *
2470 : * Will be used for merging fiemap extent
2471 : */
2472 : struct fiemap_cache {
2473 : u64 offset;
2474 : u64 phys;
2475 : u64 len;
2476 : u32 flags;
2477 : bool cached;
2478 : };
2479 :
2480 : /*
2481 : * Helper to submit fiemap extent.
2482 : *
2483 : * Will try to merge current fiemap extent specified by @offset, @phys,
2484 : * @len and @flags with cached one.
2485 : * And only when we fails to merge, cached one will be submitted as
2486 : * fiemap extent.
2487 : *
2488 : * Return value is the same as fiemap_fill_next_extent().
2489 : */
2490 1327675 : static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
2491 : struct fiemap_cache *cache,
2492 : u64 offset, u64 phys, u64 len, u32 flags)
2493 : {
2494 1327675 : int ret = 0;
2495 :
2496 : /* Set at the end of extent_fiemap(). */
2497 1327675 : ASSERT((flags & FIEMAP_EXTENT_LAST) == 0);
2498 :
2499 1327675 : if (!cache->cached)
2500 45228 : goto assign;
2501 :
2502 : /*
2503 : * Sanity check, extent_fiemap() should have ensured that new
2504 : * fiemap extent won't overlap with cached one.
2505 : * Not recoverable.
2506 : *
2507 : * NOTE: Physical address can overlap, due to compression
2508 : */
2509 1282447 : if (cache->offset + cache->len > offset) {
2510 0 : WARN_ON(1);
2511 0 : return -EINVAL;
2512 : }
2513 :
2514 : /*
2515 : * Only merges fiemap extents if
2516 : * 1) Their logical addresses are continuous
2517 : *
2518 : * 2) Their physical addresses are continuous
2519 : * So truly compressed (physical size smaller than logical size)
2520 : * extents won't get merged with each other
2521 : *
2522 : * 3) Share same flags
2523 : */
2524 1282447 : if (cache->offset + cache->len == offset &&
2525 1043469 : cache->phys + cache->len == phys &&
2526 337696 : cache->flags == flags) {
2527 255851 : cache->len += len;
2528 255851 : return 0;
2529 : }
2530 :
2531 : /* Not mergeable, need to submit cached one */
2532 1026596 : ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
2533 : cache->len, cache->flags);
2534 1026559 : cache->cached = false;
2535 1026559 : if (ret)
2536 : return ret;
2537 1005093 : assign:
2538 1050321 : cache->cached = true;
2539 1050321 : cache->offset = offset;
2540 1050321 : cache->phys = phys;
2541 1050321 : cache->len = len;
2542 1050321 : cache->flags = flags;
2543 :
2544 1050321 : return 0;
2545 : }
2546 :
2547 : /*
2548 : * Emit last fiemap cache
2549 : *
2550 : * The last fiemap cache may still be cached in the following case:
2551 : * 0 4k 8k
2552 : * |<- Fiemap range ->|
2553 : * |<------------ First extent ----------->|
2554 : *
2555 : * In this case, the first extent range will be cached but not emitted.
2556 : * So we must emit it before ending extent_fiemap().
2557 : */
2558 47625 : static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo,
2559 : struct fiemap_cache *cache)
2560 : {
2561 47625 : int ret;
2562 :
2563 47625 : if (!cache->cached)
2564 : return 0;
2565 :
2566 23774 : ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
2567 : cache->len, cache->flags);
2568 23776 : cache->cached = false;
2569 23776 : if (ret > 0)
2570 : ret = 0;
2571 : return ret;
2572 : }
2573 :
2574 1311984 : static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path)
2575 : {
2576 1311984 : struct extent_buffer *clone;
2577 1311984 : struct btrfs_key key;
2578 1311984 : int slot;
2579 1311984 : int ret;
2580 :
2581 1311984 : path->slots[0]++;
2582 1311984 : if (path->slots[0] < btrfs_header_nritems(path->nodes[0]))
2583 : return 0;
2584 :
2585 13712 : ret = btrfs_next_leaf(inode->root, path);
2586 13712 : if (ret != 0)
2587 : return ret;
2588 :
2589 : /*
2590 : * Don't bother with cloning if there are no more file extent items for
2591 : * our inode.
2592 : */
2593 4994 : btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2594 4994 : if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY)
2595 : return 1;
2596 :
2597 : /* See the comment at fiemap_search_slot() about why we clone. */
2598 4920 : clone = btrfs_clone_extent_buffer(path->nodes[0]);
2599 4920 : if (!clone)
2600 : return -ENOMEM;
2601 :
2602 4920 : slot = path->slots[0];
2603 4920 : btrfs_release_path(path);
2604 4920 : path->nodes[0] = clone;
2605 4920 : path->slots[0] = slot;
2606 :
2607 4920 : return 0;
2608 : }
2609 :
2610 : /*
2611 : * Search for the first file extent item that starts at a given file offset or
2612 : * the one that starts immediately before that offset.
2613 : * Returns: 0 on success, < 0 on error, 1 if not found.
2614 : */
2615 47634 : static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path,
2616 : u64 file_offset)
2617 : {
2618 47634 : const u64 ino = btrfs_ino(inode);
2619 47634 : struct btrfs_root *root = inode->root;
2620 47634 : struct extent_buffer *clone;
2621 47634 : struct btrfs_key key;
2622 47634 : int slot;
2623 47634 : int ret;
2624 :
2625 47634 : key.objectid = ino;
2626 47634 : key.type = BTRFS_EXTENT_DATA_KEY;
2627 47634 : key.offset = file_offset;
2628 :
2629 47634 : ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2630 47636 : if (ret < 0)
2631 : return ret;
2632 :
2633 47636 : if (ret > 0 && path->slots[0] > 0) {
2634 16295 : btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
2635 16292 : if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY)
2636 7805 : path->slots[0]--;
2637 : }
2638 :
2639 47633 : if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
2640 102 : ret = btrfs_next_leaf(root, path);
2641 102 : if (ret != 0)
2642 : return ret;
2643 :
2644 37 : btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2645 37 : if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2646 : return 1;
2647 : }
2648 :
2649 : /*
2650 : * We clone the leaf and use it during fiemap. This is because while
2651 : * using the leaf we do expensive things like checking if an extent is
2652 : * shared, which can take a long time. In order to prevent blocking
2653 : * other tasks for too long, we use a clone of the leaf. We have locked
2654 : * the file range in the inode's io tree, so we know none of our file
2655 : * extent items can change. This way we avoid blocking other tasks that
2656 : * want to insert items for other inodes in the same leaf or b+tree
2657 : * rebalance operations (triggered for example when someone is trying
2658 : * to push items into this leaf when trying to insert an item in a
2659 : * neighbour leaf).
2660 : * We also need the private clone because holding a read lock on an
2661 : * extent buffer of the subvolume's b+tree will make lockdep unhappy
2662 : * when we call fiemap_fill_next_extent(), because that may cause a page
2663 : * fault when filling the user space buffer with fiemap data.
2664 : */
2665 47560 : clone = btrfs_clone_extent_buffer(path->nodes[0]);
2666 47560 : if (!clone)
2667 : return -ENOMEM;
2668 :
2669 47560 : slot = path->slots[0];
2670 47560 : btrfs_release_path(path);
2671 47563 : path->nodes[0] = clone;
2672 47563 : path->slots[0] = slot;
2673 :
2674 47563 : return 0;
2675 : }
2676 :
2677 : /*
2678 : * Process a range which is a hole or a prealloc extent in the inode's subvolume
2679 : * btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc
2680 : * extent. The end offset (@end) is inclusive.
2681 : */
2682 451928 : static int fiemap_process_hole(struct btrfs_inode *inode,
2683 : struct fiemap_extent_info *fieinfo,
2684 : struct fiemap_cache *cache,
2685 : struct extent_state **delalloc_cached_state,
2686 : struct btrfs_backref_share_check_ctx *backref_ctx,
2687 : u64 disk_bytenr, u64 extent_offset,
2688 : u64 extent_gen,
2689 : u64 start, u64 end)
2690 : {
2691 451928 : const u64 i_size = i_size_read(&inode->vfs_inode);
2692 451928 : u64 cur_offset = start;
2693 451928 : u64 last_delalloc_end = 0;
2694 451928 : u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN;
2695 451928 : bool checked_extent_shared = false;
2696 451928 : int ret;
2697 :
2698 : /*
2699 : * There can be no delalloc past i_size, so don't waste time looking for
2700 : * it beyond i_size.
2701 : */
2702 475076 : while (cur_offset < end && cur_offset < i_size) {
2703 437385 : u64 delalloc_start;
2704 437385 : u64 delalloc_end;
2705 437385 : u64 prealloc_start;
2706 437385 : u64 prealloc_len = 0;
2707 437385 : bool delalloc;
2708 :
2709 437385 : delalloc = btrfs_find_delalloc_in_range(inode, cur_offset, end,
2710 : delalloc_cached_state,
2711 : &delalloc_start,
2712 : &delalloc_end);
2713 437427 : if (!delalloc)
2714 : break;
2715 :
2716 : /*
2717 : * If this is a prealloc extent we have to report every section
2718 : * of it that has no delalloc.
2719 : */
2720 23148 : if (disk_bytenr != 0) {
2721 22730 : if (last_delalloc_end == 0) {
2722 22729 : prealloc_start = start;
2723 22729 : prealloc_len = delalloc_start - start;
2724 : } else {
2725 1 : prealloc_start = last_delalloc_end + 1;
2726 1 : prealloc_len = delalloc_start - prealloc_start;
2727 : }
2728 : }
2729 :
2730 22730 : if (prealloc_len > 0) {
2731 4 : if (!checked_extent_shared && fieinfo->fi_extents_max) {
2732 3 : ret = btrfs_is_data_extent_shared(inode,
2733 : disk_bytenr,
2734 : extent_gen,
2735 : backref_ctx);
2736 3 : if (ret < 0)
2737 0 : return ret;
2738 3 : else if (ret > 0)
2739 0 : prealloc_flags |= FIEMAP_EXTENT_SHARED;
2740 :
2741 : checked_extent_shared = true;
2742 : }
2743 4 : ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
2744 : disk_bytenr + extent_offset,
2745 : prealloc_len, prealloc_flags);
2746 4 : if (ret)
2747 0 : return ret;
2748 4 : extent_offset += prealloc_len;
2749 : }
2750 :
2751 23148 : ret = emit_fiemap_extent(fieinfo, cache, delalloc_start, 0,
2752 23148 : delalloc_end + 1 - delalloc_start,
2753 : FIEMAP_EXTENT_DELALLOC |
2754 : FIEMAP_EXTENT_UNKNOWN);
2755 23148 : if (ret)
2756 0 : return ret;
2757 :
2758 23148 : last_delalloc_end = delalloc_end;
2759 23148 : cur_offset = delalloc_end + 1;
2760 23148 : extent_offset += cur_offset - delalloc_start;
2761 23148 : cond_resched();
2762 : }
2763 :
2764 : /*
2765 : * Either we found no delalloc for the whole prealloc extent or we have
2766 : * a prealloc extent that spans i_size or starts at or after i_size.
2767 : */
2768 451970 : if (disk_bytenr != 0 && last_delalloc_end < end) {
2769 156746 : u64 prealloc_start;
2770 156746 : u64 prealloc_len;
2771 :
2772 156746 : if (last_delalloc_end == 0) {
2773 155522 : prealloc_start = start;
2774 155522 : prealloc_len = end + 1 - start;
2775 : } else {
2776 1224 : prealloc_start = last_delalloc_end + 1;
2777 1224 : prealloc_len = end + 1 - prealloc_start;
2778 : }
2779 :
2780 156746 : if (!checked_extent_shared && fieinfo->fi_extents_max) {
2781 120979 : ret = btrfs_is_data_extent_shared(inode,
2782 : disk_bytenr,
2783 : extent_gen,
2784 : backref_ctx);
2785 120979 : if (ret < 0)
2786 : return ret;
2787 120979 : else if (ret > 0)
2788 3636 : prealloc_flags |= FIEMAP_EXTENT_SHARED;
2789 : }
2790 156746 : ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
2791 : disk_bytenr + extent_offset,
2792 : prealloc_len, prealloc_flags);
2793 156746 : if (ret)
2794 456 : return ret;
2795 : }
2796 :
2797 : return 0;
2798 : }
2799 :
2800 47634 : static int fiemap_find_last_extent_offset(struct btrfs_inode *inode,
2801 : struct btrfs_path *path,
2802 : u64 *last_extent_end_ret)
2803 : {
2804 47634 : const u64 ino = btrfs_ino(inode);
2805 47634 : struct btrfs_root *root = inode->root;
2806 47634 : struct extent_buffer *leaf;
2807 47634 : struct btrfs_file_extent_item *ei;
2808 47634 : struct btrfs_key key;
2809 47634 : u64 disk_bytenr;
2810 47634 : int ret;
2811 :
2812 : /*
2813 : * Lookup the last file extent. We're not using i_size here because
2814 : * there might be preallocation past i_size.
2815 : */
2816 47634 : ret = btrfs_lookup_file_extent(NULL, root, path, ino, (u64)-1, 0);
2817 : /* There can't be a file extent item at offset (u64)-1 */
2818 47634 : ASSERT(ret != 0);
2819 47634 : if (ret < 0)
2820 : return ret;
2821 :
2822 : /*
2823 : * For a non-existing key, btrfs_search_slot() always leaves us at a
2824 : * slot > 0, except if the btree is empty, which is impossible because
2825 : * at least it has the inode item for this inode and all the items for
2826 : * the root inode 256.
2827 : */
2828 47634 : ASSERT(path->slots[0] > 0);
2829 47634 : path->slots[0]--;
2830 47634 : leaf = path->nodes[0];
2831 47634 : btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2832 47633 : if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) {
2833 : /* No file extent items in the subvolume tree. */
2834 745 : *last_extent_end_ret = 0;
2835 745 : return 0;
2836 : }
2837 :
2838 : /*
2839 : * For an inline extent, the disk_bytenr is where inline data starts at,
2840 : * so first check if we have an inline extent item before checking if we
2841 : * have an implicit hole (disk_bytenr == 0).
2842 : */
2843 46888 : ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
2844 46885 : if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) {
2845 2 : *last_extent_end_ret = btrfs_file_extent_end(path);
2846 2 : return 0;
2847 : }
2848 :
2849 : /*
2850 : * Find the last file extent item that is not a hole (when NO_HOLES is
2851 : * not enabled). This should take at most 2 iterations in the worst
2852 : * case: we have one hole file extent item at slot 0 of a leaf and
2853 : * another hole file extent item as the last item in the previous leaf.
2854 : * This is because we merge file extent items that represent holes.
2855 : */
2856 46887 : disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2857 46886 : while (disk_bytenr == 0) {
2858 0 : ret = btrfs_previous_item(root, path, ino, BTRFS_EXTENT_DATA_KEY);
2859 0 : if (ret < 0) {
2860 0 : return ret;
2861 0 : } else if (ret > 0) {
2862 : /* No file extent items that are not holes. */
2863 0 : *last_extent_end_ret = 0;
2864 0 : return 0;
2865 : }
2866 0 : leaf = path->nodes[0];
2867 0 : ei = btrfs_item_ptr(leaf, path->slots[0],
2868 : struct btrfs_file_extent_item);
2869 0 : disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2870 : }
2871 :
2872 46886 : *last_extent_end_ret = btrfs_file_extent_end(path);
2873 46885 : return 0;
2874 : }
2875 :
2876 47635 : int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
2877 : u64 start, u64 len)
2878 : {
2879 47635 : const u64 ino = btrfs_ino(inode);
2880 47635 : struct extent_state *cached_state = NULL;
2881 47635 : struct extent_state *delalloc_cached_state = NULL;
2882 47635 : struct btrfs_path *path;
2883 47635 : struct fiemap_cache cache = { 0 };
2884 47635 : struct btrfs_backref_share_check_ctx *backref_ctx;
2885 47635 : u64 last_extent_end;
2886 47635 : u64 prev_extent_end;
2887 47635 : u64 lockstart;
2888 47635 : u64 lockend;
2889 47635 : bool stopped = false;
2890 47635 : int ret;
2891 :
2892 47635 : backref_ctx = btrfs_alloc_backref_share_check_ctx();
2893 47635 : path = btrfs_alloc_path();
2894 47633 : if (!backref_ctx || !path) {
2895 0 : ret = -ENOMEM;
2896 0 : goto out;
2897 : }
2898 :
2899 47633 : lockstart = round_down(start, inode->root->fs_info->sectorsize);
2900 47633 : lockend = round_up(start + len, inode->root->fs_info->sectorsize);
2901 47633 : prev_extent_end = lockstart;
2902 :
2903 47633 : btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
2904 47632 : lock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
2905 :
2906 47636 : ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end);
2907 47631 : if (ret < 0)
2908 0 : goto out_unlock;
2909 47631 : btrfs_release_path(path);
2910 :
2911 47635 : path->reada = READA_FORWARD;
2912 47635 : ret = fiemap_search_slot(inode, path, lockstart);
2913 47636 : if (ret < 0) {
2914 0 : goto out_unlock;
2915 47636 : } else if (ret > 0) {
2916 : /*
2917 : * No file extent item found, but we may have delalloc between
2918 : * the current offset and i_size. So check for that.
2919 : */
2920 73 : ret = 0;
2921 73 : goto check_eof_delalloc;
2922 : }
2923 :
2924 1350795 : while (prev_extent_end < lockend) {
2925 1346456 : struct extent_buffer *leaf = path->nodes[0];
2926 1346456 : struct btrfs_file_extent_item *ei;
2927 1346456 : struct btrfs_key key;
2928 1346456 : u64 extent_end;
2929 1346456 : u64 extent_len;
2930 1346456 : u64 extent_offset = 0;
2931 1346456 : u64 extent_gen;
2932 1346456 : u64 disk_bytenr = 0;
2933 1346456 : u64 flags = 0;
2934 1346456 : int extent_type;
2935 1346456 : u8 compression;
2936 :
2937 1346456 : btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2938 1346453 : if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2939 : break;
2940 :
2941 1336664 : extent_end = btrfs_file_extent_end(path);
2942 :
2943 : /*
2944 : * The first iteration can leave us at an extent item that ends
2945 : * before our range's start. Move to the next item.
2946 : */
2947 1336616 : if (extent_end <= lockstart)
2948 7557 : goto next_item;
2949 :
2950 1329059 : backref_ctx->curr_leaf_bytenr = leaf->start;
2951 :
2952 : /* We have in implicit hole (NO_HOLES feature enabled). */
2953 1329059 : if (prev_extent_end < key.offset) {
2954 255478 : const u64 range_end = min(key.offset, lockend) - 1;
2955 :
2956 255478 : ret = fiemap_process_hole(inode, fieinfo, &cache,
2957 : &delalloc_cached_state,
2958 : backref_ctx, 0, 0, 0,
2959 : prev_extent_end, range_end);
2960 255525 : if (ret < 0) {
2961 0 : goto out_unlock;
2962 255525 : } else if (ret > 0) {
2963 : /* fiemap_fill_next_extent() told us to stop. */
2964 : stopped = true;
2965 : break;
2966 : }
2967 :
2968 : /* We've reached the end of the fiemap range, stop. */
2969 255525 : if (key.offset >= lockend) {
2970 : stopped = true;
2971 : break;
2972 : }
2973 : }
2974 :
2975 1325930 : extent_len = extent_end - key.offset;
2976 1325930 : ei = btrfs_item_ptr(leaf, path->slots[0],
2977 : struct btrfs_file_extent_item);
2978 1325938 : compression = btrfs_file_extent_compression(leaf, ei);
2979 1325961 : extent_type = btrfs_file_extent_type(leaf, ei);
2980 1325953 : extent_gen = btrfs_file_extent_generation(leaf, ei);
2981 :
2982 1325908 : if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
2983 1325898 : disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2984 1325925 : if (compression == BTRFS_COMPRESS_NONE)
2985 647107 : extent_offset = btrfs_file_extent_offset(leaf, ei);
2986 : }
2987 :
2988 1325958 : if (compression != BTRFS_COMPRESS_NONE)
2989 678817 : flags |= FIEMAP_EXTENT_ENCODED;
2990 :
2991 1325958 : if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
2992 1 : flags |= FIEMAP_EXTENT_DATA_INLINE;
2993 1 : flags |= FIEMAP_EXTENT_NOT_ALIGNED;
2994 1 : ret = emit_fiemap_extent(fieinfo, &cache, key.offset, 0,
2995 : extent_len, flags);
2996 1325957 : } else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
2997 178251 : ret = fiemap_process_hole(inode, fieinfo, &cache,
2998 : &delalloc_cached_state,
2999 : backref_ctx,
3000 : disk_bytenr, extent_offset,
3001 : extent_gen, key.offset,
3002 : extent_end - 1);
3003 1147706 : } else if (disk_bytenr == 0) {
3004 : /* We have an explicit hole. */
3005 3 : ret = fiemap_process_hole(inode, fieinfo, &cache,
3006 : &delalloc_cached_state,
3007 : backref_ctx, 0, 0, 0,
3008 : key.offset, extent_end - 1);
3009 : } else {
3010 : /* We have a regular extent. */
3011 1147703 : if (fieinfo->fi_extents_max) {
3012 1053955 : ret = btrfs_is_data_extent_shared(inode,
3013 : disk_bytenr,
3014 : extent_gen,
3015 : backref_ctx);
3016 1054021 : if (ret < 0)
3017 0 : goto out_unlock;
3018 1054021 : else if (ret > 0)
3019 451036 : flags |= FIEMAP_EXTENT_SHARED;
3020 : }
3021 :
3022 1147769 : ret = emit_fiemap_extent(fieinfo, &cache, key.offset,
3023 : disk_bytenr + extent_offset,
3024 : extent_len, flags);
3025 : }
3026 :
3027 1325865 : if (ret < 0) {
3028 0 : goto out_unlock;
3029 1325865 : } else if (ret > 0) {
3030 : /* fiemap_fill_next_extent() told us to stop. */
3031 : stopped = true;
3032 : break;
3033 : }
3034 :
3035 : prev_extent_end = extent_end;
3036 1311956 : next_item:
3037 1311956 : if (fatal_signal_pending(current)) {
3038 0 : ret = -EINTR;
3039 0 : goto out_unlock;
3040 : }
3041 :
3042 1311984 : ret = fiemap_next_leaf_item(inode, path);
3043 1312013 : if (ret < 0) {
3044 0 : goto out_unlock;
3045 1312013 : } else if (ret > 0) {
3046 : /* No more file extent items for this inode. */
3047 : break;
3048 : }
3049 1303221 : cond_resched();
3050 : }
3051 :
3052 4339 : check_eof_delalloc:
3053 : /*
3054 : * Release (and free) the path before emitting any final entries to
3055 : * fiemap_fill_next_extent() to keep lockdep happy. This is because
3056 : * once we find no more file extent items exist, we may have a
3057 : * non-cloned leaf, and fiemap_fill_next_extent() can trigger page
3058 : * faults when copying data to the user space buffer.
3059 : */
3060 47635 : btrfs_free_path(path);
3061 47631 : path = NULL;
3062 :
3063 47631 : if (!stopped && prev_extent_end < lockend) {
3064 18157 : ret = fiemap_process_hole(inode, fieinfo, &cache,
3065 : &delalloc_cached_state, backref_ctx,
3066 : 0, 0, 0, prev_extent_end, lockend - 1);
3067 18152 : if (ret < 0)
3068 0 : goto out_unlock;
3069 : prev_extent_end = lockend;
3070 : }
3071 :
3072 47626 : if (cache.cached && cache.offset + cache.len >= last_extent_end) {
3073 16778 : const u64 i_size = i_size_read(&inode->vfs_inode);
3074 :
3075 16778 : if (prev_extent_end < i_size) {
3076 2 : u64 delalloc_start;
3077 2 : u64 delalloc_end;
3078 2 : bool delalloc;
3079 :
3080 2 : delalloc = btrfs_find_delalloc_in_range(inode,
3081 : prev_extent_end,
3082 : i_size - 1,
3083 : &delalloc_cached_state,
3084 : &delalloc_start,
3085 : &delalloc_end);
3086 2 : if (!delalloc)
3087 0 : cache.flags |= FIEMAP_EXTENT_LAST;
3088 : } else {
3089 16776 : cache.flags |= FIEMAP_EXTENT_LAST;
3090 : }
3091 : }
3092 :
3093 47626 : ret = emit_last_fiemap_cache(fieinfo, &cache);
3094 :
3095 47626 : out_unlock:
3096 47626 : unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
3097 47631 : btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
3098 47622 : out:
3099 47622 : free_extent_state(delalloc_cached_state);
3100 47620 : btrfs_free_backref_share_ctx(backref_ctx);
3101 47626 : btrfs_free_path(path);
3102 47620 : return ret;
3103 : }
3104 :
3105 : static void __free_extent_buffer(struct extent_buffer *eb)
3106 : {
3107 30779726 : kmem_cache_free(extent_buffer_cache, eb);
3108 : }
3109 :
3110 43205031 : static int extent_buffer_under_io(const struct extent_buffer *eb)
3111 : {
3112 43205031 : return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
3113 0 : test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3114 : }
3115 :
3116 : static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page)
3117 : {
3118 0 : struct btrfs_subpage *subpage;
3119 :
3120 0 : lockdep_assert_held(&page->mapping->private_lock);
3121 :
3122 0 : if (PagePrivate(page)) {
3123 0 : subpage = (struct btrfs_subpage *)page->private;
3124 0 : if (atomic_read(&subpage->eb_refs))
3125 : return true;
3126 : /*
3127 : * Even there is no eb refs here, we may still have
3128 : * end_page_read() call relying on page::private.
3129 : */
3130 0 : if (atomic_read(&subpage->readers))
3131 : return true;
3132 : }
3133 : return false;
3134 : }
3135 :
3136 122402694 : static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
3137 : {
3138 122402694 : struct btrfs_fs_info *fs_info = eb->fs_info;
3139 122402694 : const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3140 :
3141 : /*
3142 : * For mapped eb, we're going to change the page private, which should
3143 : * be done under the private_lock.
3144 : */
3145 122402694 : if (mapped)
3146 45008397 : spin_lock(&page->mapping->private_lock);
3147 :
3148 122404405 : if (!PagePrivate(page)) {
3149 0 : if (mapped)
3150 0 : spin_unlock(&page->mapping->private_lock);
3151 0 : return;
3152 : }
3153 :
3154 122404405 : if (fs_info->nodesize >= PAGE_SIZE) {
3155 : /*
3156 : * We do this since we'll remove the pages after we've
3157 : * removed the eb from the radix tree, so we could race
3158 : * and have this page now attached to the new eb. So
3159 : * only clear page_private if it's still connected to
3160 : * this eb.
3161 : */
3162 122404405 : if (PagePrivate(page) &&
3163 122404405 : page->private == (unsigned long)eb) {
3164 122404292 : BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3165 122404292 : BUG_ON(PageDirty(page));
3166 122402974 : BUG_ON(PageWriteback(page));
3167 : /*
3168 : * We need to make sure we haven't be attached
3169 : * to a new eb.
3170 : */
3171 122402156 : detach_page_private(page);
3172 : }
3173 122407533 : if (mapped)
3174 45010108 : spin_unlock(&page->mapping->private_lock);
3175 122407525 : return;
3176 : }
3177 :
3178 : /*
3179 : * For subpage, we can have dummy eb with page private. In this case,
3180 : * we can directly detach the private as such page is only attached to
3181 : * one dummy eb, no sharing.
3182 : */
3183 0 : if (!mapped) {
3184 0 : btrfs_detach_subpage(fs_info, page);
3185 0 : return;
3186 : }
3187 :
3188 0 : btrfs_page_dec_eb_refs(fs_info, page);
3189 :
3190 : /*
3191 : * We can only detach the page private if there are no other ebs in the
3192 : * page range and no unfinished IO.
3193 : */
3194 0 : if (!page_range_has_eb(fs_info, page))
3195 0 : btrfs_detach_subpage(fs_info, page);
3196 :
3197 0 : spin_unlock(&page->mapping->private_lock);
3198 : }
3199 :
3200 : /* Release all pages attached to the extent buffer */
3201 30856553 : static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
3202 : {
3203 30856553 : int i;
3204 30856553 : int num_pages;
3205 :
3206 30856553 : ASSERT(!extent_buffer_under_io(eb));
3207 :
3208 30855603 : num_pages = num_extent_pages(eb);
3209 153260370 : for (i = 0; i < num_pages; i++) {
3210 122403157 : struct page *page = eb->pages[i];
3211 :
3212 122403157 : if (!page)
3213 92 : continue;
3214 :
3215 122403065 : detach_extent_buffer_page(eb, page);
3216 :
3217 : /* One for when we allocated the page */
3218 122406218 : put_page(page);
3219 : }
3220 30857213 : }
3221 :
3222 : /*
3223 : * Helper for releasing the extent buffer.
3224 : */
3225 23 : static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
3226 : {
3227 23 : btrfs_release_extent_buffer_pages(eb);
3228 23 : btrfs_leak_debug_del_eb(eb);
3229 23 : __free_extent_buffer(eb);
3230 23 : }
3231 :
3232 : static struct extent_buffer *
3233 30853682 : __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
3234 : unsigned long len)
3235 : {
3236 30853682 : struct extent_buffer *eb = NULL;
3237 :
3238 30853682 : eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
3239 30854430 : eb->start = start;
3240 30854430 : eb->len = len;
3241 30854430 : eb->fs_info = fs_info;
3242 30854430 : init_rwsem(&eb->lock);
3243 :
3244 30854409 : btrfs_leak_debug_add_eb(eb);
3245 :
3246 30854409 : spin_lock_init(&eb->refs_lock);
3247 30853377 : atomic_set(&eb->refs, 1);
3248 :
3249 30853377 : ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE);
3250 :
3251 30853377 : return eb;
3252 : }
3253 :
3254 19348861 : struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
3255 : {
3256 19348861 : int i;
3257 19348861 : struct extent_buffer *new;
3258 19348861 : int num_pages = num_extent_pages(src);
3259 19348861 : int ret;
3260 :
3261 19348861 : new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
3262 19349145 : if (new == NULL)
3263 : return NULL;
3264 :
3265 : /*
3266 : * Set UNMAPPED before calling btrfs_release_extent_buffer(), as
3267 : * btrfs_release_extent_buffer() have different behavior for
3268 : * UNMAPPED subpage extent buffer.
3269 : */
3270 19349145 : set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
3271 :
3272 19350716 : ret = btrfs_alloc_page_array(num_pages, new->pages);
3273 19349387 : if (ret) {
3274 0 : btrfs_release_extent_buffer(new);
3275 0 : return NULL;
3276 : }
3277 :
3278 96744257 : for (i = 0; i < num_pages; i++) {
3279 77393694 : int ret;
3280 77393694 : struct page *p = new->pages[i];
3281 :
3282 77393694 : ret = attach_extent_buffer_page(new, p, NULL);
3283 77393708 : if (ret < 0) {
3284 0 : btrfs_release_extent_buffer(new);
3285 0 : return NULL;
3286 : }
3287 77393708 : WARN_ON(PageDirty(p));
3288 77393696 : copy_page(page_address(p), page_address(src->pages[i]));
3289 : }
3290 19350563 : set_extent_buffer_uptodate(new);
3291 :
3292 19350563 : return new;
3293 : }
3294 :
3295 3219 : struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3296 : u64 start, unsigned long len)
3297 : {
3298 3219 : struct extent_buffer *eb;
3299 3219 : int num_pages;
3300 3219 : int i;
3301 3219 : int ret;
3302 :
3303 3219 : eb = __alloc_extent_buffer(fs_info, start, len);
3304 3219 : if (!eb)
3305 : return NULL;
3306 :
3307 3219 : num_pages = num_extent_pages(eb);
3308 3219 : ret = btrfs_alloc_page_array(num_pages, eb->pages);
3309 3219 : if (ret)
3310 0 : goto err;
3311 :
3312 16317 : for (i = 0; i < num_pages; i++) {
3313 13098 : struct page *p = eb->pages[i];
3314 :
3315 13098 : ret = attach_extent_buffer_page(eb, p, NULL);
3316 13098 : if (ret < 0)
3317 0 : goto err;
3318 : }
3319 :
3320 3219 : set_extent_buffer_uptodate(eb);
3321 3219 : btrfs_set_header_nritems(eb, 0);
3322 3219 : set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3323 :
3324 3219 : return eb;
3325 : err:
3326 0 : for (i = 0; i < num_pages; i++) {
3327 0 : if (eb->pages[i]) {
3328 0 : detach_extent_buffer_page(eb, eb->pages[i]);
3329 0 : __free_page(eb->pages[i]);
3330 : }
3331 : }
3332 0 : __free_extent_buffer(eb);
3333 0 : return NULL;
3334 : }
3335 :
3336 3219 : struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3337 : u64 start)
3338 : {
3339 3219 : return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
3340 : }
3341 :
3342 907408038 : static void check_buffer_tree_ref(struct extent_buffer *eb)
3343 : {
3344 907408038 : int refs;
3345 : /*
3346 : * The TREE_REF bit is first set when the extent_buffer is added
3347 : * to the radix tree. It is also reset, if unset, when a new reference
3348 : * is created by find_extent_buffer.
3349 : *
3350 : * It is only cleared in two cases: freeing the last non-tree
3351 : * reference to the extent_buffer when its STALE bit is set or
3352 : * calling release_folio when the tree reference is the only reference.
3353 : *
3354 : * In both cases, care is taken to ensure that the extent_buffer's
3355 : * pages are not under io. However, release_folio can be concurrently
3356 : * called with creating new references, which is prone to race
3357 : * conditions between the calls to check_buffer_tree_ref in those
3358 : * codepaths and clearing TREE_REF in try_release_extent_buffer.
3359 : *
3360 : * The actual lifetime of the extent_buffer in the radix tree is
3361 : * adequately protected by the refcount, but the TREE_REF bit and
3362 : * its corresponding reference are not. To protect against this
3363 : * class of races, we call check_buffer_tree_ref from the codepaths
3364 : * which trigger io. Note that once io is initiated, TREE_REF can no
3365 : * longer be cleared, so that is the moment at which any such race is
3366 : * best fixed.
3367 : */
3368 907408038 : refs = atomic_read(&eb->refs);
3369 1803312150 : if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3370 : return;
3371 :
3372 11509986 : spin_lock(&eb->refs_lock);
3373 11503931 : if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3374 11503885 : atomic_inc(&eb->refs);
3375 11503928 : spin_unlock(&eb->refs_lock);
3376 : }
3377 :
3378 584750267 : static void mark_extent_buffer_accessed(struct extent_buffer *eb,
3379 : struct page *accessed)
3380 : {
3381 584750267 : int num_pages, i;
3382 :
3383 584750267 : check_buffer_tree_ref(eb);
3384 :
3385 584606581 : num_pages = num_extent_pages(eb);
3386 2909051854 : for (i = 0; i < num_pages; i++) {
3387 2324542748 : struct page *p = eb->pages[i];
3388 :
3389 2324542748 : if (p != accessed)
3390 2324542725 : mark_page_accessed(p);
3391 : }
3392 584509106 : }
3393 :
3394 597217022 : struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
3395 : u64 start)
3396 : {
3397 597217022 : struct extent_buffer *eb;
3398 :
3399 597217022 : eb = find_extent_buffer_nolock(fs_info, start);
3400 597895659 : if (!eb)
3401 : return NULL;
3402 : /*
3403 : * Lock our eb's refs_lock to avoid races with free_extent_buffer().
3404 : * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
3405 : * another task running free_extent_buffer() might have seen that flag
3406 : * set, eb->refs == 2, that the buffer isn't under IO (dirty and
3407 : * writeback flags not set) and it's still in the tree (flag
3408 : * EXTENT_BUFFER_TREE_REF set), therefore being in the process of
3409 : * decrementing the extent buffer's reference count twice. So here we
3410 : * could race and increment the eb's reference count, clear its stale
3411 : * flag, mark it as dirty and drop our reference before the other task
3412 : * finishes executing free_extent_buffer, which would later result in
3413 : * an attempt to free an extent buffer that is dirty.
3414 : */
3415 1169606664 : if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
3416 1214776 : spin_lock(&eb->refs_lock);
3417 1214764 : spin_unlock(&eb->refs_lock);
3418 : }
3419 584803325 : mark_extent_buffer_accessed(eb, NULL);
3420 584803325 : return eb;
3421 : }
3422 :
3423 : #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3424 : struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
3425 : u64 start)
3426 : {
3427 : struct extent_buffer *eb, *exists = NULL;
3428 : int ret;
3429 :
3430 : eb = find_extent_buffer(fs_info, start);
3431 : if (eb)
3432 : return eb;
3433 : eb = alloc_dummy_extent_buffer(fs_info, start);
3434 : if (!eb)
3435 : return ERR_PTR(-ENOMEM);
3436 : eb->fs_info = fs_info;
3437 : again:
3438 : ret = radix_tree_preload(GFP_NOFS);
3439 : if (ret) {
3440 : exists = ERR_PTR(ret);
3441 : goto free_eb;
3442 : }
3443 : spin_lock(&fs_info->buffer_lock);
3444 : ret = radix_tree_insert(&fs_info->buffer_radix,
3445 : start >> fs_info->sectorsize_bits, eb);
3446 : spin_unlock(&fs_info->buffer_lock);
3447 : radix_tree_preload_end();
3448 : if (ret == -EEXIST) {
3449 : exists = find_extent_buffer(fs_info, start);
3450 : if (exists)
3451 : goto free_eb;
3452 : else
3453 : goto again;
3454 : }
3455 : check_buffer_tree_ref(eb);
3456 : set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
3457 :
3458 : return eb;
3459 : free_eb:
3460 : btrfs_release_extent_buffer(eb);
3461 : return exists;
3462 : }
3463 : #endif
3464 :
3465 45010151 : static struct extent_buffer *grab_extent_buffer(
3466 : struct btrfs_fs_info *fs_info, struct page *page)
3467 : {
3468 45010151 : struct extent_buffer *exists;
3469 :
3470 : /*
3471 : * For subpage case, we completely rely on radix tree to ensure we
3472 : * don't try to insert two ebs for the same bytenr. So here we always
3473 : * return NULL and just continue.
3474 : */
3475 45010151 : if (fs_info->nodesize < PAGE_SIZE)
3476 : return NULL;
3477 :
3478 : /* Page not yet attached to an extent buffer */
3479 45010151 : if (!PagePrivate(page))
3480 : return NULL;
3481 :
3482 : /*
3483 : * We could have already allocated an eb for this page and attached one
3484 : * so lets see if we can get a ref on the existing eb, and if we can we
3485 : * know it's good and we can just return that one, else we know we can
3486 : * just overwrite page->private.
3487 : */
3488 23 : exists = (struct extent_buffer *)page->private;
3489 46 : if (atomic_inc_not_zero(&exists->refs))
3490 : return exists;
3491 :
3492 0 : WARN_ON(PageDirty(page));
3493 0 : detach_page_private(page);
3494 0 : return NULL;
3495 : }
3496 :
3497 51920633 : static int check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
3498 : {
3499 51920633 : if (!IS_ALIGNED(start, fs_info->sectorsize)) {
3500 0 : btrfs_err(fs_info, "bad tree block start %llu", start);
3501 0 : return -EINVAL;
3502 : }
3503 :
3504 51920633 : if (fs_info->nodesize < PAGE_SIZE &&
3505 0 : offset_in_page(start) + fs_info->nodesize > PAGE_SIZE) {
3506 0 : btrfs_err(fs_info,
3507 : "tree block crosses page boundary, start %llu nodesize %u",
3508 : start, fs_info->nodesize);
3509 0 : return -EINVAL;
3510 : }
3511 51920633 : if (fs_info->nodesize >= PAGE_SIZE &&
3512 51920633 : !PAGE_ALIGNED(start)) {
3513 0 : btrfs_err(fs_info,
3514 : "tree block is not page aligned, start %llu nodesize %u",
3515 : start, fs_info->nodesize);
3516 0 : return -EINVAL;
3517 : }
3518 : return 0;
3519 : }
3520 :
3521 51920594 : struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
3522 : u64 start, u64 owner_root, int level)
3523 : {
3524 51920594 : unsigned long len = fs_info->nodesize;
3525 51920594 : int num_pages;
3526 51920594 : int i;
3527 51920594 : unsigned long index = start >> PAGE_SHIFT;
3528 51920594 : struct extent_buffer *eb;
3529 51920594 : struct extent_buffer *exists = NULL;
3530 51920594 : struct page *p;
3531 51920594 : struct address_space *mapping = fs_info->btree_inode->i_mapping;
3532 51920594 : u64 lockdep_owner = owner_root;
3533 51920594 : int uptodate = 1;
3534 51920594 : int ret;
3535 :
3536 51920594 : if (check_eb_alignment(fs_info, start))
3537 : return ERR_PTR(-EINVAL);
3538 :
3539 : #if BITS_PER_LONG == 32
3540 : if (start >= MAX_LFS_FILESIZE) {
3541 : btrfs_err_rl(fs_info,
3542 : "extent buffer %llu is beyond 32bit page cache limit", start);
3543 : btrfs_err_32bit_limit(fs_info);
3544 : return ERR_PTR(-EOVERFLOW);
3545 : }
3546 : if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3547 : btrfs_warn_32bit_limit(fs_info);
3548 : #endif
3549 :
3550 51920200 : eb = find_extent_buffer(fs_info, start);
3551 51920388 : if (eb)
3552 : return eb;
3553 :
3554 11503438 : eb = __alloc_extent_buffer(fs_info, start, len);
3555 11503322 : if (!eb)
3556 : return ERR_PTR(-ENOMEM);
3557 :
3558 : /*
3559 : * The reloc trees are just snapshots, so we need them to appear to be
3560 : * just like any other fs tree WRT lockdep.
3561 : */
3562 11503322 : if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3563 : lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3564 :
3565 11503322 : btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level);
3566 :
3567 11503322 : num_pages = num_extent_pages(eb);
3568 56512696 : for (i = 0; i < num_pages; i++, index++) {
3569 45008817 : struct btrfs_subpage *prealloc = NULL;
3570 :
3571 45008817 : p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
3572 45007475 : if (!p) {
3573 0 : exists = ERR_PTR(-ENOMEM);
3574 0 : goto free_eb;
3575 : }
3576 :
3577 : /*
3578 : * Preallocate page->private for subpage case, so that we won't
3579 : * allocate memory with private_lock hold. The memory will be
3580 : * freed by attach_extent_buffer_page() or freed manually if
3581 : * we exit earlier.
3582 : *
3583 : * Although we have ensured one subpage eb can only have one
3584 : * page, but it may change in the future for 16K page size
3585 : * support, so we still preallocate the memory in the loop.
3586 : */
3587 45007475 : if (fs_info->nodesize < PAGE_SIZE) {
3588 0 : prealloc = btrfs_alloc_subpage(fs_info, BTRFS_SUBPAGE_METADATA);
3589 0 : if (IS_ERR(prealloc)) {
3590 0 : ret = PTR_ERR(prealloc);
3591 0 : unlock_page(p);
3592 0 : put_page(p);
3593 0 : exists = ERR_PTR(ret);
3594 0 : goto free_eb;
3595 : }
3596 : }
3597 :
3598 45007475 : spin_lock(&mapping->private_lock);
3599 45010151 : exists = grab_extent_buffer(fs_info, p);
3600 45010151 : if (exists) {
3601 23 : spin_unlock(&mapping->private_lock);
3602 23 : unlock_page(p);
3603 23 : put_page(p);
3604 23 : mark_extent_buffer_accessed(exists, p);
3605 23 : btrfs_free_subpage(prealloc);
3606 23 : goto free_eb;
3607 : }
3608 : /* Should not fail, as we have preallocated the memory */
3609 45010128 : ret = attach_extent_buffer_page(eb, p, prealloc);
3610 45010128 : ASSERT(!ret);
3611 : /*
3612 : * To inform we have extra eb under allocation, so that
3613 : * detach_extent_buffer_page() won't release the page private
3614 : * when the eb hasn't yet been inserted into radix tree.
3615 : *
3616 : * The ref will be decreased when the eb released the page, in
3617 : * detach_extent_buffer_page().
3618 : * Thus needs no special handling in error path.
3619 : */
3620 45010128 : btrfs_page_inc_eb_refs(fs_info, p);
3621 45010128 : spin_unlock(&mapping->private_lock);
3622 :
3623 45010124 : WARN_ON(btrfs_page_test_dirty(fs_info, p, eb->start, eb->len));
3624 45009870 : eb->pages[i] = p;
3625 45009870 : if (!btrfs_page_test_uptodate(fs_info, p, eb->start, eb->len))
3626 13059577 : uptodate = 0;
3627 :
3628 : /*
3629 : * We can't unlock the pages just yet since the extent buffer
3630 : * hasn't been properly inserted in the radix tree, this
3631 : * opens a race with btree_release_folio which can free a page
3632 : * while we are still filling in all pages for the buffer and
3633 : * we could crash.
3634 : */
3635 : }
3636 11503879 : if (uptodate)
3637 8192805 : set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3638 3311074 : again:
3639 11503915 : ret = radix_tree_preload(GFP_NOFS);
3640 11503331 : if (ret) {
3641 0 : exists = ERR_PTR(ret);
3642 0 : goto free_eb;
3643 : }
3644 :
3645 11503331 : spin_lock(&fs_info->buffer_lock);
3646 11503944 : ret = radix_tree_insert(&fs_info->buffer_radix,
3647 11503944 : start >> fs_info->sectorsize_bits, eb);
3648 11503944 : spin_unlock(&fs_info->buffer_lock);
3649 11503943 : radix_tree_preload_end();
3650 11503938 : if (ret == -EEXIST) {
3651 0 : exists = find_extent_buffer(fs_info, start);
3652 0 : if (exists)
3653 0 : goto free_eb;
3654 : else
3655 0 : goto again;
3656 : }
3657 : /* add one reference for the tree */
3658 11503938 : check_buffer_tree_ref(eb);
3659 11503885 : set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
3660 :
3661 : /*
3662 : * Now it's safe to unlock the pages because any calls to
3663 : * btree_release_folio will correctly detect that a page belongs to a
3664 : * live buffer and won't free them prematurely.
3665 : */
3666 68016214 : for (i = 0; i < num_pages; i++)
3667 45008512 : unlock_page(eb->pages[i]);
3668 : return eb;
3669 :
3670 23 : free_eb:
3671 23 : WARN_ON(!atomic_dec_and_test(&eb->refs));
3672 115 : for (i = 0; i < num_pages; i++) {
3673 92 : if (eb->pages[i])
3674 0 : unlock_page(eb->pages[i]);
3675 : }
3676 :
3677 23 : btrfs_release_extent_buffer(eb);
3678 23 : return exists;
3679 : }
3680 :
3681 30779703 : static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3682 : {
3683 30779703 : struct extent_buffer *eb =
3684 30779703 : container_of(head, struct extent_buffer, rcu_head);
3685 :
3686 30779703 : __free_extent_buffer(eb);
3687 30802181 : }
3688 :
3689 1033711467 : static int release_extent_buffer(struct extent_buffer *eb)
3690 : __releases(&eb->refs_lock)
3691 : {
3692 1033711467 : lockdep_assert_held(&eb->refs_lock);
3693 :
3694 1033711467 : WARN_ON(atomic_read(&eb->refs) == 0);
3695 1033711467 : if (atomic_dec_and_test(&eb->refs)) {
3696 30856810 : if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
3697 11503837 : struct btrfs_fs_info *fs_info = eb->fs_info;
3698 :
3699 11503837 : spin_unlock(&eb->refs_lock);
3700 :
3701 11503857 : spin_lock(&fs_info->buffer_lock);
3702 11503939 : radix_tree_delete(&fs_info->buffer_radix,
3703 11503939 : eb->start >> fs_info->sectorsize_bits);
3704 11503939 : spin_unlock(&fs_info->buffer_lock);
3705 : } else {
3706 19353610 : spin_unlock(&eb->refs_lock);
3707 : }
3708 :
3709 30857521 : btrfs_leak_debug_del_eb(eb);
3710 : /* Should be safe to release our pages at this point */
3711 30857521 : btrfs_release_extent_buffer_pages(eb);
3712 : #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3713 : if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
3714 : __free_extent_buffer(eb);
3715 : return 1;
3716 : }
3717 : #endif
3718 30857078 : call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
3719 30857078 : return 1;
3720 : }
3721 1002593059 : spin_unlock(&eb->refs_lock);
3722 :
3723 1002593059 : return 0;
3724 : }
3725 :
3726 1158949291 : void free_extent_buffer(struct extent_buffer *eb)
3727 : {
3728 1158949291 : int refs;
3729 1158949291 : if (!eb)
3730 : return;
3731 :
3732 1158800233 : refs = atomic_read(&eb->refs);
3733 1159203410 : while (1) {
3734 1159203410 : if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
3735 158115772 : || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
3736 : refs == 1))
3737 : break;
3738 277586440 : if (atomic_try_cmpxchg(&eb->refs, &refs, refs - 1))
3739 : return;
3740 : }
3741 :
3742 1020436737 : spin_lock(&eb->refs_lock);
3743 1580918031 : if (atomic_read(&eb->refs) == 2 &&
3744 1118020 : test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
3745 2054501 : !extent_buffer_under_io(eb) &&
3746 : test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3747 936481 : atomic_dec(&eb->refs);
3748 :
3749 : /*
3750 : * I know this is terrible, but it's temporary until we stop tracking
3751 : * the uptodate bits and such for the extent buffers.
3752 : */
3753 1021679812 : release_extent_buffer(eb);
3754 : }
3755 :
3756 9186051 : void free_extent_buffer_stale(struct extent_buffer *eb)
3757 : {
3758 9186051 : if (!eb)
3759 : return;
3760 :
3761 9186051 : spin_lock(&eb->refs_lock);
3762 9186077 : set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
3763 :
3764 16730850 : if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
3765 : test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3766 7544761 : atomic_dec(&eb->refs);
3767 9186086 : release_extent_buffer(eb);
3768 : }
3769 :
3770 569335 : static void btree_clear_page_dirty(struct page *page)
3771 : {
3772 569335 : ASSERT(PageDirty(page));
3773 569335 : ASSERT(PageLocked(page));
3774 569335 : clear_page_dirty_for_io(page);
3775 569336 : xa_lock_irq(&page->mapping->i_pages);
3776 569336 : if (!PageDirty(page))
3777 569336 : __xa_clear_mark(&page->mapping->i_pages,
3778 : page_index(page), PAGECACHE_TAG_DIRTY);
3779 569336 : xa_unlock_irq(&page->mapping->i_pages);
3780 569336 : }
3781 :
3782 0 : static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb)
3783 : {
3784 0 : struct btrfs_fs_info *fs_info = eb->fs_info;
3785 0 : struct page *page = eb->pages[0];
3786 0 : bool last;
3787 :
3788 : /* btree_clear_page_dirty() needs page locked */
3789 0 : lock_page(page);
3790 0 : last = btrfs_subpage_clear_and_test_dirty(fs_info, page, eb->start,
3791 0 : eb->len);
3792 0 : if (last)
3793 0 : btree_clear_page_dirty(page);
3794 0 : unlock_page(page);
3795 0 : WARN_ON(atomic_read(&eb->refs) == 0);
3796 0 : }
3797 :
3798 18654342 : void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
3799 : struct extent_buffer *eb)
3800 : {
3801 18654342 : struct btrfs_fs_info *fs_info = eb->fs_info;
3802 18654342 : int i;
3803 18654342 : int num_pages;
3804 18654342 : struct page *page;
3805 :
3806 18654342 : btrfs_assert_tree_write_locked(eb);
3807 :
3808 18654342 : if (trans && btrfs_header_generation(eb) != trans->transid)
3809 : return;
3810 :
3811 11513807 : if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
3812 : return;
3813 :
3814 143207 : percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len,
3815 : fs_info->dirty_metadata_batch);
3816 :
3817 143207 : if (eb->fs_info->nodesize < PAGE_SIZE)
3818 0 : return clear_subpage_extent_buffer_dirty(eb);
3819 :
3820 143207 : num_pages = num_extent_pages(eb);
3821 :
3822 712543 : for (i = 0; i < num_pages; i++) {
3823 569336 : page = eb->pages[i];
3824 569336 : if (!PageDirty(page))
3825 0 : continue;
3826 569334 : lock_page(page);
3827 569335 : btree_clear_page_dirty(page);
3828 569336 : unlock_page(page);
3829 : }
3830 143207 : WARN_ON(atomic_read(&eb->refs) == 0);
3831 : }
3832 :
3833 312065141 : void set_extent_buffer_dirty(struct extent_buffer *eb)
3834 : {
3835 312065141 : int i;
3836 312065141 : int num_pages;
3837 312065141 : bool was_dirty;
3838 :
3839 312065141 : check_buffer_tree_ref(eb);
3840 :
3841 312052383 : was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
3842 :
3843 312168630 : num_pages = num_extent_pages(eb);
3844 312168630 : WARN_ON(atomic_read(&eb->refs) == 0);
3845 312168630 : WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
3846 :
3847 312168630 : if (!was_dirty) {
3848 9449718 : bool subpage = eb->fs_info->nodesize < PAGE_SIZE;
3849 :
3850 : /*
3851 : * For subpage case, we can have other extent buffers in the
3852 : * same page, and in clear_subpage_extent_buffer_dirty() we
3853 : * have to clear page dirty without subpage lock held.
3854 : * This can cause race where our page gets dirty cleared after
3855 : * we just set it.
3856 : *
3857 : * Thankfully, clear_subpage_extent_buffer_dirty() has locked
3858 : * its page for other reasons, we can use page lock to prevent
3859 : * the above race.
3860 : */
3861 9449718 : if (subpage)
3862 0 : lock_page(eb->pages[0]);
3863 46216512 : for (i = 0; i < num_pages; i++)
3864 36766596 : btrfs_page_set_dirty(eb->fs_info, eb->pages[i],
3865 36766596 : eb->start, eb->len);
3866 9449916 : if (subpage)
3867 0 : unlock_page(eb->pages[0]);
3868 9449916 : percpu_counter_add_batch(&eb->fs_info->dirty_metadata_bytes,
3869 9449916 : eb->len,
3870 9449916 : eb->fs_info->dirty_metadata_batch);
3871 : }
3872 : #ifdef CONFIG_BTRFS_DEBUG
3873 : for (i = 0; i < num_pages; i++)
3874 : ASSERT(PageDirty(eb->pages[i]));
3875 : #endif
3876 312168821 : }
3877 :
3878 3219 : void clear_extent_buffer_uptodate(struct extent_buffer *eb)
3879 : {
3880 3219 : struct btrfs_fs_info *fs_info = eb->fs_info;
3881 3219 : struct page *page;
3882 3219 : int num_pages;
3883 3219 : int i;
3884 :
3885 3219 : clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3886 3219 : num_pages = num_extent_pages(eb);
3887 16317 : for (i = 0; i < num_pages; i++) {
3888 13098 : page = eb->pages[i];
3889 13098 : if (!page)
3890 0 : continue;
3891 :
3892 : /*
3893 : * This is special handling for metadata subpage, as regular
3894 : * btrfs_is_subpage() can not handle cloned/dummy metadata.
3895 : */
3896 13098 : if (fs_info->nodesize >= PAGE_SIZE)
3897 13098 : ClearPageUptodate(page);
3898 : else
3899 0 : btrfs_subpage_clear_uptodate(fs_info, page, eb->start,
3900 0 : eb->len);
3901 : }
3902 3219 : }
3903 :
3904 28900214 : void set_extent_buffer_uptodate(struct extent_buffer *eb)
3905 : {
3906 28900214 : struct btrfs_fs_info *fs_info = eb->fs_info;
3907 28900214 : struct page *page;
3908 28900214 : int num_pages;
3909 28900214 : int i;
3910 :
3911 28900214 : set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3912 28901123 : num_pages = num_extent_pages(eb);
3913 143462937 : for (i = 0; i < num_pages; i++) {
3914 114562464 : page = eb->pages[i];
3915 :
3916 : /*
3917 : * This is special handling for metadata subpage, as regular
3918 : * btrfs_is_subpage() can not handle cloned/dummy metadata.
3919 : */
3920 114562464 : if (fs_info->nodesize >= PAGE_SIZE)
3921 114562464 : SetPageUptodate(page);
3922 : else
3923 0 : btrfs_subpage_set_uptodate(fs_info, page, eb->start,
3924 0 : eb->len);
3925 : }
3926 28900473 : }
3927 :
3928 94520 : static void extent_buffer_read_end_io(struct btrfs_bio *bbio)
3929 : {
3930 94520 : struct extent_buffer *eb = bbio->private;
3931 94520 : struct btrfs_fs_info *fs_info = eb->fs_info;
3932 94520 : bool uptodate = !bbio->bio.bi_status;
3933 94520 : struct bvec_iter_all iter_all;
3934 94520 : struct bio_vec *bvec;
3935 94520 : u32 bio_offset = 0;
3936 :
3937 94520 : eb->read_mirror = bbio->mirror_num;
3938 :
3939 189039 : if (uptodate &&
3940 94520 : btrfs_validate_extent_buffer(eb, &bbio->parent_check) < 0)
3941 : uptodate = false;
3942 :
3943 94518 : if (uptodate) {
3944 94518 : set_extent_buffer_uptodate(eb);
3945 : } else {
3946 1 : clear_extent_buffer_uptodate(eb);
3947 1 : set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
3948 : }
3949 :
3950 475285 : bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
3951 380767 : u64 start = eb->start + bio_offset;
3952 380767 : struct page *page = bvec->bv_page;
3953 380767 : u32 len = bvec->bv_len;
3954 :
3955 380767 : if (uptodate)
3956 380763 : btrfs_page_set_uptodate(fs_info, page, start, len);
3957 : else
3958 4 : btrfs_page_clear_uptodate(fs_info, page, start, len);
3959 :
3960 380765 : bio_offset += len;
3961 : }
3962 :
3963 94520 : clear_bit(EXTENT_BUFFER_READING, &eb->bflags);
3964 94519 : smp_mb__after_atomic();
3965 94519 : wake_up_bit(&eb->bflags, EXTENT_BUFFER_READING);
3966 94519 : free_extent_buffer(eb);
3967 :
3968 94520 : bio_put(&bbio->bio);
3969 94520 : }
3970 :
3971 23723911 : int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num,
3972 : struct btrfs_tree_parent_check *check)
3973 : {
3974 23723911 : int num_pages = num_extent_pages(eb), i;
3975 23723911 : struct btrfs_bio *bbio;
3976 :
3977 47447822 : if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3978 : return 0;
3979 :
3980 : /*
3981 : * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
3982 : * operation, which could potentially still be in flight. In this case
3983 : * we simply want to return an error.
3984 : */
3985 94709 : if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
3986 : return -EIO;
3987 :
3988 : /* Someone else is already reading the buffer, just wait for it. */
3989 94706 : if (test_and_set_bit(EXTENT_BUFFER_READING, &eb->bflags))
3990 186 : goto done;
3991 :
3992 94520 : clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
3993 94520 : eb->read_mirror = 0;
3994 94520 : check_buffer_tree_ref(eb);
3995 94520 : atomic_inc(&eb->refs);
3996 :
3997 94520 : bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
3998 : REQ_OP_READ | REQ_META, eb->fs_info,
3999 : extent_buffer_read_end_io, eb);
4000 94520 : bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
4001 94520 : bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
4002 94520 : bbio->file_offset = eb->start;
4003 189040 : memcpy(&bbio->parent_check, check, sizeof(*check));
4004 94520 : if (eb->fs_info->nodesize < PAGE_SIZE) {
4005 0 : __bio_add_page(&bbio->bio, eb->pages[0], eb->len,
4006 0 : eb->start - page_offset(eb->pages[0]));
4007 : } else {
4008 475284 : for (i = 0; i < num_pages; i++)
4009 380764 : __bio_add_page(&bbio->bio, eb->pages[i], PAGE_SIZE, 0);
4010 : }
4011 94520 : btrfs_submit_bio(bbio, mirror_num);
4012 :
4013 94706 : done:
4014 94706 : if (wait == WAIT_COMPLETE) {
4015 85603 : wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
4016 85603 : if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4017 1 : return -EIO;
4018 : }
4019 :
4020 : return 0;
4021 : }
4022 :
4023 0 : static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
4024 : unsigned long len)
4025 : {
4026 0 : btrfs_warn(eb->fs_info,
4027 : "access to eb bytenr %llu len %lu out of range start %lu len %lu",
4028 : eb->start, eb->len, start, len);
4029 0 : WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4030 :
4031 0 : return true;
4032 : }
4033 :
4034 : /*
4035 : * Check if the [start, start + len) range is valid before reading/writing
4036 : * the eb.
4037 : * NOTE: @start and @len are offset inside the eb, not logical address.
4038 : *
4039 : * Caller should not touch the dst/src memory if this function returns error.
4040 : */
4041 10611782847 : static inline int check_eb_range(const struct extent_buffer *eb,
4042 : unsigned long start, unsigned long len)
4043 : {
4044 10611782847 : unsigned long offset;
4045 :
4046 : /* start, start + len should not go beyond eb->len nor overflow */
4047 10611782847 : if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
4048 0 : return report_eb_range(eb, start, len);
4049 :
4050 : return false;
4051 : }
4052 :
4053 9869695326 : void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
4054 : unsigned long start, unsigned long len)
4055 : {
4056 9869695326 : size_t cur;
4057 9869695326 : size_t offset;
4058 9869695326 : struct page *page;
4059 9869695326 : char *kaddr;
4060 9869695326 : char *dst = (char *)dstv;
4061 9869695326 : unsigned long i = get_eb_page_index(start);
4062 :
4063 9869695326 : if (check_eb_range(eb, start, len))
4064 : return;
4065 :
4066 9871183921 : offset = get_eb_offset_in_page(eb, start);
4067 :
4068 19503057556 : while (len > 0) {
4069 9633635781 : page = eb->pages[i];
4070 :
4071 9633635781 : cur = min(len, (PAGE_SIZE - offset));
4072 9633635781 : kaddr = page_address(page);
4073 19267271562 : memcpy(dst, kaddr + offset, cur);
4074 :
4075 9631873635 : dst += cur;
4076 9631873635 : len -= cur;
4077 9631873635 : offset = 0;
4078 9631873635 : i++;
4079 : }
4080 : }
4081 :
4082 7485 : int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
4083 : void __user *dstv,
4084 : unsigned long start, unsigned long len)
4085 : {
4086 7485 : size_t cur;
4087 7485 : size_t offset;
4088 7485 : struct page *page;
4089 7485 : char *kaddr;
4090 7485 : char __user *dst = (char __user *)dstv;
4091 7485 : unsigned long i = get_eb_page_index(start);
4092 7485 : int ret = 0;
4093 :
4094 7485 : WARN_ON(start > eb->len);
4095 7485 : WARN_ON(start + len > eb->start + eb->len);
4096 :
4097 7485 : offset = get_eb_offset_in_page(eb, start);
4098 :
4099 15083 : while (len > 0) {
4100 7598 : page = eb->pages[i];
4101 :
4102 7598 : cur = min(len, (PAGE_SIZE - offset));
4103 7598 : kaddr = page_address(page);
4104 7598 : if (copy_to_user_nofault(dst, kaddr + offset, cur)) {
4105 : ret = -EFAULT;
4106 : break;
4107 : }
4108 :
4109 7598 : dst += cur;
4110 7598 : len -= cur;
4111 7598 : offset = 0;
4112 7598 : i++;
4113 : }
4114 :
4115 7485 : return ret;
4116 : }
4117 :
4118 22437850 : int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
4119 : unsigned long start, unsigned long len)
4120 : {
4121 22437850 : size_t cur;
4122 22437850 : size_t offset;
4123 22437850 : struct page *page;
4124 22437850 : char *kaddr;
4125 22437850 : char *ptr = (char *)ptrv;
4126 22437850 : unsigned long i = get_eb_page_index(start);
4127 22437850 : int ret = 0;
4128 :
4129 22437850 : if (check_eb_range(eb, start, len))
4130 : return -EINVAL;
4131 :
4132 22437724 : offset = get_eb_offset_in_page(eb, start);
4133 :
4134 37219170 : while (len > 0) {
4135 22469305 : page = eb->pages[i];
4136 :
4137 22469305 : cur = min(len, (PAGE_SIZE - offset));
4138 :
4139 22469305 : kaddr = page_address(page);
4140 22469305 : ret = memcmp(ptr, kaddr + offset, cur);
4141 22469340 : if (ret)
4142 : break;
4143 :
4144 14781446 : ptr += cur;
4145 14781446 : len -= cur;
4146 14781446 : offset = 0;
4147 14781446 : i++;
4148 : }
4149 : return ret;
4150 : }
4151 :
4152 : /*
4153 : * Check that the extent buffer is uptodate.
4154 : *
4155 : * For regular sector size == PAGE_SIZE case, check if @page is uptodate.
4156 : * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
4157 : */
4158 322648663 : static void assert_eb_page_uptodate(const struct extent_buffer *eb,
4159 : struct page *page)
4160 : {
4161 322648663 : struct btrfs_fs_info *fs_info = eb->fs_info;
4162 :
4163 : /*
4164 : * If we are using the commit root we could potentially clear a page
4165 : * Uptodate while we're using the extent buffer that we've previously
4166 : * looked up. We don't want to complain in this case, as the page was
4167 : * valid before, we just didn't write it out. Instead we want to catch
4168 : * the case where we didn't actually read the block properly, which
4169 : * would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
4170 : */
4171 645297326 : if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
4172 : return;
4173 :
4174 322648663 : if (fs_info->nodesize < PAGE_SIZE) {
4175 0 : if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, page,
4176 : eb->start, eb->len)))
4177 0 : btrfs_subpage_dump_bitmap(fs_info, page, eb->start, eb->len);
4178 : } else {
4179 322648663 : WARN_ON(!PageUptodate(page));
4180 : }
4181 : }
4182 :
4183 9448284 : void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb,
4184 : const void *srcv)
4185 : {
4186 9448284 : char *kaddr;
4187 :
4188 9448284 : assert_eb_page_uptodate(eb, eb->pages[0]);
4189 9448188 : kaddr = page_address(eb->pages[0]) +
4190 : get_eb_offset_in_page(eb, offsetof(struct btrfs_header,
4191 : chunk_tree_uuid));
4192 18896376 : memcpy(kaddr, srcv, BTRFS_FSID_SIZE);
4193 9448188 : }
4194 :
4195 18514049 : void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv)
4196 : {
4197 18514049 : char *kaddr;
4198 :
4199 18514049 : assert_eb_page_uptodate(eb, eb->pages[0]);
4200 18513577 : kaddr = page_address(eb->pages[0]) +
4201 : get_eb_offset_in_page(eb, offsetof(struct btrfs_header, fsid));
4202 37027154 : memcpy(kaddr, srcv, BTRFS_FSID_SIZE);
4203 18513577 : }
4204 :
4205 153077400 : void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
4206 : unsigned long start, unsigned long len)
4207 : {
4208 153077400 : size_t cur;
4209 153077400 : size_t offset;
4210 153077400 : struct page *page;
4211 153077400 : char *kaddr;
4212 153077400 : char *src = (char *)srcv;
4213 153077400 : unsigned long i = get_eb_page_index(start);
4214 :
4215 153077400 : WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags));
4216 :
4217 153077400 : if (check_eb_range(eb, start, len))
4218 : return;
4219 :
4220 153066576 : offset = get_eb_offset_in_page(eb, start);
4221 :
4222 306885311 : while (len > 0) {
4223 153824737 : page = eb->pages[i];
4224 153824737 : assert_eb_page_uptodate(eb, page);
4225 :
4226 153818735 : cur = min(len, PAGE_SIZE - offset);
4227 153818735 : kaddr = page_address(page);
4228 307637470 : memcpy(kaddr + offset, src, cur);
4229 :
4230 153818735 : src += cur;
4231 153818735 : len -= cur;
4232 153818735 : offset = 0;
4233 153818735 : i++;
4234 : }
4235 : }
4236 :
4237 22232283 : void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
4238 : unsigned long len)
4239 : {
4240 22232283 : size_t cur;
4241 22232283 : size_t offset;
4242 22232283 : struct page *page;
4243 22232283 : char *kaddr;
4244 22232283 : unsigned long i = get_eb_page_index(start);
4245 :
4246 22232283 : if (check_eb_range(eb, start, len))
4247 : return;
4248 :
4249 22232084 : offset = get_eb_offset_in_page(eb, start);
4250 :
4251 54609626 : while (len > 0) {
4252 32380486 : page = eb->pages[i];
4253 32380486 : assert_eb_page_uptodate(eb, page);
4254 :
4255 32377542 : cur = min(len, PAGE_SIZE - offset);
4256 32377542 : kaddr = page_address(page);
4257 32377542 : memset(kaddr + offset, 0, cur);
4258 :
4259 32377542 : len -= cur;
4260 32377542 : offset = 0;
4261 32377542 : i++;
4262 : }
4263 : }
4264 :
4265 9065780 : void copy_extent_buffer_full(const struct extent_buffer *dst,
4266 : const struct extent_buffer *src)
4267 : {
4268 9065780 : int i;
4269 9065780 : int num_pages;
4270 :
4271 9065780 : ASSERT(dst->len == src->len);
4272 :
4273 9065780 : if (dst->fs_info->nodesize >= PAGE_SIZE) {
4274 9065780 : num_pages = num_extent_pages(dst);
4275 44333276 : for (i = 0; i < num_pages; i++)
4276 35267554 : copy_page(page_address(dst->pages[i]),
4277 35267554 : page_address(src->pages[i]));
4278 : } else {
4279 0 : size_t src_offset = get_eb_offset_in_page(src, 0);
4280 0 : size_t dst_offset = get_eb_offset_in_page(dst, 0);
4281 :
4282 0 : ASSERT(src->fs_info->nodesize < PAGE_SIZE);
4283 0 : memcpy(page_address(dst->pages[0]) + dst_offset,
4284 : page_address(src->pages[0]) + src_offset,
4285 : src->len);
4286 : }
4287 9065722 : }
4288 :
4289 11081707 : void copy_extent_buffer(const struct extent_buffer *dst,
4290 : const struct extent_buffer *src,
4291 : unsigned long dst_offset, unsigned long src_offset,
4292 : unsigned long len)
4293 : {
4294 11081707 : u64 dst_len = dst->len;
4295 11081707 : size_t cur;
4296 11081707 : size_t offset;
4297 11081707 : struct page *page;
4298 11081707 : char *kaddr;
4299 11081707 : unsigned long i = get_eb_page_index(dst_offset);
4300 :
4301 22163350 : if (check_eb_range(dst, dst_offset, len) ||
4302 11081672 : check_eb_range(src, src_offset, len))
4303 0 : return;
4304 :
4305 11081643 : WARN_ON(src->len != dst_len);
4306 :
4307 11081643 : offset = get_eb_offset_in_page(dst, dst_offset);
4308 :
4309 23505229 : while (len > 0) {
4310 12423524 : page = dst->pages[i];
4311 12423524 : assert_eb_page_uptodate(dst, page);
4312 :
4313 12423457 : cur = min(len, (unsigned long)(PAGE_SIZE - offset));
4314 :
4315 12423457 : kaddr = page_address(page);
4316 12423457 : read_extent_buffer(src, kaddr + offset, src_offset, cur);
4317 :
4318 12423586 : src_offset += cur;
4319 12423586 : len -= cur;
4320 12423586 : offset = 0;
4321 12423586 : i++;
4322 : }
4323 : }
4324 :
4325 : /*
4326 : * eb_bitmap_offset() - calculate the page and offset of the byte containing the
4327 : * given bit number
4328 : * @eb: the extent buffer
4329 : * @start: offset of the bitmap item in the extent buffer
4330 : * @nr: bit number
4331 : * @page_index: return index of the page in the extent buffer that contains the
4332 : * given bit number
4333 : * @page_offset: return offset into the page given by page_index
4334 : *
4335 : * This helper hides the ugliness of finding the byte in an extent buffer which
4336 : * contains a given bit.
4337 : */
4338 : static inline void eb_bitmap_offset(const struct extent_buffer *eb,
4339 : unsigned long start, unsigned long nr,
4340 : unsigned long *page_index,
4341 : size_t *page_offset)
4342 : {
4343 96116266 : size_t byte_offset = BIT_BYTE(nr);
4344 96116266 : size_t offset;
4345 :
4346 : /*
4347 : * The byte we want is the offset of the extent buffer + the offset of
4348 : * the bitmap item in the extent buffer + the offset of the byte in the
4349 : * bitmap item.
4350 : */
4351 96116266 : offset = start + offset_in_page(eb->start) + byte_offset;
4352 :
4353 96116266 : *page_index = offset >> PAGE_SHIFT;
4354 96116266 : *page_offset = offset_in_page(offset);
4355 : }
4356 :
4357 : /*
4358 : * Determine whether a bit in a bitmap item is set.
4359 : *
4360 : * @eb: the extent buffer
4361 : * @start: offset of the bitmap item in the extent buffer
4362 : * @nr: bit number to test
4363 : */
4364 85636554 : int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
4365 : unsigned long nr)
4366 : {
4367 85636554 : u8 *kaddr;
4368 85636554 : struct page *page;
4369 85636554 : unsigned long i;
4370 85636554 : size_t offset;
4371 :
4372 85636554 : eb_bitmap_offset(eb, start, nr, &i, &offset);
4373 85636554 : page = eb->pages[i];
4374 85636554 : assert_eb_page_uptodate(eb, page);
4375 85636552 : kaddr = page_address(page);
4376 85636552 : return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
4377 : }
4378 :
4379 : /*
4380 : * Set an area of a bitmap to 1.
4381 : *
4382 : * @eb: the extent buffer
4383 : * @start: offset of the bitmap item in the extent buffer
4384 : * @pos: bit number of the first bit
4385 : * @len: number of bits to set
4386 : */
4387 5146947 : void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
4388 : unsigned long pos, unsigned long len)
4389 : {
4390 5146947 : u8 *kaddr;
4391 5146947 : struct page *page;
4392 5146947 : unsigned long i;
4393 5146947 : size_t offset;
4394 5146947 : const unsigned int size = pos + len;
4395 5146947 : int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
4396 5146947 : u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(pos);
4397 :
4398 5146947 : eb_bitmap_offset(eb, start, pos, &i, &offset);
4399 5146947 : page = eb->pages[i];
4400 5146947 : assert_eb_page_uptodate(eb, page);
4401 5146947 : kaddr = page_address(page);
4402 :
4403 7658608 : while (len >= bits_to_set) {
4404 2511661 : kaddr[offset] |= mask_to_set;
4405 2511661 : len -= bits_to_set;
4406 2511661 : bits_to_set = BITS_PER_BYTE;
4407 2511661 : mask_to_set = ~0;
4408 2511661 : if (++offset >= PAGE_SIZE && len > 0) {
4409 4 : offset = 0;
4410 4 : page = eb->pages[++i];
4411 4 : assert_eb_page_uptodate(eb, page);
4412 4 : kaddr = page_address(page);
4413 : }
4414 : }
4415 5146947 : if (len) {
4416 2818533 : mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
4417 2818533 : kaddr[offset] |= mask_to_set;
4418 : }
4419 5146947 : }
4420 :
4421 :
4422 : /*
4423 : * Clear an area of a bitmap.
4424 : *
4425 : * @eb: the extent buffer
4426 : * @start: offset of the bitmap item in the extent buffer
4427 : * @pos: bit number of the first bit
4428 : * @len: number of bits to clear
4429 : */
4430 5332765 : void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
4431 : unsigned long start, unsigned long pos,
4432 : unsigned long len)
4433 : {
4434 5332765 : u8 *kaddr;
4435 5332765 : struct page *page;
4436 5332765 : unsigned long i;
4437 5332765 : size_t offset;
4438 5332765 : const unsigned int size = pos + len;
4439 5332765 : int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
4440 5332765 : u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos);
4441 :
4442 5332765 : eb_bitmap_offset(eb, start, pos, &i, &offset);
4443 5332765 : page = eb->pages[i];
4444 5332765 : assert_eb_page_uptodate(eb, page);
4445 5332765 : kaddr = page_address(page);
4446 :
4447 7910527 : while (len >= bits_to_clear) {
4448 2577762 : kaddr[offset] &= ~mask_to_clear;
4449 2577762 : len -= bits_to_clear;
4450 2577762 : bits_to_clear = BITS_PER_BYTE;
4451 2577762 : mask_to_clear = ~0;
4452 2577762 : if (++offset >= PAGE_SIZE && len > 0) {
4453 1 : offset = 0;
4454 1 : page = eb->pages[++i];
4455 1 : assert_eb_page_uptodate(eb, page);
4456 1 : kaddr = page_address(page);
4457 : }
4458 : }
4459 5332765 : if (len) {
4460 2984268 : mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
4461 2984268 : kaddr[offset] &= ~mask_to_clear;
4462 : }
4463 5332765 : }
4464 :
4465 : static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4466 : {
4467 312225091 : unsigned long distance = (src > dst) ? src - dst : dst - src;
4468 312225091 : return distance < len;
4469 : }
4470 :
4471 435986842 : static void copy_pages(struct page *dst_page, struct page *src_page,
4472 : unsigned long dst_off, unsigned long src_off,
4473 : unsigned long len)
4474 : {
4475 435986842 : char *dst_kaddr = page_address(dst_page);
4476 435986842 : char *src_kaddr;
4477 435986842 : int must_memmove = 0;
4478 :
4479 435986842 : if (dst_page != src_page) {
4480 123761751 : src_kaddr = page_address(src_page);
4481 : } else {
4482 312225091 : src_kaddr = dst_kaddr;
4483 624450182 : if (areas_overlap(src_off, dst_off, len))
4484 290432873 : must_memmove = 1;
4485 : }
4486 :
4487 123761751 : if (must_memmove)
4488 580865746 : memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
4489 : else
4490 291107938 : memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
4491 435986842 : }
4492 :
4493 91802449 : void memcpy_extent_buffer(const struct extent_buffer *dst,
4494 : unsigned long dst_offset, unsigned long src_offset,
4495 : unsigned long len)
4496 : {
4497 91802449 : size_t cur;
4498 91802449 : size_t dst_off_in_page;
4499 91802449 : size_t src_off_in_page;
4500 91802449 : unsigned long dst_i;
4501 91802449 : unsigned long src_i;
4502 :
4503 183602096 : if (check_eb_range(dst, dst_offset, len) ||
4504 91800789 : check_eb_range(dst, src_offset, len))
4505 0 : return;
4506 :
4507 310361242 : while (len > 0) {
4508 218554387 : dst_off_in_page = get_eb_offset_in_page(dst, dst_offset);
4509 218554387 : src_off_in_page = get_eb_offset_in_page(dst, src_offset);
4510 :
4511 218554387 : dst_i = get_eb_page_index(dst_offset);
4512 218554387 : src_i = get_eb_page_index(src_offset);
4513 :
4514 218554387 : cur = min(len, (unsigned long)(PAGE_SIZE -
4515 : src_off_in_page));
4516 218554387 : cur = min_t(unsigned long, cur,
4517 : (unsigned long)(PAGE_SIZE - dst_off_in_page));
4518 :
4519 218554387 : copy_pages(dst->pages[dst_i], dst->pages[src_i],
4520 : dst_off_in_page, src_off_in_page, cur);
4521 :
4522 218561595 : src_offset += cur;
4523 218561595 : dst_offset += cur;
4524 218561595 : len -= cur;
4525 : }
4526 : }
4527 :
4528 195409978 : void memmove_extent_buffer(const struct extent_buffer *dst,
4529 : unsigned long dst_offset, unsigned long src_offset,
4530 : unsigned long len)
4531 : {
4532 195409978 : size_t cur;
4533 195409978 : size_t dst_off_in_page;
4534 195409978 : size_t src_off_in_page;
4535 195409978 : unsigned long dst_end = dst_offset + len - 1;
4536 195409978 : unsigned long src_end = src_offset + len - 1;
4537 195409978 : unsigned long dst_i;
4538 195409978 : unsigned long src_i;
4539 :
4540 390809964 : if (check_eb_range(dst, dst_offset, len) ||
4541 195403241 : check_eb_range(dst, src_offset, len))
4542 0 : return;
4543 195399986 : if (dst_offset < src_offset) {
4544 89190900 : memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4545 89190900 : return;
4546 : }
4547 323631065 : while (len > 0) {
4548 217403025 : dst_i = get_eb_page_index(dst_end);
4549 217403025 : src_i = get_eb_page_index(src_end);
4550 :
4551 217403025 : dst_off_in_page = get_eb_offset_in_page(dst, dst_end);
4552 217403025 : src_off_in_page = get_eb_offset_in_page(dst, src_end);
4553 :
4554 217403025 : cur = min_t(unsigned long, len, src_off_in_page + 1);
4555 217403025 : cur = min(cur, dst_off_in_page + 1);
4556 217403025 : copy_pages(dst->pages[dst_i], dst->pages[src_i],
4557 217403025 : dst_off_in_page - cur + 1,
4558 217403025 : src_off_in_page - cur + 1, cur);
4559 :
4560 217421979 : dst_end -= cur;
4561 217421979 : src_end -= cur;
4562 217421979 : len -= cur;
4563 : }
4564 : }
4565 :
4566 : #define GANG_LOOKUP_SIZE 16
4567 0 : static struct extent_buffer *get_next_extent_buffer(
4568 : struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
4569 : {
4570 0 : struct extent_buffer *gang[GANG_LOOKUP_SIZE];
4571 0 : struct extent_buffer *found = NULL;
4572 0 : u64 page_start = page_offset(page);
4573 0 : u64 cur = page_start;
4574 :
4575 0 : ASSERT(in_range(bytenr, page_start, PAGE_SIZE));
4576 : lockdep_assert_held(&fs_info->buffer_lock);
4577 :
4578 0 : while (cur < page_start + PAGE_SIZE) {
4579 0 : int ret;
4580 0 : int i;
4581 :
4582 0 : ret = radix_tree_gang_lookup(&fs_info->buffer_radix,
4583 0 : (void **)gang, cur >> fs_info->sectorsize_bits,
4584 0 : min_t(unsigned int, GANG_LOOKUP_SIZE,
4585 : PAGE_SIZE / fs_info->nodesize));
4586 0 : if (ret == 0)
4587 0 : goto out;
4588 0 : for (i = 0; i < ret; i++) {
4589 : /* Already beyond page end */
4590 0 : if (gang[i]->start >= page_start + PAGE_SIZE)
4591 0 : goto out;
4592 : /* Found one */
4593 0 : if (gang[i]->start >= bytenr) {
4594 0 : found = gang[i];
4595 0 : goto out;
4596 : }
4597 : }
4598 0 : cur = gang[ret - 1]->start + gang[ret - 1]->len;
4599 : }
4600 0 : out:
4601 0 : return found;
4602 : }
4603 :
4604 0 : static int try_release_subpage_extent_buffer(struct page *page)
4605 : {
4606 0 : struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
4607 0 : u64 cur = page_offset(page);
4608 0 : const u64 end = page_offset(page) + PAGE_SIZE;
4609 0 : int ret;
4610 :
4611 0 : while (cur < end) {
4612 0 : struct extent_buffer *eb = NULL;
4613 :
4614 : /*
4615 : * Unlike try_release_extent_buffer() which uses page->private
4616 : * to grab buffer, for subpage case we rely on radix tree, thus
4617 : * we need to ensure radix tree consistency.
4618 : *
4619 : * We also want an atomic snapshot of the radix tree, thus go
4620 : * with spinlock rather than RCU.
4621 : */
4622 0 : spin_lock(&fs_info->buffer_lock);
4623 0 : eb = get_next_extent_buffer(fs_info, page, cur);
4624 0 : if (!eb) {
4625 : /* No more eb in the page range after or at cur */
4626 0 : spin_unlock(&fs_info->buffer_lock);
4627 : break;
4628 : }
4629 0 : cur = eb->start + eb->len;
4630 :
4631 : /*
4632 : * The same as try_release_extent_buffer(), to ensure the eb
4633 : * won't disappear out from under us.
4634 : */
4635 0 : spin_lock(&eb->refs_lock);
4636 0 : if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4637 0 : spin_unlock(&eb->refs_lock);
4638 0 : spin_unlock(&fs_info->buffer_lock);
4639 : break;
4640 : }
4641 0 : spin_unlock(&fs_info->buffer_lock);
4642 :
4643 : /*
4644 : * If tree ref isn't set then we know the ref on this eb is a
4645 : * real ref, so just return, this eb will likely be freed soon
4646 : * anyway.
4647 : */
4648 0 : if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4649 0 : spin_unlock(&eb->refs_lock);
4650 : break;
4651 : }
4652 :
4653 : /*
4654 : * Here we don't care about the return value, we will always
4655 : * check the page private at the end. And
4656 : * release_extent_buffer() will release the refs_lock.
4657 : */
4658 0 : release_extent_buffer(eb);
4659 : }
4660 : /*
4661 : * Finally to check if we have cleared page private, as if we have
4662 : * released all ebs in the page, the page private should be cleared now.
4663 : */
4664 0 : spin_lock(&page->mapping->private_lock);
4665 0 : if (!PagePrivate(page))
4666 : ret = 1;
4667 : else
4668 0 : ret = 0;
4669 0 : spin_unlock(&page->mapping->private_lock);
4670 0 : return ret;
4671 :
4672 : }
4673 :
4674 3225583 : int try_release_extent_buffer(struct page *page)
4675 : {
4676 3225583 : struct extent_buffer *eb;
4677 :
4678 3225583 : if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
4679 0 : return try_release_subpage_extent_buffer(page);
4680 :
4681 : /*
4682 : * We need to make sure nobody is changing page->private, as we rely on
4683 : * page->private as the pointer to extent buffer.
4684 : */
4685 3225583 : spin_lock(&page->mapping->private_lock);
4686 3225583 : if (!PagePrivate(page)) {
4687 0 : spin_unlock(&page->mapping->private_lock);
4688 0 : return 1;
4689 : }
4690 :
4691 3225583 : eb = (struct extent_buffer *)page->private;
4692 3225583 : BUG_ON(!eb);
4693 :
4694 : /*
4695 : * This is a little awful but should be ok, we need to make sure that
4696 : * the eb doesn't disappear out from under us while we're looking at
4697 : * this page.
4698 : */
4699 3225583 : spin_lock(&eb->refs_lock);
4700 3225583 : if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4701 202908 : spin_unlock(&eb->refs_lock);
4702 202908 : spin_unlock(&page->mapping->private_lock);
4703 202908 : return 0;
4704 : }
4705 3022675 : spin_unlock(&page->mapping->private_lock);
4706 :
4707 : /*
4708 : * If tree ref isn't set then we know the ref on this eb is a real ref,
4709 : * so just return, this page will likely be freed soon anyway.
4710 : */
4711 3022675 : if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4712 0 : spin_unlock(&eb->refs_lock);
4713 0 : return 0;
4714 : }
4715 :
4716 3022675 : return release_extent_buffer(eb);
4717 : }
4718 :
4719 : /*
4720 : * btrfs_readahead_tree_block - attempt to readahead a child block
4721 : * @fs_info: the fs_info
4722 : * @bytenr: bytenr to read
4723 : * @owner_root: objectid of the root that owns this eb
4724 : * @gen: generation for the uptodate check, can be 0
4725 : * @level: level for the eb
4726 : *
4727 : * Attempt to readahead a tree block at @bytenr. If @gen is 0 then we do a
4728 : * normal uptodate check of the eb, without checking the generation. If we have
4729 : * to read the block we will not block on anything.
4730 : */
4731 17334628 : void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
4732 : u64 bytenr, u64 owner_root, u64 gen, int level)
4733 : {
4734 17334628 : struct btrfs_tree_parent_check check = {
4735 : .has_first_key = 0,
4736 : .level = level,
4737 : .transid = gen
4738 : };
4739 17334628 : struct extent_buffer *eb;
4740 17334628 : int ret;
4741 :
4742 17334628 : eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
4743 17334616 : if (IS_ERR(eb))
4744 17325927 : return;
4745 :
4746 17334616 : if (btrfs_buffer_uptodate(eb, gen, 1)) {
4747 17325300 : free_extent_buffer(eb);
4748 17325300 : return;
4749 : }
4750 :
4751 9103 : ret = read_extent_buffer_pages(eb, WAIT_NONE, 0, &check);
4752 9103 : if (ret < 0)
4753 0 : free_extent_buffer_stale(eb);
4754 : else
4755 9103 : free_extent_buffer(eb);
4756 : }
4757 :
4758 : /*
4759 : * btrfs_readahead_node_child - readahead a node's child block
4760 : * @node: parent node we're reading from
4761 : * @slot: slot in the parent node for the child we want to read
4762 : *
4763 : * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
4764 : * the slot in the node provided.
4765 : */
4766 15316114 : void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
4767 : {
4768 30632104 : btrfs_readahead_tree_block(node->fs_info,
4769 : btrfs_node_blockptr(node, slot),
4770 : btrfs_header_owner(node),
4771 : btrfs_node_ptr_generation(node, slot),
4772 15316114 : btrfs_header_level(node) - 1);
4773 15316244 : }
|
