Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 :
3 : #include <linux/bitops.h>
4 : #include <linux/slab.h>
5 : #include <linux/bio.h>
6 : #include <linux/mm.h>
7 : #include <linux/pagemap.h>
8 : #include <linux/page-flags.h>
9 : #include <linux/sched/mm.h>
10 : #include <linux/spinlock.h>
11 : #include <linux/blkdev.h>
12 : #include <linux/swap.h>
13 : #include <linux/writeback.h>
14 : #include <linux/pagevec.h>
15 : #include <linux/prefetch.h>
16 : #include <linux/fsverity.h>
17 : #include "misc.h"
18 : #include "extent_io.h"
19 : #include "extent-io-tree.h"
20 : #include "extent_map.h"
21 : #include "ctree.h"
22 : #include "btrfs_inode.h"
23 : #include "bio.h"
24 : #include "check-integrity.h"
25 : #include "locking.h"
26 : #include "rcu-string.h"
27 : #include "backref.h"
28 : #include "disk-io.h"
29 : #include "subpage.h"
30 : #include "zoned.h"
31 : #include "block-group.h"
32 : #include "compression.h"
33 : #include "fs.h"
34 : #include "accessors.h"
35 : #include "file-item.h"
36 : #include "file.h"
37 : #include "dev-replace.h"
38 : #include "super.h"
39 : #include "transaction.h"
40 :
41 : static struct kmem_cache *extent_buffer_cache;
42 :
43 : #ifdef CONFIG_BTRFS_DEBUG
44 : static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
45 : {
46 : struct btrfs_fs_info *fs_info = eb->fs_info;
47 : unsigned long flags;
48 :
49 : spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
50 : list_add(&eb->leak_list, &fs_info->allocated_ebs);
51 : spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
52 : }
53 :
54 : static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
55 : {
56 : struct btrfs_fs_info *fs_info = eb->fs_info;
57 : unsigned long flags;
58 :
59 : spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
60 : list_del(&eb->leak_list);
61 : spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
62 : }
63 :
64 : void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
65 : {
66 : struct extent_buffer *eb;
67 : unsigned long flags;
68 :
69 : /*
70 : * If we didn't get into open_ctree our allocated_ebs will not be
71 : * initialized, so just skip this.
72 : */
73 : if (!fs_info->allocated_ebs.next)
74 : return;
75 :
76 : WARN_ON(!list_empty(&fs_info->allocated_ebs));
77 : spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
78 : while (!list_empty(&fs_info->allocated_ebs)) {
79 : eb = list_first_entry(&fs_info->allocated_ebs,
80 : struct extent_buffer, leak_list);
81 : pr_err(
82 : "BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n",
83 : eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
84 : btrfs_header_owner(eb));
85 : list_del(&eb->leak_list);
86 : kmem_cache_free(extent_buffer_cache, eb);
87 : }
88 : spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
89 : }
90 : #else
91 : #define btrfs_leak_debug_add_eb(eb) do {} while (0)
92 : #define btrfs_leak_debug_del_eb(eb) do {} while (0)
93 : #endif
94 :
95 : /*
96 : * Structure to record info about the bio being assembled, and other info like
97 : * how many bytes are there before stripe/ordered extent boundary.
98 : */
99 : struct btrfs_bio_ctrl {
100 : struct btrfs_bio *bbio;
101 : enum btrfs_compression_type compress_type;
102 : u32 len_to_oe_boundary;
103 : blk_opf_t opf;
104 : btrfs_bio_end_io_t end_io_func;
105 : struct writeback_control *wbc;
106 : };
107 :
108 18456945 : static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
109 : {
110 18456945 : struct btrfs_bio *bbio = bio_ctrl->bbio;
111 :
112 18456945 : if (!bbio)
113 : return;
114 :
115 : /* Caller should ensure the bio has at least some range added */
116 4307793 : ASSERT(bbio->bio.bi_iter.bi_size);
117 :
118 4307793 : if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ &&
119 1767867 : bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
120 1157 : btrfs_submit_compressed_read(bbio);
121 : else
122 4305981 : btrfs_submit_bio(bbio, 0);
123 :
124 : /* The bbio is owned by the end_io handler now */
125 4308958 : bio_ctrl->bbio = NULL;
126 : }
127 :
128 : /*
129 : * Submit or fail the current bio in the bio_ctrl structure.
130 : */
131 8162124 : static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
132 : {
133 8162124 : struct btrfs_bio *bbio = bio_ctrl->bbio;
134 :
135 8162124 : if (!bbio)
136 : return;
137 :
138 21603 : if (ret) {
139 0 : ASSERT(ret < 0);
140 0 : btrfs_bio_end_io(bbio, errno_to_blk_status(ret));
141 : /* The bio is owned by the end_io handler now */
142 0 : bio_ctrl->bbio = NULL;
143 : } else {
144 21603 : submit_one_bio(bio_ctrl);
145 : }
146 : }
147 :
148 11 : int __init extent_buffer_init_cachep(void)
149 : {
150 11 : extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
151 : sizeof(struct extent_buffer), 0,
152 : SLAB_MEM_SPREAD, NULL);
153 11 : if (!extent_buffer_cache)
154 0 : return -ENOMEM;
155 :
156 : return 0;
157 : }
158 :
159 0 : void __cold extent_buffer_free_cachep(void)
160 : {
161 : /*
162 : * Make sure all delayed rcu free are flushed before we
163 : * destroy caches.
164 : */
165 0 : rcu_barrier();
166 0 : kmem_cache_destroy(extent_buffer_cache);
167 0 : }
168 :
169 39764 : void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
170 : {
171 39764 : unsigned long index = start >> PAGE_SHIFT;
172 39764 : unsigned long end_index = end >> PAGE_SHIFT;
173 39764 : struct page *page;
174 :
175 5095404 : while (index <= end_index) {
176 5055635 : page = find_get_page(inode->i_mapping, index);
177 5055141 : BUG_ON(!page); /* Pages should be in the extent_io_tree */
178 5055141 : clear_page_dirty_for_io(page);
179 5053567 : put_page(page);
180 5055640 : index++;
181 : }
182 39769 : }
183 :
184 5 : void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
185 : {
186 5 : struct address_space *mapping = inode->i_mapping;
187 5 : unsigned long index = start >> PAGE_SHIFT;
188 5 : unsigned long end_index = end >> PAGE_SHIFT;
189 5 : struct folio *folio;
190 :
191 10 : while (index <= end_index) {
192 5 : folio = filemap_get_folio(mapping, index);
193 5 : filemap_dirty_folio(mapping, folio);
194 5 : folio_account_redirty(folio);
195 5 : index += folio_nr_pages(folio);
196 5 : folio_put(folio);
197 : }
198 5 : }
199 :
200 : /*
201 : * Process one page for __process_pages_contig().
202 : *
203 : * Return >0 if we hit @page == @locked_page.
204 : * Return 0 if we updated the page status.
205 : * Return -EGAIN if the we need to try again.
206 : * (For PAGE_LOCK case but got dirty page or page not belong to mapping)
207 : */
208 110331734 : static int process_one_page(struct btrfs_fs_info *fs_info,
209 : struct address_space *mapping,
210 : struct page *page, struct page *locked_page,
211 : unsigned long page_ops, u64 start, u64 end)
212 : {
213 110331734 : u32 len;
214 :
215 110331734 : ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
216 110331734 : len = end + 1 - start;
217 :
218 110331734 : if (page_ops & PAGE_SET_ORDERED)
219 50244067 : btrfs_page_clamp_set_ordered(fs_info, page, start, len);
220 110332989 : if (page_ops & PAGE_START_WRITEBACK) {
221 6138447 : btrfs_page_clamp_clear_dirty(fs_info, page, start, len);
222 6138516 : btrfs_page_clamp_set_writeback(fs_info, page, start, len);
223 : }
224 110332991 : if (page_ops & PAGE_END_WRITEBACK)
225 1077419 : btrfs_page_clamp_clear_writeback(fs_info, page, start, len);
226 :
227 110333041 : if (page == locked_page)
228 : return 1;
229 :
230 105814435 : if (page_ops & PAGE_LOCK) {
231 52909854 : int ret;
232 :
233 52909854 : ret = btrfs_page_start_writer_lock(fs_info, page, start, len);
234 52910903 : if (ret)
235 : return ret;
236 105821427 : if (!PageDirty(page) || page->mapping != mapping) {
237 56 : btrfs_page_end_writer_lock(fs_info, page, start, len);
238 56 : return -EAGAIN;
239 : }
240 : }
241 105815040 : if (page_ops & PAGE_UNLOCK)
242 52911091 : btrfs_page_end_writer_lock(fs_info, page, start, len);
243 : return 0;
244 : }
245 :
246 4768063 : static int __process_pages_contig(struct address_space *mapping,
247 : struct page *locked_page,
248 : u64 start, u64 end, unsigned long page_ops,
249 : u64 *processed_end)
250 : {
251 4768063 : struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb);
252 4768063 : pgoff_t start_index = start >> PAGE_SHIFT;
253 4768063 : pgoff_t end_index = end >> PAGE_SHIFT;
254 4768063 : pgoff_t index = start_index;
255 4768063 : unsigned long pages_processed = 0;
256 4768063 : struct folio_batch fbatch;
257 4768063 : int err = 0;
258 4768063 : int i;
259 :
260 4768063 : if (page_ops & PAGE_LOCK) {
261 : ASSERT(page_ops == PAGE_LOCK);
262 : ASSERT(processed_end && *processed_end == start);
263 : }
264 :
265 4768063 : folio_batch_init(&fbatch);
266 15952822 : while (index <= end_index) {
267 11184811 : int found_folios;
268 :
269 11184811 : found_folios = filemap_get_folios_contig(mapping, &index,
270 : end_index, &fbatch);
271 :
272 11184668 : if (found_folios == 0) {
273 : /*
274 : * Only if we're going to lock these pages, we can find
275 : * nothing at @index.
276 : */
277 0 : ASSERT(page_ops & PAGE_LOCK);
278 0 : err = -EAGAIN;
279 0 : goto out;
280 : }
281 :
282 121517302 : for (i = 0; i < found_folios; i++) {
283 110332634 : int process_ret;
284 110332634 : struct folio *folio = fbatch.folios[i];
285 110332634 : process_ret = process_one_page(fs_info, mapping,
286 : &folio->page, locked_page, page_ops,
287 : start, end);
288 110335164 : if (process_ret < 0) {
289 56 : err = -EAGAIN;
290 56 : folio_batch_release(&fbatch);
291 56 : goto out;
292 : }
293 110335108 : pages_processed += folio_nr_pages(folio);
294 : }
295 11184668 : folio_batch_release(&fbatch);
296 11184913 : cond_resched();
297 : }
298 4768011 : out:
299 4768067 : if (err && processed_end) {
300 : /*
301 : * Update @processed_end. I know this is awful since it has
302 : * two different return value patterns (inclusive vs exclusive).
303 : *
304 : * But the exclusive pattern is necessary if @start is 0, or we
305 : * underflow and check against processed_end won't work as
306 : * expected.
307 : */
308 56 : if (pages_processed)
309 56 : *processed_end = min(end,
310 : ((u64)(start_index + pages_processed) << PAGE_SHIFT) - 1);
311 : else
312 0 : *processed_end = start;
313 : }
314 4768067 : return err;
315 : }
316 :
317 56 : static noinline void __unlock_for_delalloc(struct inode *inode,
318 : struct page *locked_page,
319 : u64 start, u64 end)
320 : {
321 56 : unsigned long index = start >> PAGE_SHIFT;
322 56 : unsigned long end_index = end >> PAGE_SHIFT;
323 :
324 56 : ASSERT(locked_page);
325 56 : if (index == locked_page->index && end_index == index)
326 : return;
327 :
328 6 : __process_pages_contig(inode->i_mapping, locked_page, start, end,
329 : PAGE_UNLOCK, NULL);
330 : }
331 :
332 3472661 : static noinline int lock_delalloc_pages(struct inode *inode,
333 : struct page *locked_page,
334 : u64 delalloc_start,
335 : u64 delalloc_end)
336 : {
337 3472661 : unsigned long index = delalloc_start >> PAGE_SHIFT;
338 3472661 : unsigned long end_index = delalloc_end >> PAGE_SHIFT;
339 3472661 : u64 processed_end = delalloc_start;
340 3472661 : int ret;
341 :
342 3472661 : ASSERT(locked_page);
343 3472661 : if (index == locked_page->index && index == end_index)
344 : return 0;
345 :
346 1046413 : ret = __process_pages_contig(inode->i_mapping, locked_page, delalloc_start,
347 : delalloc_end, PAGE_LOCK, &processed_end);
348 1046377 : if (ret == -EAGAIN && processed_end > delalloc_start)
349 56 : __unlock_for_delalloc(inode, locked_page, delalloc_start,
350 : processed_end);
351 : return ret;
352 : }
353 :
354 : /*
355 : * Find and lock a contiguous range of bytes in the file marked as delalloc, no
356 : * more than @max_bytes.
357 : *
358 : * @start: The original start bytenr to search.
359 : * Will store the extent range start bytenr.
360 : * @end: The original end bytenr of the search range
361 : * Will store the extent range end bytenr.
362 : *
363 : * Return true if we find a delalloc range which starts inside the original
364 : * range, and @start/@end will store the delalloc range start/end.
365 : *
366 : * Return false if we can't find any delalloc range which starts inside the
367 : * original range, and @start/@end will be the non-delalloc range start/end.
368 : */
369 : EXPORT_FOR_TESTS
370 51310127 : noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
371 : struct page *locked_page, u64 *start,
372 : u64 *end)
373 : {
374 51310127 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
375 51310127 : struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
376 51310127 : const u64 orig_start = *start;
377 51310127 : const u64 orig_end = *end;
378 : /* The sanity tests may not set a valid fs_info. */
379 51310127 : u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
380 51310127 : u64 delalloc_start;
381 51310127 : u64 delalloc_end;
382 51310127 : bool found;
383 51310127 : struct extent_state *cached_state = NULL;
384 51310127 : int ret;
385 51310127 : int loops = 0;
386 :
387 : /* Caller should pass a valid @end to indicate the search range end */
388 51310127 : ASSERT(orig_end > orig_start);
389 :
390 : /* The range should at least cover part of the page */
391 51310127 : ASSERT(!(orig_start >= page_offset(locked_page) + PAGE_SIZE ||
392 : orig_end <= page_offset(locked_page)));
393 : again:
394 : /* step one, find a bunch of delalloc bytes starting at start */
395 51310183 : delalloc_start = *start;
396 51310183 : delalloc_end = 0;
397 51310183 : found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
398 : max_bytes, &cached_state);
399 51309377 : if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
400 47836689 : *start = delalloc_start;
401 :
402 : /* @delalloc_end can be -1, never go beyond @orig_end */
403 47836689 : *end = min(delalloc_end, orig_end);
404 47836689 : free_extent_state(cached_state);
405 47836689 : return false;
406 : }
407 :
408 : /*
409 : * start comes from the offset of locked_page. We have to lock
410 : * pages in order, so we can't process delalloc bytes before
411 : * locked_page
412 : */
413 3472688 : if (delalloc_start < *start)
414 21355 : delalloc_start = *start;
415 :
416 : /*
417 : * make sure to limit the number of pages we try to lock down
418 : */
419 3472688 : if (delalloc_end + 1 - delalloc_start > max_bytes)
420 675 : delalloc_end = delalloc_start + max_bytes - 1;
421 :
422 : /* step two, lock all the pages after the page that has start */
423 3472688 : ret = lock_delalloc_pages(inode, locked_page,
424 : delalloc_start, delalloc_end);
425 3472411 : ASSERT(!ret || ret == -EAGAIN);
426 3472411 : if (ret == -EAGAIN) {
427 : /* some of the pages are gone, lets avoid looping by
428 : * shortening the size of the delalloc range we're searching
429 : */
430 56 : free_extent_state(cached_state);
431 56 : cached_state = NULL;
432 56 : if (!loops) {
433 56 : max_bytes = PAGE_SIZE;
434 56 : loops = 1;
435 56 : goto again;
436 : } else {
437 0 : found = false;
438 0 : goto out_failed;
439 : }
440 : }
441 :
442 : /* step three, lock the state bits for the whole range */
443 3472355 : lock_extent(tree, delalloc_start, delalloc_end, &cached_state);
444 :
445 : /* then test to make sure it is all still delalloc */
446 3472572 : ret = test_range_bit(tree, delalloc_start, delalloc_end,
447 : EXTENT_DELALLOC, 1, cached_state);
448 3472442 : if (!ret) {
449 0 : unlock_extent(tree, delalloc_start, delalloc_end,
450 : &cached_state);
451 0 : __unlock_for_delalloc(inode, locked_page,
452 : delalloc_start, delalloc_end);
453 0 : cond_resched();
454 0 : goto again;
455 : }
456 3472442 : free_extent_state(cached_state);
457 3472661 : *start = delalloc_start;
458 3472661 : *end = delalloc_end;
459 : out_failed:
460 : return found;
461 : }
462 :
463 3721796 : void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
464 : struct page *locked_page,
465 : u32 clear_bits, unsigned long page_ops)
466 : {
467 3721796 : clear_extent_bit(&inode->io_tree, start, end, clear_bits, NULL);
468 :
469 3721777 : __process_pages_contig(inode->vfs_inode.i_mapping, locked_page,
470 : start, end, page_ops, NULL);
471 3721721 : }
472 :
473 : static bool btrfs_verify_page(struct page *page, u64 start)
474 : {
475 : if (!fsverity_active(page->mapping->host) ||
476 : PageUptodate(page) ||
477 : start >= i_size_read(page->mapping->host))
478 : return true;
479 : return fsverity_verify_page(page);
480 : }
481 :
482 59115212 : static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len)
483 : {
484 59115212 : struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
485 :
486 59115212 : ASSERT(page_offset(page) <= start &&
487 : start + len <= page_offset(page) + PAGE_SIZE);
488 :
489 59115212 : if (uptodate && btrfs_verify_page(page, start))
490 59115210 : btrfs_page_set_uptodate(fs_info, page, start, len);
491 : else
492 2 : btrfs_page_clear_uptodate(fs_info, page, start, len);
493 :
494 59118038 : if (!btrfs_is_subpage(fs_info, page))
495 59116450 : unlock_page(page);
496 : else
497 0 : btrfs_subpage_end_reader(fs_info, page, start, len);
498 59117242 : }
499 :
500 : /* lots and lots of room for performance fixes in the end_bio funcs */
501 :
502 0 : void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
503 : {
504 0 : struct btrfs_inode *inode;
505 0 : const bool uptodate = (err == 0);
506 0 : int ret = 0;
507 :
508 0 : ASSERT(page && page->mapping);
509 0 : inode = BTRFS_I(page->mapping->host);
510 0 : btrfs_writepage_endio_finish_ordered(inode, page, start, end, uptodate);
511 :
512 0 : if (!uptodate) {
513 0 : const struct btrfs_fs_info *fs_info = inode->root->fs_info;
514 0 : u32 len;
515 :
516 0 : ASSERT(end + 1 - start <= U32_MAX);
517 0 : len = end + 1 - start;
518 :
519 0 : btrfs_page_clear_uptodate(fs_info, page, start, len);
520 0 : ret = err < 0 ? err : -EIO;
521 0 : mapping_set_error(page->mapping, ret);
522 : }
523 0 : }
524 :
525 : /*
526 : * after a writepage IO is done, we need to:
527 : * clear the uptodate bits on error
528 : * clear the writeback bits in the extent tree for this IO
529 : * end_page_writeback if the page has no more pending IO
530 : *
531 : * Scheduling is not allowed, so the extent state tree is expected
532 : * to have one and only one object corresponding to this IO.
533 : */
534 2539546 : static void end_bio_extent_writepage(struct btrfs_bio *bbio)
535 : {
536 2539546 : struct bio *bio = &bbio->bio;
537 2539546 : int error = blk_status_to_errno(bio->bi_status);
538 2539546 : struct bio_vec *bvec;
539 2539546 : struct bvec_iter_all iter_all;
540 :
541 2539546 : ASSERT(!bio_flagged(bio, BIO_CLONED));
542 52774302 : bio_for_each_segment_all(bvec, bio, iter_all) {
543 50234756 : struct page *page = bvec->bv_page;
544 50234756 : struct inode *inode = page->mapping->host;
545 50234756 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
546 50234756 : const u32 sectorsize = fs_info->sectorsize;
547 50234756 : u64 start = page_offset(page) + bvec->bv_offset;
548 50234756 : u32 len = bvec->bv_len;
549 :
550 : /* Our read/write should always be sector aligned. */
551 50234756 : if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
552 0 : btrfs_err(fs_info,
553 : "partial page write in btrfs with offset %u and length %u",
554 : bvec->bv_offset, bvec->bv_len);
555 50234756 : else if (!IS_ALIGNED(bvec->bv_len, sectorsize))
556 0 : btrfs_info(fs_info,
557 : "incomplete page write with offset %u and length %u",
558 : bvec->bv_offset, bvec->bv_len);
559 :
560 50234756 : btrfs_finish_ordered_extent(bbio->ordered, page, start, len, !error);
561 50234756 : if (error) {
562 129537 : btrfs_page_clear_uptodate(fs_info, page, start, len);
563 129537 : mapping_set_error(page->mapping, error);
564 : }
565 50234756 : btrfs_page_clear_writeback(fs_info, page, start, len);
566 : }
567 :
568 2539546 : bio_put(bio);
569 2539546 : }
570 :
571 : /*
572 : * Record previously processed extent range
573 : *
574 : * For endio_readpage_release_extent() to handle a full extent range, reducing
575 : * the extent io operations.
576 : */
577 : struct processed_extent {
578 : struct btrfs_inode *inode;
579 : /* Start of the range in @inode */
580 : u64 start;
581 : /* End of the range in @inode */
582 : u64 end;
583 : bool uptodate;
584 : };
585 :
586 : /*
587 : * Try to release processed extent range
588 : *
589 : * May not release the extent range right now if the current range is
590 : * contiguous to processed extent.
591 : *
592 : * Will release processed extent when any of @inode, @uptodate, the range is
593 : * no longer contiguous to the processed range.
594 : *
595 : * Passing @inode == NULL will force processed extent to be released.
596 : */
597 20842852 : static void endio_readpage_release_extent(struct processed_extent *processed,
598 : struct btrfs_inode *inode, u64 start, u64 end,
599 : bool uptodate)
600 : {
601 20842852 : struct extent_state *cached = NULL;
602 20842852 : struct extent_io_tree *tree;
603 :
604 : /* The first extent, initialize @processed */
605 20842852 : if (!processed->inode)
606 1769632 : goto update;
607 :
608 : /*
609 : * Contiguous to processed extent, just uptodate the end.
610 : *
611 : * Several things to notice:
612 : *
613 : * - bio can be merged as long as on-disk bytenr is contiguous
614 : * This means we can have page belonging to other inodes, thus need to
615 : * check if the inode still matches.
616 : * - bvec can contain range beyond current page for multi-page bvec
617 : * Thus we need to do processed->end + 1 >= start check
618 : */
619 19073220 : if (processed->inode == inode && processed->uptodate == uptodate &&
620 17303882 : processed->end + 1 >= start && end >= processed->end) {
621 17303918 : processed->end = end;
622 17303918 : return;
623 : }
624 :
625 1769302 : tree = &processed->inode->io_tree;
626 : /*
627 : * Now we don't have range contiguous to the processed range, release
628 : * the processed range now.
629 : */
630 1769302 : unlock_extent(tree, processed->start, processed->end, &cached);
631 :
632 3539168 : update:
633 : /* Update processed to current range */
634 3539168 : processed->inode = inode;
635 3539168 : processed->start = start;
636 3539168 : processed->end = end;
637 3539168 : processed->uptodate = uptodate;
638 : }
639 :
640 59101465 : static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page)
641 : {
642 59101465 : ASSERT(PageLocked(page));
643 59085714 : if (!btrfs_is_subpage(fs_info, page))
644 : return;
645 :
646 0 : ASSERT(PagePrivate(page));
647 0 : btrfs_subpage_start_reader(fs_info, page, page_offset(page), PAGE_SIZE);
648 : }
649 :
650 : /*
651 : * after a readpage IO is done, we need to:
652 : * clear the uptodate bits on error
653 : * set the uptodate bits if things worked
654 : * set the page up to date if all extents in the tree are uptodate
655 : * clear the lock bit in the extent tree
656 : * unlock the page if there are no other extents locked for it
657 : *
658 : * Scheduling is not allowed, so the extent state tree is expected
659 : * to have one and only one object corresponding to this IO.
660 : */
661 1769622 : static void end_bio_extent_readpage(struct btrfs_bio *bbio)
662 : {
663 1769622 : struct bio *bio = &bbio->bio;
664 1769622 : struct bio_vec *bvec;
665 1769622 : struct processed_extent processed = { 0 };
666 : /*
667 : * The offset to the beginning of a bio, since one bio can never be
668 : * larger than UINT_MAX, u32 here is enough.
669 : */
670 1769622 : u32 bio_offset = 0;
671 1769622 : struct bvec_iter_all iter_all;
672 :
673 1769622 : ASSERT(!bio_flagged(bio, BIO_CLONED));
674 20842772 : bio_for_each_segment_all(bvec, bio, iter_all) {
675 19071965 : bool uptodate = !bio->bi_status;
676 19071965 : struct page *page = bvec->bv_page;
677 19071965 : struct inode *inode = page->mapping->host;
678 19071965 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
679 19071965 : const u32 sectorsize = fs_info->sectorsize;
680 19071965 : u64 start;
681 19071965 : u64 end;
682 19071965 : u32 len;
683 :
684 19071965 : btrfs_debug(fs_info,
685 : "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
686 : bio->bi_iter.bi_sector, bio->bi_status,
687 : bbio->mirror_num);
688 :
689 : /*
690 : * We always issue full-sector reads, but if some block in a
691 : * page fails to read, blk_update_request() will advance
692 : * bv_offset and adjust bv_len to compensate. Print a warning
693 : * for unaligned offsets, and an error if they don't add up to
694 : * a full sector.
695 : */
696 19071965 : if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
697 0 : btrfs_err(fs_info,
698 : "partial page read in btrfs with offset %u and length %u",
699 : bvec->bv_offset, bvec->bv_len);
700 19071965 : else if (!IS_ALIGNED(bvec->bv_offset + bvec->bv_len,
701 : sectorsize))
702 0 : btrfs_info(fs_info,
703 : "incomplete page read with offset %u and length %u",
704 : bvec->bv_offset, bvec->bv_len);
705 :
706 19071965 : start = page_offset(page) + bvec->bv_offset;
707 19071965 : end = start + bvec->bv_len - 1;
708 19071965 : len = bvec->bv_len;
709 :
710 19071965 : if (likely(uptodate)) {
711 19071963 : loff_t i_size = i_size_read(inode);
712 19071963 : pgoff_t end_index = i_size >> PAGE_SHIFT;
713 :
714 : /*
715 : * Zero out the remaining part if this range straddles
716 : * i_size.
717 : *
718 : * Here we should only zero the range inside the bvec,
719 : * not touch anything else.
720 : *
721 : * NOTE: i_size is exclusive while end is inclusive.
722 : */
723 19071963 : if (page->index == end_index && i_size <= end) {
724 21086 : u32 zero_start = max(offset_in_page(i_size),
725 : offset_in_page(start));
726 :
727 21086 : zero_user_segment(page, zero_start,
728 21086 : offset_in_page(end) + 1);
729 : }
730 : }
731 :
732 : /* Update page status and unlock. */
733 19071965 : end_page_read(page, uptodate, start, len);
734 19073481 : endio_readpage_release_extent(&processed, BTRFS_I(inode),
735 : start, end, uptodate);
736 :
737 19073481 : ASSERT(bio_offset + len > bio_offset);
738 19073481 : bio_offset += len;
739 :
740 : }
741 : /* Release the last extent */
742 1769631 : endio_readpage_release_extent(&processed, NULL, 0, 0, false);
743 1769560 : bio_put(bio);
744 1769610 : }
745 :
746 : /*
747 : * Populate every free slot in a provided array with pages.
748 : *
749 : * @nr_pages: number of pages to allocate
750 : * @page_array: the array to fill with pages; any existing non-null entries in
751 : * the array will be skipped
752 : *
753 : * Return: 0 if all pages were able to be allocated;
754 : * -ENOMEM otherwise, and the caller is responsible for freeing all
755 : * non-null page pointers in the array.
756 : */
757 19340613 : int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array)
758 : {
759 19340613 : unsigned int allocated;
760 :
761 19340360 : for (allocated = 0; allocated < nr_pages;) {
762 19340360 : unsigned int last = allocated;
763 :
764 19340360 : allocated = alloc_pages_bulk_array(GFP_NOFS, nr_pages, page_array);
765 :
766 19339867 : if (allocated == nr_pages)
767 : return 0;
768 :
769 : /*
770 : * During this iteration, no page could be allocated, even
771 : * though alloc_pages_bulk_array() falls back to alloc_page()
772 : * if it could not bulk-allocate. So we must be out of memory.
773 : */
774 43 : if (allocated == last)
775 : return -ENOMEM;
776 :
777 43 : memalloc_retry_wait(GFP_NOFS);
778 : }
779 : return 0;
780 : }
781 :
782 65258215 : static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
783 : struct page *page, u64 disk_bytenr,
784 : unsigned int pg_offset)
785 : {
786 65258215 : struct bio *bio = &bio_ctrl->bbio->bio;
787 65258215 : struct bio_vec *bvec = bio_last_bvec_all(bio);
788 65258215 : const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
789 :
790 65258215 : if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
791 : /*
792 : * For compression, all IO should have its logical bytenr set
793 : * to the starting bytenr of the compressed extent.
794 : */
795 33092 : return bio->bi_iter.bi_sector == sector;
796 : }
797 :
798 : /*
799 : * The contig check requires the following conditions to be met:
800 : *
801 : * 1) The pages are belonging to the same inode
802 : * This is implied by the call chain.
803 : *
804 : * 2) The range has adjacent logical bytenr
805 : *
806 : * 3) The range has adjacent file offset
807 : * This is required for the usage of btrfs_bio->file_offset.
808 : */
809 130235350 : return bio_end_sector(bio) == sector &&
810 65010227 : page_offset(bvec->bv_page) + bvec->bv_offset + bvec->bv_len ==
811 65010227 : page_offset(page) + pg_offset;
812 : }
813 :
814 4308032 : static void alloc_new_bio(struct btrfs_inode *inode,
815 : struct btrfs_bio_ctrl *bio_ctrl,
816 : u64 disk_bytenr, u64 file_offset)
817 : {
818 4308032 : struct btrfs_fs_info *fs_info = inode->root->fs_info;
819 4308032 : struct btrfs_bio *bbio;
820 :
821 4308032 : bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, fs_info,
822 : bio_ctrl->end_io_func, NULL);
823 4308714 : bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
824 4308714 : bbio->inode = inode;
825 4308714 : bbio->file_offset = file_offset;
826 4308714 : bio_ctrl->bbio = bbio;
827 4308714 : bio_ctrl->len_to_oe_boundary = U32_MAX;
828 :
829 : /* Limit data write bios to the ordered boundary. */
830 4308714 : if (bio_ctrl->wbc) {
831 2539486 : struct btrfs_ordered_extent *ordered;
832 :
833 2539486 : ordered = btrfs_lookup_ordered_extent(inode, file_offset);
834 2539488 : if (ordered) {
835 2539488 : bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
836 : ordered->file_offset +
837 : ordered->disk_num_bytes - file_offset);
838 2539488 : bbio->ordered = ordered;
839 : }
840 :
841 : /*
842 : * Pick the last added device to support cgroup writeback. For
843 : * multi-device file systems this means blk-cgroup policies have
844 : * to always be set on the last added/replaced device.
845 : * This is a bit odd but has been like that for a long time.
846 : */
847 2539488 : bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
848 2539386 : wbc_init_bio(bio_ctrl->wbc, &bbio->bio);
849 : }
850 4308630 : }
851 :
852 : /*
853 : * @disk_bytenr: logical bytenr where the write will be
854 : * @page: page to add to the bio
855 : * @size: portion of page that we want to write to
856 : * @pg_offset: offset of the new bio or to check whether we are adding
857 : * a contiguous page to the previous one
858 : *
859 : * The will either add the page into the existing @bio_ctrl->bbio, or allocate a
860 : * new one in @bio_ctrl->bbio.
861 : * The mirror number for this IO should already be initizlied in
862 : * @bio_ctrl->mirror_num.
863 : */
864 69304607 : static void submit_extent_page(struct btrfs_bio_ctrl *bio_ctrl,
865 : u64 disk_bytenr, struct page *page,
866 : size_t size, unsigned long pg_offset)
867 : {
868 69304607 : struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
869 :
870 69304607 : ASSERT(pg_offset + size <= PAGE_SIZE);
871 69304607 : ASSERT(bio_ctrl->end_io_func);
872 :
873 134563484 : if (bio_ctrl->bbio &&
874 65258638 : !btrfs_bio_is_contig(bio_ctrl, page, disk_bytenr, pg_offset))
875 226512 : submit_one_bio(bio_ctrl);
876 :
877 69340873 : do {
878 69340873 : u32 len = size;
879 :
880 : /* Allocate new bio if needed */
881 69340873 : if (!bio_ctrl->bbio) {
882 8615946 : alloc_new_bio(inode, bio_ctrl, disk_bytenr,
883 4307973 : page_offset(page) + pg_offset);
884 : }
885 :
886 : /* Cap to the current ordered extent boundary if there is one. */
887 69341532 : if (len > bio_ctrl->len_to_oe_boundary) {
888 : ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
889 : ASSERT(is_data_inode(&inode->vfs_inode));
890 : len = bio_ctrl->len_to_oe_boundary;
891 : }
892 :
893 69341532 : if (bio_add_page(&bio_ctrl->bbio->bio, page, len, pg_offset) != len) {
894 : /* bio full: move on to a new one */
895 35910 : submit_one_bio(bio_ctrl);
896 35910 : continue;
897 : }
898 :
899 69305697 : if (bio_ctrl->wbc)
900 50231930 : wbc_account_cgroup_owner(bio_ctrl->wbc, page, len);
901 :
902 69301290 : size -= len;
903 69301290 : pg_offset += len;
904 69301290 : disk_bytenr += len;
905 69301290 : bio_ctrl->len_to_oe_boundary -= len;
906 :
907 : /* Ordered extent boundary: move on to a new bio. */
908 69301290 : if (bio_ctrl->len_to_oe_boundary == 0)
909 2485523 : submit_one_bio(bio_ctrl);
910 69337372 : } while (size);
911 69301345 : }
912 :
913 120818522 : static int attach_extent_buffer_page(struct extent_buffer *eb,
914 : struct page *page,
915 : struct btrfs_subpage *prealloc)
916 : {
917 120818522 : struct btrfs_fs_info *fs_info = eb->fs_info;
918 120818522 : int ret = 0;
919 :
920 : /*
921 : * If the page is mapped to btree inode, we should hold the private
922 : * lock to prevent race.
923 : * For cloned or dummy extent buffers, their pages are not mapped and
924 : * will not race with any other ebs.
925 : */
926 120818522 : if (page->mapping)
927 120818522 : lockdep_assert_held(&page->mapping->private_lock);
928 :
929 120818522 : if (fs_info->nodesize >= PAGE_SIZE) {
930 120818522 : if (!PagePrivate(page))
931 120818522 : attach_page_private(page, eb);
932 : else
933 0 : WARN_ON(page->private != (unsigned long)eb);
934 120827185 : return 0;
935 : }
936 :
937 : /* Already mapped, just free prealloc */
938 0 : if (PagePrivate(page)) {
939 0 : btrfs_free_subpage(prealloc);
940 0 : return 0;
941 : }
942 :
943 0 : if (prealloc)
944 : /* Has preallocated memory for subpage */
945 0 : attach_page_private(page, prealloc);
946 : else
947 : /* Do new allocation to attach subpage */
948 0 : ret = btrfs_attach_subpage(fs_info, page,
949 : BTRFS_SUBPAGE_METADATA);
950 : return ret;
951 : }
952 :
953 192192227 : int set_page_extent_mapped(struct page *page)
954 : {
955 192192227 : struct btrfs_fs_info *fs_info;
956 :
957 192192227 : ASSERT(page->mapping);
958 :
959 192192227 : if (PagePrivate(page))
960 : return 0;
961 :
962 103637490 : fs_info = btrfs_sb(page->mapping->host->i_sb);
963 :
964 103637490 : if (btrfs_is_subpage(fs_info, page))
965 0 : return btrfs_attach_subpage(fs_info, page, BTRFS_SUBPAGE_DATA);
966 :
967 103632477 : attach_page_private(page, (void *)EXTENT_PAGE_PRIVATE);
968 103632477 : return 0;
969 : }
970 :
971 135831736 : void clear_page_extent_mapped(struct page *page)
972 : {
973 135831736 : struct btrfs_fs_info *fs_info;
974 :
975 135831736 : ASSERT(page->mapping);
976 :
977 135831736 : if (!PagePrivate(page))
978 : return;
979 :
980 103694667 : fs_info = btrfs_sb(page->mapping->host->i_sb);
981 103694667 : if (btrfs_is_subpage(fs_info, page))
982 0 : return btrfs_detach_subpage(fs_info, page);
983 :
984 103693821 : detach_page_private(page);
985 : }
986 :
987 : static struct extent_map *
988 56407006 : __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
989 : u64 start, u64 len, struct extent_map **em_cached)
990 : {
991 56407006 : struct extent_map *em;
992 :
993 56407006 : if (em_cached && *em_cached) {
994 43422797 : em = *em_cached;
995 86845198 : if (extent_map_in_tree(em) && start >= em->start &&
996 : start < extent_map_end(em)) {
997 38077984 : refcount_inc(&em->refs);
998 38077984 : return em;
999 : }
1000 :
1001 5344813 : free_extent_map(em);
1002 5344555 : *em_cached = NULL;
1003 : }
1004 :
1005 18328764 : em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, start, len);
1006 18329064 : if (em_cached && !IS_ERR(em)) {
1007 6726607 : BUG_ON(*em_cached);
1008 6726607 : refcount_inc(&em->refs);
1009 6726680 : *em_cached = em;
1010 : }
1011 : return em;
1012 : }
1013 : /*
1014 : * basic readpage implementation. Locked extent state structs are inserted
1015 : * into the tree that are removed when the IO is done (by the end_io
1016 : * handlers)
1017 : * XXX JDM: This needs looking at to ensure proper page locking
1018 : * return 0 on success, otherwise return error
1019 : */
1020 59110560 : static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
1021 : struct btrfs_bio_ctrl *bio_ctrl, u64 *prev_em_start)
1022 : {
1023 59110560 : struct inode *inode = page->mapping->host;
1024 59110560 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1025 59110560 : u64 start = page_offset(page);
1026 59110560 : const u64 end = start + PAGE_SIZE - 1;
1027 59110560 : u64 cur = start;
1028 59110560 : u64 extent_offset;
1029 59110560 : u64 last_byte = i_size_read(inode);
1030 59110560 : u64 block_start;
1031 59110560 : struct extent_map *em;
1032 59110560 : int ret = 0;
1033 59110560 : size_t pg_offset = 0;
1034 59110560 : size_t iosize;
1035 59110560 : size_t blocksize = inode->i_sb->s_blocksize;
1036 59110560 : struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
1037 :
1038 59110560 : ret = set_page_extent_mapped(page);
1039 59104200 : if (ret < 0) {
1040 0 : unlock_extent(tree, start, end, NULL);
1041 0 : unlock_page(page);
1042 0 : return ret;
1043 : }
1044 :
1045 59104200 : if (page->index == last_byte >> PAGE_SHIFT) {
1046 1377184 : size_t zero_offset = offset_in_page(last_byte);
1047 :
1048 1377184 : if (zero_offset) {
1049 179772 : iosize = PAGE_SIZE - zero_offset;
1050 179772 : memzero_page(page, zero_offset, iosize);
1051 : }
1052 : }
1053 59104202 : bio_ctrl->end_io_func = end_bio_extent_readpage;
1054 59104202 : begin_page_read(fs_info, page);
1055 115515810 : while (cur <= end) {
1056 59085414 : enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
1057 59085414 : bool force_bio_submit = false;
1058 59085414 : u64 disk_bytenr;
1059 :
1060 59085414 : ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1061 59085414 : if (cur >= last_byte) {
1062 2680260 : iosize = PAGE_SIZE - pg_offset;
1063 2680260 : memzero_page(page, pg_offset, iosize);
1064 2705261 : unlock_extent(tree, cur, cur + iosize - 1, NULL);
1065 2704956 : end_page_read(page, true, cur, iosize);
1066 2704956 : break;
1067 : }
1068 56405154 : em = __get_extent_map(inode, page, pg_offset, cur,
1069 56405154 : end - cur + 1, em_cached);
1070 56412166 : if (IS_ERR(em)) {
1071 0 : unlock_extent(tree, cur, end, NULL);
1072 0 : end_page_read(page, false, cur, end + 1 - cur);
1073 0 : return PTR_ERR(em);
1074 : }
1075 56412166 : extent_offset = cur - em->start;
1076 56412166 : BUG_ON(extent_map_end(em) <= cur);
1077 56412166 : BUG_ON(end < cur);
1078 :
1079 112824332 : if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
1080 34265 : compress_type = em->compress_type;
1081 :
1082 56412166 : iosize = min(extent_map_end(em) - cur, end - cur + 1);
1083 56412166 : iosize = ALIGN(iosize, blocksize);
1084 56412166 : if (compress_type != BTRFS_COMPRESS_NONE)
1085 34265 : disk_bytenr = em->block_start;
1086 : else
1087 56377901 : disk_bytenr = em->block_start + extent_offset;
1088 56412166 : block_start = em->block_start;
1089 112824332 : if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
1090 5091105 : block_start = EXTENT_MAP_HOLE;
1091 :
1092 : /*
1093 : * If we have a file range that points to a compressed extent
1094 : * and it's followed by a consecutive file range that points
1095 : * to the same compressed extent (possibly with a different
1096 : * offset and/or length, so it either points to the whole extent
1097 : * or only part of it), we must make sure we do not submit a
1098 : * single bio to populate the pages for the 2 ranges because
1099 : * this makes the compressed extent read zero out the pages
1100 : * belonging to the 2nd range. Imagine the following scenario:
1101 : *
1102 : * File layout
1103 : * [0 - 8K] [8K - 24K]
1104 : * | |
1105 : * | |
1106 : * points to extent X, points to extent X,
1107 : * offset 4K, length of 8K offset 0, length 16K
1108 : *
1109 : * [extent X, compressed length = 4K uncompressed length = 16K]
1110 : *
1111 : * If the bio to read the compressed extent covers both ranges,
1112 : * it will decompress extent X into the pages belonging to the
1113 : * first range and then it will stop, zeroing out the remaining
1114 : * pages that belong to the other range that points to extent X.
1115 : * So here we make sure we submit 2 bios, one for the first
1116 : * range and another one for the third range. Both will target
1117 : * the same physical extent from disk, but we can't currently
1118 : * make the compressed bio endio callback populate the pages
1119 : * for both ranges because each compressed bio is tightly
1120 : * coupled with a single extent map, and each range can have
1121 : * an extent map with a different offset value relative to the
1122 : * uncompressed data of our extent and different lengths. This
1123 : * is a corner case so we prioritize correctness over
1124 : * non-optimal behavior (submitting 2 bios for the same extent).
1125 : */
1126 112824332 : if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
1127 34265 : prev_em_start && *prev_em_start != (u64)-1 &&
1128 : *prev_em_start != em->start)
1129 1089 : force_bio_submit = true;
1130 :
1131 56412166 : if (prev_em_start)
1132 44809524 : *prev_em_start = em->start;
1133 :
1134 56412166 : free_extent_map(em);
1135 56416531 : em = NULL;
1136 :
1137 : /* we've found a hole, just zero and go on */
1138 56416531 : if (block_start == EXTENT_MAP_HOLE) {
1139 37319736 : memzero_page(page, pg_offset, iosize);
1140 :
1141 37320597 : unlock_extent(tree, cur, cur + iosize - 1, NULL);
1142 37320588 : end_page_read(page, true, cur, iosize);
1143 37316884 : cur = cur + iosize;
1144 37316884 : pg_offset += iosize;
1145 37316884 : continue;
1146 : }
1147 : /* the get_extent function already copied into the page */
1148 19096795 : if (block_start == EXTENT_MAP_INLINE) {
1149 21294 : unlock_extent(tree, cur, cur + iosize - 1, NULL);
1150 21294 : end_page_read(page, true, cur, iosize);
1151 21294 : cur = cur + iosize;
1152 21294 : pg_offset += iosize;
1153 21294 : continue;
1154 : }
1155 :
1156 19075501 : if (bio_ctrl->compress_type != compress_type) {
1157 143 : submit_one_bio(bio_ctrl);
1158 143 : bio_ctrl->compress_type = compress_type;
1159 : }
1160 :
1161 19075501 : if (force_bio_submit)
1162 1089 : submit_one_bio(bio_ctrl);
1163 19075501 : submit_extent_page(bio_ctrl, disk_bytenr, page, iosize,
1164 : pg_offset);
1165 19073430 : cur = cur + iosize;
1166 19073430 : pg_offset += iosize;
1167 : }
1168 :
1169 : return 0;
1170 : }
1171 :
1172 14294693 : int btrfs_read_folio(struct file *file, struct folio *folio)
1173 : {
1174 14294693 : struct page *page = &folio->page;
1175 14294693 : struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
1176 14294693 : u64 start = page_offset(page);
1177 14294693 : u64 end = start + PAGE_SIZE - 1;
1178 14294693 : struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ };
1179 14294693 : int ret;
1180 :
1181 14294693 : btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1182 :
1183 14303413 : ret = btrfs_do_readpage(page, NULL, &bio_ctrl, NULL);
1184 : /*
1185 : * If btrfs_do_readpage() failed we will want to submit the assembled
1186 : * bio to do the cleanup.
1187 : */
1188 14305799 : submit_one_bio(&bio_ctrl);
1189 14302882 : return ret;
1190 : }
1191 :
1192 3279821 : static inline void contiguous_readpages(struct page *pages[], int nr_pages,
1193 : u64 start, u64 end,
1194 : struct extent_map **em_cached,
1195 : struct btrfs_bio_ctrl *bio_ctrl,
1196 : u64 *prev_em_start)
1197 : {
1198 3279821 : struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host);
1199 3279821 : int index;
1200 :
1201 3279821 : btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1202 :
1203 51371473 : for (index = 0; index < nr_pages; index++) {
1204 44811845 : btrfs_do_readpage(pages[index], em_cached, bio_ctrl,
1205 : prev_em_start);
1206 44809778 : put_page(pages[index]);
1207 : }
1208 3279807 : }
1209 :
1210 : /*
1211 : * helper for __extent_writepage, doing all of the delayed allocation setup.
1212 : *
1213 : * This returns 1 if btrfs_run_delalloc_range function did all the work required
1214 : * to write the page (copy into inline extent). In this case the IO has
1215 : * been started and the page is already unlocked.
1216 : *
1217 : * This returns 0 if all went well (page still locked)
1218 : * This returns < 0 if there were errors (page still locked)
1219 : */
1220 51307662 : static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1221 : struct page *page, struct writeback_control *wbc)
1222 : {
1223 51307662 : const u64 page_end = page_offset(page) + PAGE_SIZE - 1;
1224 51307662 : u64 delalloc_start = page_offset(page);
1225 51307662 : u64 delalloc_to_write = 0;
1226 : /* How many pages are started by btrfs_run_delalloc_range() */
1227 51307662 : unsigned long nr_written = 0;
1228 51307662 : int ret;
1229 51307662 : int page_started = 0;
1230 :
1231 102614255 : while (delalloc_start < page_end) {
1232 51307626 : u64 delalloc_end = page_end;
1233 51307626 : bool found;
1234 :
1235 51307626 : found = find_lock_delalloc_range(&inode->vfs_inode, page,
1236 : &delalloc_start,
1237 : &delalloc_end);
1238 51306524 : if (!found) {
1239 47833929 : delalloc_start = delalloc_end + 1;
1240 47833929 : continue;
1241 : }
1242 3472595 : ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
1243 : delalloc_end, &page_started, &nr_written, wbc);
1244 3472664 : if (ret)
1245 0 : return ret;
1246 :
1247 : /*
1248 : * delalloc_end is already one less than the total length, so
1249 : * we don't subtract one from PAGE_SIZE
1250 : */
1251 3472664 : delalloc_to_write += (delalloc_end - delalloc_start +
1252 3472664 : PAGE_SIZE) >> PAGE_SHIFT;
1253 3472664 : delalloc_start = delalloc_end + 1;
1254 : }
1255 51306629 : if (wbc->nr_to_write < delalloc_to_write) {
1256 3286 : int thresh = 8192;
1257 :
1258 3286 : if (delalloc_to_write < thresh * 2)
1259 2887 : thresh = delalloc_to_write;
1260 3286 : wbc->nr_to_write = min_t(u64, delalloc_to_write,
1261 : thresh);
1262 : }
1263 :
1264 : /* Did btrfs_run_dealloc_range() already unlock and start the IO? */
1265 51306629 : if (page_started) {
1266 : /*
1267 : * We've unlocked the page, so we can't update the mapping's
1268 : * writeback index, just update nr_to_write.
1269 : */
1270 1078130 : wbc->nr_to_write -= nr_written;
1271 1078130 : return 1;
1272 : }
1273 :
1274 : return 0;
1275 : }
1276 :
1277 : /*
1278 : * Find the first byte we need to write.
1279 : *
1280 : * For subpage, one page can contain several sectors, and
1281 : * __extent_writepage_io() will just grab all extent maps in the page
1282 : * range and try to submit all non-inline/non-compressed extents.
1283 : *
1284 : * This is a big problem for subpage, we shouldn't re-submit already written
1285 : * data at all.
1286 : * This function will lookup subpage dirty bit to find which range we really
1287 : * need to submit.
1288 : *
1289 : * Return the next dirty range in [@start, @end).
1290 : * If no dirty range is found, @start will be page_offset(page) + PAGE_SIZE.
1291 : */
1292 50228680 : static void find_next_dirty_byte(struct btrfs_fs_info *fs_info,
1293 : struct page *page, u64 *start, u64 *end)
1294 : {
1295 50228680 : struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
1296 50228680 : struct btrfs_subpage_info *spi = fs_info->subpage_info;
1297 50228680 : u64 orig_start = *start;
1298 : /* Declare as unsigned long so we can use bitmap ops */
1299 50228680 : unsigned long flags;
1300 50228680 : int range_start_bit;
1301 50228680 : int range_end_bit;
1302 :
1303 : /*
1304 : * For regular sector size == page size case, since one page only
1305 : * contains one sector, we return the page offset directly.
1306 : */
1307 50228680 : if (!btrfs_is_subpage(fs_info, page)) {
1308 50227661 : *start = page_offset(page);
1309 50227661 : *end = page_offset(page) + PAGE_SIZE;
1310 50227661 : return;
1311 : }
1312 :
1313 0 : range_start_bit = spi->dirty_offset +
1314 0 : (offset_in_page(orig_start) >> fs_info->sectorsize_bits);
1315 :
1316 : /* We should have the page locked, but just in case */
1317 0 : spin_lock_irqsave(&subpage->lock, flags);
1318 0 : bitmap_next_set_region(subpage->bitmaps, &range_start_bit, &range_end_bit,
1319 0 : spi->dirty_offset + spi->bitmap_nr_bits);
1320 0 : spin_unlock_irqrestore(&subpage->lock, flags);
1321 :
1322 0 : range_start_bit -= spi->dirty_offset;
1323 0 : range_end_bit -= spi->dirty_offset;
1324 :
1325 0 : *start = page_offset(page) + range_start_bit * fs_info->sectorsize;
1326 0 : *end = page_offset(page) + range_end_bit * fs_info->sectorsize;
1327 : }
1328 :
1329 : /*
1330 : * helper for __extent_writepage. This calls the writepage start hooks,
1331 : * and does the loop to map the page into extents and bios.
1332 : *
1333 : * We return 1 if the IO is started and the page is unlocked,
1334 : * 0 if all went well (page still locked)
1335 : * < 0 if there were errors (page still locked)
1336 : */
1337 50229647 : static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode,
1338 : struct page *page,
1339 : struct btrfs_bio_ctrl *bio_ctrl,
1340 : loff_t i_size,
1341 : int *nr_ret)
1342 : {
1343 50229647 : struct btrfs_fs_info *fs_info = inode->root->fs_info;
1344 50229647 : u64 cur = page_offset(page);
1345 50229647 : u64 end = cur + PAGE_SIZE - 1;
1346 50229647 : u64 extent_offset;
1347 50229647 : u64 block_start;
1348 50229647 : struct extent_map *em;
1349 50229647 : int ret = 0;
1350 50229647 : int nr = 0;
1351 :
1352 50229647 : ret = btrfs_writepage_cow_fixup(page);
1353 50229615 : if (ret) {
1354 : /* Fixup worker will requeue */
1355 0 : redirty_page_for_writepage(bio_ctrl->wbc, page);
1356 0 : unlock_page(page);
1357 0 : return 1;
1358 : }
1359 :
1360 50229615 : bio_ctrl->end_io_func = end_bio_extent_writepage;
1361 100457663 : while (cur <= end) {
1362 50229496 : u64 disk_bytenr;
1363 50229496 : u64 em_end;
1364 50229496 : u64 dirty_range_start = cur;
1365 50229496 : u64 dirty_range_end;
1366 50229496 : u32 iosize;
1367 :
1368 50229496 : if (cur >= i_size) {
1369 0 : btrfs_writepage_endio_finish_ordered(inode, page, cur,
1370 : end, true);
1371 : /*
1372 : * This range is beyond i_size, thus we don't need to
1373 : * bother writing back.
1374 : * But we still need to clear the dirty subpage bit, or
1375 : * the next time the page gets dirtied, we will try to
1376 : * writeback the sectors with subpage dirty bits,
1377 : * causing writeback without ordered extent.
1378 : */
1379 0 : btrfs_page_clear_dirty(fs_info, page, cur, end + 1 - cur);
1380 0 : break;
1381 : }
1382 :
1383 50229496 : find_next_dirty_byte(fs_info, page, &dirty_range_start,
1384 : &dirty_range_end);
1385 50227709 : if (cur < dirty_range_start) {
1386 0 : cur = dirty_range_start;
1387 0 : continue;
1388 : }
1389 :
1390 50227709 : em = btrfs_get_extent(inode, NULL, 0, cur, end - cur + 1);
1391 50229105 : if (IS_ERR(em)) {
1392 0 : ret = PTR_ERR_OR_ZERO(em);
1393 0 : goto out_error;
1394 : }
1395 :
1396 50229105 : extent_offset = cur - em->start;
1397 50229105 : em_end = extent_map_end(em);
1398 50229105 : ASSERT(cur <= em_end);
1399 50229105 : ASSERT(cur < end);
1400 50229105 : ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize));
1401 50229105 : ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize));
1402 :
1403 50229105 : block_start = em->block_start;
1404 50229105 : disk_bytenr = em->block_start + extent_offset;
1405 :
1406 50229105 : ASSERT(!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags));
1407 50229105 : ASSERT(block_start != EXTENT_MAP_HOLE);
1408 50229105 : ASSERT(block_start != EXTENT_MAP_INLINE);
1409 :
1410 : /*
1411 : * Note that em_end from extent_map_end() and dirty_range_end from
1412 : * find_next_dirty_byte() are all exclusive
1413 : */
1414 50229105 : iosize = min(min(em_end, end + 1), dirty_range_end) - cur;
1415 50229105 : free_extent_map(em);
1416 50232521 : em = NULL;
1417 :
1418 50232521 : btrfs_set_range_writeback(inode, cur, cur + iosize - 1);
1419 100461845 : if (!PageWriteback(page)) {
1420 0 : btrfs_err(inode->root->fs_info,
1421 : "page %lu not writeback, cur %llu end %llu",
1422 : page->index, cur, end);
1423 : }
1424 :
1425 : /*
1426 : * Although the PageDirty bit is cleared before entering this
1427 : * function, subpage dirty bit is not cleared.
1428 : * So clear subpage dirty bit here so next time we won't submit
1429 : * page for range already written to disk.
1430 : */
1431 50230714 : btrfs_page_clear_dirty(fs_info, page, cur, iosize);
1432 :
1433 50230782 : submit_extent_page(bio_ctrl, disk_bytenr, page, iosize,
1434 50230782 : cur - page_offset(page));
1435 50228048 : cur += iosize;
1436 50228048 : nr++;
1437 : }
1438 :
1439 50228167 : btrfs_page_assert_not_dirty(fs_info, page);
1440 50227201 : *nr_ret = nr;
1441 50227201 : return 0;
1442 :
1443 : out_error:
1444 : /*
1445 : * If we finish without problem, we should not only clear page dirty,
1446 : * but also empty subpage dirty bits
1447 : */
1448 0 : *nr_ret = nr;
1449 0 : return ret;
1450 : }
1451 :
1452 : /*
1453 : * the writepage semantics are similar to regular writepage. extent
1454 : * records are inserted to lock ranges in the tree, and as dirty areas
1455 : * are found, they are marked writeback. Then the lock bits are removed
1456 : * and the end_io handler clears the writeback ranges
1457 : *
1458 : * Return 0 if everything goes well.
1459 : * Return <0 for error.
1460 : */
1461 51307314 : static int __extent_writepage(struct page *page, struct btrfs_bio_ctrl *bio_ctrl)
1462 : {
1463 51307314 : struct folio *folio = page_folio(page);
1464 51306375 : struct inode *inode = page->mapping->host;
1465 51306375 : const u64 page_start = page_offset(page);
1466 51306375 : const u64 page_end = page_start + PAGE_SIZE - 1;
1467 51306375 : int ret;
1468 51306375 : int nr = 0;
1469 51306375 : size_t pg_offset;
1470 51306375 : loff_t i_size = i_size_read(inode);
1471 51306375 : unsigned long end_index = i_size >> PAGE_SHIFT;
1472 :
1473 51306375 : trace___extent_writepage(page, inode, bio_ctrl->wbc);
1474 :
1475 102611827 : WARN_ON(!PageLocked(page));
1476 :
1477 51305928 : pg_offset = offset_in_page(i_size);
1478 51305928 : if (page->index > end_index ||
1479 1949257 : (page->index == end_index && !pg_offset)) {
1480 65 : folio_invalidate(folio, 0, folio_size(folio));
1481 65 : folio_unlock(folio);
1482 65 : return 0;
1483 : }
1484 :
1485 51305863 : if (page->index == end_index)
1486 1949245 : memzero_page(page, pg_offset, PAGE_SIZE - pg_offset);
1487 :
1488 51305933 : ret = set_page_extent_mapped(page);
1489 51305476 : if (ret < 0)
1490 0 : goto done;
1491 :
1492 51305476 : ret = writepage_delalloc(BTRFS_I(inode), page, bio_ctrl->wbc);
1493 51306926 : if (ret == 1)
1494 : return 0;
1495 50228789 : if (ret)
1496 0 : goto done;
1497 :
1498 50228789 : ret = __extent_writepage_io(BTRFS_I(inode), page, bio_ctrl, i_size, &nr);
1499 50227271 : if (ret == 1)
1500 : return 0;
1501 :
1502 50227271 : bio_ctrl->wbc->nr_to_write--;
1503 :
1504 50227271 : done:
1505 50227271 : if (nr == 0) {
1506 : /* make sure the mapping tag for page dirty gets cleared */
1507 0 : set_page_writeback(page);
1508 0 : end_page_writeback(page);
1509 : }
1510 50227271 : if (ret)
1511 0 : end_extent_writepage(page, ret, page_start, page_end);
1512 50227271 : unlock_page(page);
1513 50227271 : ASSERT(ret <= 0);
1514 50227271 : return ret;
1515 : }
1516 :
1517 24623 : void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
1518 : {
1519 24623 : wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
1520 : TASK_UNINTERRUPTIBLE);
1521 24623 : }
1522 :
1523 : /*
1524 : * Lock extent buffer status and pages for writeback.
1525 : *
1526 : * Return %false if the extent buffer doesn't need to be submitted (e.g. the
1527 : * extent buffer is not dirty)
1528 : * Return %true is the extent buffer is submitted to bio.
1529 : */
1530 8889011 : static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
1531 : struct writeback_control *wbc)
1532 : {
1533 8889011 : struct btrfs_fs_info *fs_info = eb->fs_info;
1534 8889011 : bool ret = false;
1535 :
1536 8889011 : btrfs_tree_lock(eb);
1537 17782377 : while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
1538 2667 : btrfs_tree_unlock(eb);
1539 2668 : if (wbc->sync_mode != WB_SYNC_ALL)
1540 : return false;
1541 2190 : wait_on_extent_buffer_writeback(eb);
1542 2190 : btrfs_tree_lock(eb);
1543 : }
1544 :
1545 : /*
1546 : * We need to do this to prevent races in people who check if the eb is
1547 : * under IO since we can end up having no IO bits set for a short period
1548 : * of time.
1549 : */
1550 8888510 : spin_lock(&eb->refs_lock);
1551 8888521 : if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
1552 8882089 : set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1553 8882094 : spin_unlock(&eb->refs_lock);
1554 8882099 : btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1555 8882084 : percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1556 8882084 : -eb->len,
1557 : fs_info->dirty_metadata_batch);
1558 8882084 : ret = true;
1559 : } else {
1560 6440 : spin_unlock(&eb->refs_lock);
1561 : }
1562 8888528 : btrfs_tree_unlock(eb);
1563 8888528 : return ret;
1564 : }
1565 :
1566 74 : static void set_btree_ioerr(struct extent_buffer *eb)
1567 : {
1568 74 : struct btrfs_fs_info *fs_info = eb->fs_info;
1569 :
1570 74 : set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1571 :
1572 : /*
1573 : * A read may stumble upon this buffer later, make sure that it gets an
1574 : * error and knows there was an error.
1575 : */
1576 74 : clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
1577 :
1578 : /*
1579 : * We need to set the mapping with the io error as well because a write
1580 : * error will flip the file system readonly, and then syncfs() will
1581 : * return a 0 because we are readonly if we don't modify the err seq for
1582 : * the superblock.
1583 : */
1584 74 : mapping_set_error(eb->fs_info->btree_inode->i_mapping, -EIO);
1585 :
1586 : /*
1587 : * If writeback for a btree extent that doesn't belong to a log tree
1588 : * failed, increment the counter transaction->eb_write_errors.
1589 : * We do this because while the transaction is running and before it's
1590 : * committing (when we call filemap_fdata[write|wait]_range against
1591 : * the btree inode), we might have
1592 : * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1593 : * returns an error or an error happens during writeback, when we're
1594 : * committing the transaction we wouldn't know about it, since the pages
1595 : * can be no longer dirty nor marked anymore for writeback (if a
1596 : * subsequent modification to the extent buffer didn't happen before the
1597 : * transaction commit), which makes filemap_fdata[write|wait]_range not
1598 : * able to find the pages tagged with SetPageError at transaction
1599 : * commit time. So if this happens we must abort the transaction,
1600 : * otherwise we commit a super block with btree roots that point to
1601 : * btree nodes/leafs whose content on disk is invalid - either garbage
1602 : * or the content of some node/leaf from a past generation that got
1603 : * cowed or deleted and is no longer valid.
1604 : *
1605 : * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1606 : * not be enough - we need to distinguish between log tree extents vs
1607 : * non-log tree extents, and the next filemap_fdatawait_range() call
1608 : * will catch and clear such errors in the mapping - and that call might
1609 : * be from a log sync and not from a transaction commit. Also, checking
1610 : * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1611 : * not done and would not be reliable - the eb might have been released
1612 : * from memory and reading it back again means that flag would not be
1613 : * set (since it's a runtime flag, not persisted on disk).
1614 : *
1615 : * Using the flags below in the btree inode also makes us achieve the
1616 : * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
1617 : * writeback for all dirty pages and before filemap_fdatawait_range()
1618 : * is called, the writeback for all dirty pages had already finished
1619 : * with errors - because we were not using AS_EIO/AS_ENOSPC,
1620 : * filemap_fdatawait_range() would return success, as it could not know
1621 : * that writeback errors happened (the pages were no longer tagged for
1622 : * writeback).
1623 : */
1624 74 : switch (eb->log_index) {
1625 70 : case -1:
1626 70 : set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags);
1627 : break;
1628 4 : case 0:
1629 4 : set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags);
1630 : break;
1631 0 : case 1:
1632 0 : set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags);
1633 : break;
1634 0 : default:
1635 0 : BUG(); /* unexpected, logic error */
1636 : }
1637 74 : }
1638 :
1639 : /*
1640 : * The endio specific version which won't touch any unsafe spinlock in endio
1641 : * context.
1642 : */
1643 585015743 : static struct extent_buffer *find_extent_buffer_nolock(
1644 : struct btrfs_fs_info *fs_info, u64 start)
1645 : {
1646 585015743 : struct extent_buffer *eb;
1647 :
1648 585015743 : rcu_read_lock();
1649 584970565 : eb = radix_tree_lookup(&fs_info->buffer_radix,
1650 584970565 : start >> fs_info->sectorsize_bits);
1651 1157670781 : if (eb && atomic_inc_not_zero(&eb->refs)) {
1652 572818590 : rcu_read_unlock();
1653 572818590 : return eb;
1654 : }
1655 12904256 : rcu_read_unlock();
1656 12904256 : return NULL;
1657 : }
1658 :
1659 8882105 : static void extent_buffer_write_end_io(struct btrfs_bio *bbio)
1660 : {
1661 8882105 : struct extent_buffer *eb = bbio->private;
1662 8882105 : struct btrfs_fs_info *fs_info = eb->fs_info;
1663 8882105 : bool uptodate = !bbio->bio.bi_status;
1664 8882105 : struct bvec_iter_all iter_all;
1665 8882105 : struct bio_vec *bvec;
1666 8882105 : u32 bio_offset = 0;
1667 :
1668 8882105 : if (!uptodate)
1669 74 : set_btree_ioerr(eb);
1670 :
1671 43433392 : bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
1672 34551287 : u64 start = eb->start + bio_offset;
1673 34551287 : struct page *page = bvec->bv_page;
1674 34551287 : u32 len = bvec->bv_len;
1675 :
1676 34551287 : if (!uptodate)
1677 296 : btrfs_page_clear_uptodate(fs_info, page, start, len);
1678 34551287 : btrfs_page_clear_writeback(fs_info, page, start, len);
1679 34551287 : bio_offset += len;
1680 : }
1681 :
1682 8882105 : clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1683 8882105 : smp_mb__after_atomic();
1684 8882105 : wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
1685 :
1686 8882105 : bio_put(&bbio->bio);
1687 8882105 : }
1688 :
1689 8882042 : static void prepare_eb_write(struct extent_buffer *eb)
1690 : {
1691 8882042 : u32 nritems;
1692 8882042 : unsigned long start;
1693 8882042 : unsigned long end;
1694 :
1695 8882042 : clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1696 :
1697 : /* Set btree blocks beyond nritems with 0 to avoid stale content */
1698 8882073 : nritems = btrfs_header_nritems(eb);
1699 8882073 : if (btrfs_header_level(eb) > 0) {
1700 879548 : end = btrfs_node_key_ptr_offset(eb, nritems);
1701 879548 : memzero_extent_buffer(eb, end, eb->len - end);
1702 : } else {
1703 : /*
1704 : * Leaf:
1705 : * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
1706 : */
1707 8002525 : start = btrfs_item_nr_offset(eb, nritems);
1708 8002525 : end = btrfs_item_nr_offset(eb, 0);
1709 8002525 : if (nritems == 0)
1710 233 : end += BTRFS_LEAF_DATA_SIZE(eb->fs_info);
1711 : else
1712 8002292 : end += btrfs_item_offset(eb, nritems - 1);
1713 8002509 : memzero_extent_buffer(eb, start, end - start);
1714 : }
1715 8882072 : }
1716 :
1717 8882069 : static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
1718 : struct writeback_control *wbc)
1719 : {
1720 8882069 : struct btrfs_fs_info *fs_info = eb->fs_info;
1721 8882069 : struct btrfs_bio *bbio;
1722 :
1723 8882069 : prepare_eb_write(eb);
1724 :
1725 9055382 : bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
1726 : REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
1727 : eb->fs_info, extent_buffer_write_end_io, eb);
1728 8882075 : bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
1729 8882075 : bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
1730 8882054 : wbc_init_bio(wbc, &bbio->bio);
1731 8882080 : bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
1732 8882080 : bbio->file_offset = eb->start;
1733 8882080 : if (fs_info->nodesize < PAGE_SIZE) {
1734 0 : struct page *p = eb->pages[0];
1735 :
1736 0 : lock_page(p);
1737 0 : btrfs_subpage_set_writeback(fs_info, p, eb->start, eb->len);
1738 0 : if (btrfs_subpage_clear_and_test_dirty(fs_info, p, eb->start,
1739 0 : eb->len)) {
1740 0 : clear_page_dirty_for_io(p);
1741 0 : wbc->nr_to_write--;
1742 : }
1743 0 : __bio_add_page(&bbio->bio, p, eb->len, eb->start - page_offset(p));
1744 0 : wbc_account_cgroup_owner(wbc, p, eb->len);
1745 0 : unlock_page(p);
1746 : } else {
1747 43433330 : for (int i = 0; i < num_extent_pages(eb); i++) {
1748 34551225 : struct page *p = eb->pages[i];
1749 :
1750 34551225 : lock_page(p);
1751 34551228 : clear_page_dirty_for_io(p);
1752 34551222 : set_page_writeback(p);
1753 34551284 : __bio_add_page(&bbio->bio, p, PAGE_SIZE, 0);
1754 34551223 : wbc_account_cgroup_owner(wbc, p, PAGE_SIZE);
1755 34551224 : wbc->nr_to_write--;
1756 34551224 : unlock_page(p);
1757 : }
1758 : }
1759 8882105 : btrfs_submit_bio(bbio, 0);
1760 8882103 : }
1761 :
1762 : /*
1763 : * Submit one subpage btree page.
1764 : *
1765 : * The main difference to submit_eb_page() is:
1766 : * - Page locking
1767 : * For subpage, we don't rely on page locking at all.
1768 : *
1769 : * - Flush write bio
1770 : * We only flush bio if we may be unable to fit current extent buffers into
1771 : * current bio.
1772 : *
1773 : * Return >=0 for the number of submitted extent buffers.
1774 : * Return <0 for fatal error.
1775 : */
1776 0 : static int submit_eb_subpage(struct page *page, struct writeback_control *wbc)
1777 : {
1778 0 : struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
1779 0 : int submitted = 0;
1780 0 : u64 page_start = page_offset(page);
1781 0 : int bit_start = 0;
1782 0 : int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits;
1783 :
1784 : /* Lock and write each dirty extent buffers in the range */
1785 0 : while (bit_start < fs_info->subpage_info->bitmap_nr_bits) {
1786 0 : struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
1787 0 : struct extent_buffer *eb;
1788 0 : unsigned long flags;
1789 0 : u64 start;
1790 :
1791 : /*
1792 : * Take private lock to ensure the subpage won't be detached
1793 : * in the meantime.
1794 : */
1795 0 : spin_lock(&page->mapping->private_lock);
1796 0 : if (!PagePrivate(page)) {
1797 0 : spin_unlock(&page->mapping->private_lock);
1798 : break;
1799 : }
1800 0 : spin_lock_irqsave(&subpage->lock, flags);
1801 0 : if (!test_bit(bit_start + fs_info->subpage_info->dirty_offset,
1802 : subpage->bitmaps)) {
1803 0 : spin_unlock_irqrestore(&subpage->lock, flags);
1804 0 : spin_unlock(&page->mapping->private_lock);
1805 0 : bit_start++;
1806 0 : continue;
1807 : }
1808 :
1809 0 : start = page_start + bit_start * fs_info->sectorsize;
1810 0 : bit_start += sectors_per_node;
1811 :
1812 : /*
1813 : * Here we just want to grab the eb without touching extra
1814 : * spin locks, so call find_extent_buffer_nolock().
1815 : */
1816 0 : eb = find_extent_buffer_nolock(fs_info, start);
1817 0 : spin_unlock_irqrestore(&subpage->lock, flags);
1818 0 : spin_unlock(&page->mapping->private_lock);
1819 :
1820 : /*
1821 : * The eb has already reached 0 refs thus find_extent_buffer()
1822 : * doesn't return it. We don't need to write back such eb
1823 : * anyway.
1824 : */
1825 0 : if (!eb)
1826 0 : continue;
1827 :
1828 0 : if (lock_extent_buffer_for_io(eb, wbc)) {
1829 0 : write_one_eb(eb, wbc);
1830 0 : submitted++;
1831 : }
1832 0 : free_extent_buffer(eb);
1833 : }
1834 0 : return submitted;
1835 : }
1836 :
1837 : /*
1838 : * Submit all page(s) of one extent buffer.
1839 : *
1840 : * @page: the page of one extent buffer
1841 : * @eb_context: to determine if we need to submit this page, if current page
1842 : * belongs to this eb, we don't need to submit
1843 : *
1844 : * The caller should pass each page in their bytenr order, and here we use
1845 : * @eb_context to determine if we have submitted pages of one extent buffer.
1846 : *
1847 : * If we have, we just skip until we hit a new page that doesn't belong to
1848 : * current @eb_context.
1849 : *
1850 : * If not, we submit all the page(s) of the extent buffer.
1851 : *
1852 : * Return >0 if we have submitted the extent buffer successfully.
1853 : * Return 0 if we don't need to submit the page, as it's already submitted by
1854 : * previous call.
1855 : * Return <0 for fatal error.
1856 : */
1857 33129634 : static int submit_eb_page(struct page *page, struct writeback_control *wbc,
1858 : struct extent_buffer **eb_context)
1859 : {
1860 33129634 : struct address_space *mapping = page->mapping;
1861 33129634 : struct btrfs_block_group *cache = NULL;
1862 33129634 : struct extent_buffer *eb;
1863 33129634 : int ret;
1864 :
1865 33129634 : if (!PagePrivate(page))
1866 : return 0;
1867 :
1868 33088071 : if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
1869 0 : return submit_eb_subpage(page, wbc);
1870 :
1871 33088071 : spin_lock(&mapping->private_lock);
1872 33088133 : if (!PagePrivate(page)) {
1873 0 : spin_unlock(&mapping->private_lock);
1874 0 : return 0;
1875 : }
1876 :
1877 33088133 : eb = (struct extent_buffer *)page->private;
1878 :
1879 : /*
1880 : * Shouldn't happen and normally this would be a BUG_ON but no point
1881 : * crashing the machine for something we can survive anyway.
1882 : */
1883 33088133 : if (WARN_ON(!eb)) {
1884 0 : spin_unlock(&mapping->private_lock);
1885 0 : return 0;
1886 : }
1887 :
1888 33088133 : if (eb == *eb_context) {
1889 24199110 : spin_unlock(&mapping->private_lock);
1890 24199110 : return 0;
1891 : }
1892 8889023 : ret = atomic_inc_not_zero(&eb->refs);
1893 8889023 : spin_unlock(&mapping->private_lock);
1894 8889023 : if (!ret)
1895 : return 0;
1896 :
1897 8889023 : if (!btrfs_check_meta_write_pointer(eb->fs_info, eb, &cache)) {
1898 : /*
1899 : * If for_sync, this hole will be filled with
1900 : * trasnsaction commit.
1901 : */
1902 0 : if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
1903 : ret = -EAGAIN;
1904 : else
1905 0 : ret = 0;
1906 0 : free_extent_buffer(eb);
1907 0 : return ret;
1908 : }
1909 :
1910 8889016 : *eb_context = eb;
1911 :
1912 8889016 : if (!lock_extent_buffer_for_io(eb, wbc)) {
1913 6918 : btrfs_revert_meta_write_pointer(cache, eb);
1914 6918 : if (cache)
1915 0 : btrfs_put_block_group(cache);
1916 6918 : free_extent_buffer(eb);
1917 6918 : return 0;
1918 : }
1919 8882071 : if (cache) {
1920 : /*
1921 : * Implies write in zoned mode. Mark the last eb in a block group.
1922 : */
1923 0 : btrfs_schedule_zone_finish_bg(cache, eb);
1924 0 : btrfs_put_block_group(cache);
1925 : }
1926 8882071 : write_one_eb(eb, wbc);
1927 8882102 : free_extent_buffer(eb);
1928 8882102 : return 1;
1929 : }
1930 :
1931 2280206 : int btree_write_cache_pages(struct address_space *mapping,
1932 : struct writeback_control *wbc)
1933 : {
1934 2280206 : struct extent_buffer *eb_context = NULL;
1935 2280206 : struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
1936 2280206 : int ret = 0;
1937 2280206 : int done = 0;
1938 2280206 : int nr_to_write_done = 0;
1939 2280206 : struct folio_batch fbatch;
1940 2280206 : unsigned int nr_folios;
1941 2280206 : pgoff_t index;
1942 2280206 : pgoff_t end; /* Inclusive */
1943 2280206 : int scanned = 0;
1944 2280206 : xa_mark_t tag;
1945 :
1946 2280206 : folio_batch_init(&fbatch);
1947 2280206 : if (wbc->range_cyclic) {
1948 67 : index = mapping->writeback_index; /* Start from prev offset */
1949 67 : end = -1;
1950 : /*
1951 : * Start from the beginning does not need to cycle over the
1952 : * range, mark it as scanned.
1953 : */
1954 67 : scanned = (index == 0);
1955 : } else {
1956 2280139 : index = wbc->range_start >> PAGE_SHIFT;
1957 2280139 : end = wbc->range_end >> PAGE_SHIFT;
1958 2280139 : scanned = 1;
1959 : }
1960 2280206 : if (wbc->sync_mode == WB_SYNC_ALL)
1961 : tag = PAGECACHE_TAG_TOWRITE;
1962 : else
1963 74 : tag = PAGECACHE_TAG_DIRTY;
1964 2280206 : btrfs_zoned_meta_io_lock(fs_info);
1965 2280189 : retry:
1966 2280190 : if (wbc->sync_mode == WB_SYNC_ALL)
1967 2280108 : tag_pages_for_writeback(mapping, index, end);
1968 9557743 : while (!done && !nr_to_write_done && (index <= end) &&
1969 3773382 : (nr_folios = filemap_get_folios_tag(mapping, &index, end,
1970 : tag, &fbatch))) {
1971 : unsigned i;
1972 :
1973 36633800 : for (i = 0; i < nr_folios; i++) {
1974 33129674 : struct folio *folio = fbatch.folios[i];
1975 :
1976 33129674 : ret = submit_eb_page(&folio->page, wbc, &eb_context);
1977 33129682 : if (ret == 0)
1978 24247582 : continue;
1979 8882100 : if (ret < 0) {
1980 : done = 1;
1981 : break;
1982 : }
1983 :
1984 : /*
1985 : * the filesystem may choose to bump up nr_to_write.
1986 : * We have to make sure to honor the new nr_to_write
1987 : * at any time
1988 : */
1989 8882100 : nr_to_write_done = wbc->nr_to_write <= 0;
1990 : }
1991 3504126 : folio_batch_release(&fbatch);
1992 3504126 : cond_resched();
1993 : }
1994 2280243 : if (!scanned && !done) {
1995 : /*
1996 : * We hit the last page and there is more work to be done: wrap
1997 : * back to the start of the file
1998 : */
1999 1 : scanned = 1;
2000 1 : index = 0;
2001 1 : goto retry;
2002 : }
2003 : /*
2004 : * If something went wrong, don't allow any metadata write bio to be
2005 : * submitted.
2006 : *
2007 : * This would prevent use-after-free if we had dirty pages not
2008 : * cleaned up, which can still happen by fuzzed images.
2009 : *
2010 : * - Bad extent tree
2011 : * Allowing existing tree block to be allocated for other trees.
2012 : *
2013 : * - Log tree operations
2014 : * Exiting tree blocks get allocated to log tree, bumps its
2015 : * generation, then get cleaned in tree re-balance.
2016 : * Such tree block will not be written back, since it's clean,
2017 : * thus no WRITTEN flag set.
2018 : * And after log writes back, this tree block is not traced by
2019 : * any dirty extent_io_tree.
2020 : *
2021 : * - Offending tree block gets re-dirtied from its original owner
2022 : * Since it has bumped generation, no WRITTEN flag, it can be
2023 : * reused without COWing. This tree block will not be traced
2024 : * by btrfs_transaction::dirty_pages.
2025 : *
2026 : * Now such dirty tree block will not be cleaned by any dirty
2027 : * extent io tree. Thus we don't want to submit such wild eb
2028 : * if the fs already has error.
2029 : *
2030 : * We can get ret > 0 from submit_extent_page() indicating how many ebs
2031 : * were submitted. Reset it to 0 to avoid false alerts for the caller.
2032 : */
2033 2280242 : if (ret > 0)
2034 : ret = 0;
2035 2280242 : if (!ret && BTRFS_FS_ERROR(fs_info))
2036 15 : ret = -EROFS;
2037 2280242 : btrfs_zoned_meta_io_unlock(fs_info);
2038 2280242 : return ret;
2039 : }
2040 :
2041 : /*
2042 : * Walk the list of dirty pages of the given address space and write all of them.
2043 : *
2044 : * @mapping: address space structure to write
2045 : * @wbc: subtract the number of written pages from *@wbc->nr_to_write
2046 : * @bio_ctrl: holds context for the write, namely the bio
2047 : *
2048 : * If a page is already under I/O, write_cache_pages() skips it, even
2049 : * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
2050 : * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
2051 : * and msync() need to guarantee that all the data which was dirty at the time
2052 : * the call was made get new I/O started against them. If wbc->sync_mode is
2053 : * WB_SYNC_ALL then we were called for data integrity and we must wait for
2054 : * existing IO to complete.
2055 : */
2056 5596444 : static int extent_write_cache_pages(struct address_space *mapping,
2057 : struct btrfs_bio_ctrl *bio_ctrl)
2058 : {
2059 5596444 : struct writeback_control *wbc = bio_ctrl->wbc;
2060 5596444 : struct inode *inode = mapping->host;
2061 5596444 : int ret = 0;
2062 5596444 : int done = 0;
2063 5596444 : int nr_to_write_done = 0;
2064 5596444 : struct folio_batch fbatch;
2065 5596444 : unsigned int nr_folios;
2066 5596444 : pgoff_t index;
2067 5596444 : pgoff_t end; /* Inclusive */
2068 5596444 : pgoff_t done_index;
2069 5596444 : int range_whole = 0;
2070 5596444 : int scanned = 0;
2071 5596444 : xa_mark_t tag;
2072 :
2073 : /*
2074 : * We have to hold onto the inode so that ordered extents can do their
2075 : * work when the IO finishes. The alternative to this is failing to add
2076 : * an ordered extent if the igrab() fails there and that is a huge pain
2077 : * to deal with, so instead just hold onto the inode throughout the
2078 : * writepages operation. If it fails here we are freeing up the inode
2079 : * anyway and we'd rather not waste our time writing out stuff that is
2080 : * going to be truncated anyway.
2081 : */
2082 5596444 : if (!igrab(inode))
2083 : return 0;
2084 :
2085 5596326 : folio_batch_init(&fbatch);
2086 5596326 : if (wbc->range_cyclic) {
2087 486353 : index = mapping->writeback_index; /* Start from prev offset */
2088 486353 : end = -1;
2089 : /*
2090 : * Start from the beginning does not need to cycle over the
2091 : * range, mark it as scanned.
2092 : */
2093 486353 : scanned = (index == 0);
2094 : } else {
2095 5109973 : index = wbc->range_start >> PAGE_SHIFT;
2096 5109973 : end = wbc->range_end >> PAGE_SHIFT;
2097 5109973 : if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2098 2187581 : range_whole = 1;
2099 : scanned = 1;
2100 : }
2101 :
2102 : /*
2103 : * We do the tagged writepage as long as the snapshot flush bit is set
2104 : * and we are the first one who do the filemap_flush() on this inode.
2105 : *
2106 : * The nr_to_write == LONG_MAX is needed to make sure other flushers do
2107 : * not race in and drop the bit.
2108 : */
2109 6251481 : if (range_whole && wbc->nr_to_write == LONG_MAX &&
2110 : test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
2111 655079 : &BTRFS_I(inode)->runtime_flags))
2112 20181 : wbc->tagged_writepages = 1;
2113 :
2114 5596402 : if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2115 : tag = PAGECACHE_TAG_TOWRITE;
2116 : else
2117 2317223 : tag = PAGECACHE_TAG_DIRTY;
2118 5596402 : retry:
2119 6021641 : if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2120 3279052 : tag_pages_for_writeback(mapping, index, end);
2121 6022016 : done_index = index;
2122 21539306 : while (!done && !nr_to_write_done && (index <= end) &&
2123 9350487 : (nr_folios = filemap_get_folios_tag(mapping, &index,
2124 : end, tag, &fbatch))) {
2125 : unsigned i;
2126 :
2127 60989488 : for (i = 0; i < nr_folios; i++) {
2128 54822002 : struct folio *folio = fbatch.folios[i];
2129 :
2130 54822002 : done_index = folio->index + folio_nr_pages(folio);
2131 : /*
2132 : * At this point we hold neither the i_pages lock nor
2133 : * the page lock: the page may be truncated or
2134 : * invalidated (changing page->mapping to NULL),
2135 : * or even swizzled back from swapper_space to
2136 : * tmpfs file mapping
2137 : */
2138 54821847 : if (!folio_trylock(folio)) {
2139 2136734 : submit_write_bio(bio_ctrl, 0);
2140 2136733 : folio_lock(folio);
2141 : }
2142 :
2143 54823121 : if (unlikely(folio->mapping != mapping)) {
2144 788 : folio_unlock(folio);
2145 788 : continue;
2146 : }
2147 :
2148 54822333 : if (wbc->sync_mode != WB_SYNC_NONE) {
2149 16148888 : if (folio_test_writeback(folio))
2150 3152 : submit_write_bio(bio_ctrl, 0);
2151 16148933 : folio_wait_writeback(folio);
2152 : }
2153 :
2154 106146258 : if (folio_test_writeback(folio) ||
2155 51324683 : !folio_clear_dirty_for_io(folio)) {
2156 3513793 : folio_unlock(folio);
2157 3513905 : continue;
2158 : }
2159 :
2160 51307817 : ret = __extent_writepage(&folio->page, bio_ctrl);
2161 51307531 : if (ret < 0) {
2162 : done = 1;
2163 : break;
2164 : }
2165 :
2166 : /*
2167 : * the filesystem may choose to bump up nr_to_write.
2168 : * We have to make sure to honor the new nr_to_write
2169 : * at any time
2170 : */
2171 51307531 : nr_to_write_done = wbc->nr_to_write <= 0;
2172 : }
2173 6167486 : folio_batch_release(&fbatch);
2174 6167441 : cond_resched();
2175 : }
2176 6021555 : if (!scanned && !done) {
2177 : /*
2178 : * We hit the last page and there is more work to be done: wrap
2179 : * back to the start of the file
2180 : */
2181 425360 : scanned = 1;
2182 425360 : index = 0;
2183 :
2184 : /*
2185 : * If we're looping we could run into a page that is locked by a
2186 : * writer and that writer could be waiting on writeback for a
2187 : * page in our current bio, and thus deadlock, so flush the
2188 : * write bio here.
2189 : */
2190 425360 : submit_write_bio(bio_ctrl, 0);
2191 425239 : goto retry;
2192 : }
2193 :
2194 5596195 : if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2195 2645511 : mapping->writeback_index = done_index;
2196 :
2197 5596195 : btrfs_add_delayed_iput(BTRFS_I(inode));
2198 5596195 : return ret;
2199 : }
2200 :
2201 : /*
2202 : * Submit the pages in the range to bio for call sites which delalloc range has
2203 : * already been ran (aka, ordered extent inserted) and all pages are still
2204 : * locked.
2205 : */
2206 216 : int extent_write_locked_range(struct inode *inode, u64 start, u64 end,
2207 : struct writeback_control *wbc)
2208 : {
2209 216 : bool found_error = false;
2210 216 : int first_error = 0;
2211 216 : int ret = 0;
2212 216 : struct address_space *mapping = inode->i_mapping;
2213 216 : struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2214 216 : const u32 sectorsize = fs_info->sectorsize;
2215 216 : loff_t i_size = i_size_read(inode);
2216 216 : u64 cur = start;
2217 216 : struct btrfs_bio_ctrl bio_ctrl = {
2218 : .wbc = wbc,
2219 216 : .opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2220 : };
2221 :
2222 216 : if (wbc->no_cgroup_owner)
2223 216 : bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
2224 :
2225 216 : ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
2226 :
2227 432 : while (cur <= end) {
2228 216 : u64 cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
2229 216 : struct page *page;
2230 216 : int nr = 0;
2231 :
2232 216 : page = find_get_page(mapping, cur >> PAGE_SHIFT);
2233 : /*
2234 : * All pages in the range are locked since
2235 : * btrfs_run_delalloc_range(), thus there is no way to clear
2236 : * the page dirty flag.
2237 : */
2238 216 : ASSERT(PageLocked(page));
2239 216 : ASSERT(PageDirty(page));
2240 216 : clear_page_dirty_for_io(page);
2241 :
2242 216 : ret = __extent_writepage_io(BTRFS_I(inode), page, &bio_ctrl,
2243 : i_size, &nr);
2244 216 : if (ret == 1)
2245 0 : goto next_page;
2246 :
2247 : /* Make sure the mapping tag for page dirty gets cleared. */
2248 216 : if (nr == 0) {
2249 0 : set_page_writeback(page);
2250 0 : end_page_writeback(page);
2251 : }
2252 216 : if (ret)
2253 0 : end_extent_writepage(page, ret, cur, cur_end);
2254 216 : btrfs_page_unlock_writer(fs_info, page, cur, cur_end + 1 - cur);
2255 216 : if (ret < 0) {
2256 0 : found_error = true;
2257 0 : first_error = ret;
2258 : }
2259 216 : next_page:
2260 216 : put_page(page);
2261 216 : cur = cur_end + 1;
2262 : }
2263 :
2264 432 : submit_write_bio(&bio_ctrl, found_error ? ret : 0);
2265 :
2266 216 : if (found_error)
2267 0 : return first_error;
2268 : return ret;
2269 : }
2270 :
2271 5596646 : int extent_writepages(struct address_space *mapping,
2272 : struct writeback_control *wbc)
2273 : {
2274 5596646 : struct inode *inode = mapping->host;
2275 5596646 : int ret = 0;
2276 5596646 : struct btrfs_bio_ctrl bio_ctrl = {
2277 : .wbc = wbc,
2278 5596646 : .opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2279 : };
2280 :
2281 : /*
2282 : * Allow only a single thread to do the reloc work in zoned mode to
2283 : * protect the write pointer updates.
2284 : */
2285 5596646 : btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
2286 5596436 : ret = extent_write_cache_pages(mapping, &bio_ctrl);
2287 5596780 : submit_write_bio(&bio_ctrl, ret);
2288 5596667 : btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
2289 5596633 : return ret;
2290 : }
2291 :
2292 1382150 : void extent_readahead(struct readahead_control *rac)
2293 : {
2294 1382150 : struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ | REQ_RAHEAD };
2295 1382150 : struct page *pagepool[16];
2296 1382150 : struct extent_map *em_cached = NULL;
2297 1382150 : u64 prev_em_start = (u64)-1;
2298 1382150 : int nr;
2299 :
2300 4661912 : while ((nr = readahead_page_batch(rac, pagepool))) {
2301 3279815 : u64 contig_start = readahead_pos(rac);
2302 3279815 : u64 contig_end = contig_start + readahead_batch_length(rac) - 1;
2303 :
2304 3279815 : contiguous_readpages(pagepool, nr, contig_start, contig_end,
2305 : &em_cached, &bio_ctrl, &prev_em_start);
2306 : }
2307 :
2308 1381971 : if (em_cached)
2309 1381971 : free_extent_map(em_cached);
2310 1382102 : submit_one_bio(&bio_ctrl);
2311 1381870 : }
2312 :
2313 : /*
2314 : * basic invalidate_folio code, this waits on any locked or writeback
2315 : * ranges corresponding to the folio, and then deletes any extent state
2316 : * records from the tree
2317 : */
2318 34505 : int extent_invalidate_folio(struct extent_io_tree *tree,
2319 : struct folio *folio, size_t offset)
2320 : {
2321 34505 : struct extent_state *cached_state = NULL;
2322 34505 : u64 start = folio_pos(folio);
2323 34505 : u64 end = start + folio_size(folio) - 1;
2324 34505 : size_t blocksize = folio->mapping->host->i_sb->s_blocksize;
2325 :
2326 : /* This function is only called for the btree inode */
2327 34505 : ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2328 :
2329 34505 : start += ALIGN(offset, blocksize);
2330 34505 : if (start > end)
2331 : return 0;
2332 :
2333 34505 : lock_extent(tree, start, end, &cached_state);
2334 34505 : folio_wait_writeback(folio);
2335 :
2336 : /*
2337 : * Currently for btree io tree, only EXTENT_LOCKED is utilized,
2338 : * so here we only need to unlock the extent range to free any
2339 : * existing extent state.
2340 : */
2341 34505 : unlock_extent(tree, start, end, &cached_state);
2342 34505 : return 0;
2343 : }
2344 :
2345 : /*
2346 : * a helper for release_folio, this tests for areas of the page that
2347 : * are locked or under IO and drops the related state bits if it is safe
2348 : * to drop the page.
2349 : */
2350 50569870 : static int try_release_extent_state(struct extent_io_tree *tree,
2351 : struct page *page, gfp_t mask)
2352 : {
2353 50569870 : u64 start = page_offset(page);
2354 50569870 : u64 end = start + PAGE_SIZE - 1;
2355 50569870 : int ret = 1;
2356 :
2357 50569870 : if (test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) {
2358 : ret = 0;
2359 : } else {
2360 50456637 : u32 clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM |
2361 : EXTENT_DELALLOC_NEW | EXTENT_CTLBITS);
2362 :
2363 : /*
2364 : * At this point we can safely clear everything except the
2365 : * locked bit, the nodatasum bit and the delalloc new bit.
2366 : * The delalloc new bit will be cleared by ordered extent
2367 : * completion.
2368 : */
2369 50456637 : ret = __clear_extent_bit(tree, start, end, clear_bits, NULL, NULL);
2370 :
2371 : /* if clear_extent_bit failed for enomem reasons,
2372 : * we can't allow the release to continue.
2373 : */
2374 50453152 : if (ret < 0)
2375 : ret = 0;
2376 : else
2377 50453152 : ret = 1;
2378 : }
2379 50567801 : return ret;
2380 : }
2381 :
2382 : /*
2383 : * a helper for release_folio. As long as there are no locked extents
2384 : * in the range corresponding to the page, both state records and extent
2385 : * map records are removed
2386 : */
2387 50570164 : int try_release_extent_mapping(struct page *page, gfp_t mask)
2388 : {
2389 50570164 : struct extent_map *em;
2390 50570164 : u64 start = page_offset(page);
2391 50570164 : u64 end = start + PAGE_SIZE - 1;
2392 50570164 : struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
2393 50570164 : struct extent_io_tree *tree = &btrfs_inode->io_tree;
2394 50570164 : struct extent_map_tree *map = &btrfs_inode->extent_tree;
2395 :
2396 50570164 : if (gfpflags_allow_blocking(mask) &&
2397 42227938 : page->mapping->host->i_size > SZ_16M) {
2398 : u64 len;
2399 4339043 : while (start <= end) {
2400 4283387 : struct btrfs_fs_info *fs_info;
2401 4283387 : u64 cur_gen;
2402 :
2403 4283387 : len = end - start + 1;
2404 4283387 : write_lock(&map->lock);
2405 4283387 : em = lookup_extent_mapping(map, start, len);
2406 4283387 : if (!em) {
2407 1060136 : write_unlock(&map->lock);
2408 1060136 : break;
2409 : }
2410 3223251 : if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
2411 3197930 : em->start != start) {
2412 3167595 : write_unlock(&map->lock);
2413 3167595 : free_extent_map(em);
2414 3167595 : break;
2415 : }
2416 55656 : if (test_range_bit(tree, em->start,
2417 : extent_map_end(em) - 1,
2418 : EXTENT_LOCKED, 0, NULL))
2419 9 : goto next;
2420 : /*
2421 : * If it's not in the list of modified extents, used
2422 : * by a fast fsync, we can remove it. If it's being
2423 : * logged we can safely remove it since fsync took an
2424 : * extra reference on the em.
2425 : */
2426 55647 : if (list_empty(&em->list) ||
2427 0 : test_bit(EXTENT_FLAG_LOGGING, &em->flags))
2428 34805 : goto remove_em;
2429 : /*
2430 : * If it's in the list of modified extents, remove it
2431 : * only if its generation is older then the current one,
2432 : * in which case we don't need it for a fast fsync.
2433 : * Otherwise don't remove it, we could be racing with an
2434 : * ongoing fast fsync that could miss the new extent.
2435 : */
2436 20842 : fs_info = btrfs_inode->root->fs_info;
2437 20842 : spin_lock(&fs_info->trans_lock);
2438 20842 : cur_gen = fs_info->generation;
2439 20842 : spin_unlock(&fs_info->trans_lock);
2440 20842 : if (em->generation >= cur_gen)
2441 1270 : goto next;
2442 19572 : remove_em:
2443 : /*
2444 : * We only remove extent maps that are not in the list of
2445 : * modified extents or that are in the list but with a
2446 : * generation lower then the current generation, so there
2447 : * is no need to set the full fsync flag on the inode (it
2448 : * hurts the fsync performance for workloads with a data
2449 : * size that exceeds or is close to the system's memory).
2450 : */
2451 54377 : remove_extent_mapping(map, em);
2452 : /* once for the rb tree */
2453 54377 : free_extent_map(em);
2454 55656 : next:
2455 55656 : start = extent_map_end(em);
2456 55656 : write_unlock(&map->lock);
2457 :
2458 : /* once for us */
2459 55656 : free_extent_map(em);
2460 :
2461 55656 : cond_resched(); /* Allow large-extent preemption. */
2462 : }
2463 : }
2464 50570164 : return try_release_extent_state(tree, page, mask);
2465 : }
2466 :
2467 : /*
2468 : * To cache previous fiemap extent
2469 : *
2470 : * Will be used for merging fiemap extent
2471 : */
2472 : struct fiemap_cache {
2473 : u64 offset;
2474 : u64 phys;
2475 : u64 len;
2476 : u32 flags;
2477 : bool cached;
2478 : };
2479 :
2480 : /*
2481 : * Helper to submit fiemap extent.
2482 : *
2483 : * Will try to merge current fiemap extent specified by @offset, @phys,
2484 : * @len and @flags with cached one.
2485 : * And only when we fails to merge, cached one will be submitted as
2486 : * fiemap extent.
2487 : *
2488 : * Return value is the same as fiemap_fill_next_extent().
2489 : */
2490 1323466 : static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
2491 : struct fiemap_cache *cache,
2492 : u64 offset, u64 phys, u64 len, u32 flags)
2493 : {
2494 1323466 : int ret = 0;
2495 :
2496 : /* Set at the end of extent_fiemap(). */
2497 1323466 : ASSERT((flags & FIEMAP_EXTENT_LAST) == 0);
2498 :
2499 1323466 : if (!cache->cached)
2500 46788 : goto assign;
2501 :
2502 : /*
2503 : * Sanity check, extent_fiemap() should have ensured that new
2504 : * fiemap extent won't overlap with cached one.
2505 : * Not recoverable.
2506 : *
2507 : * NOTE: Physical address can overlap, due to compression
2508 : */
2509 1276678 : if (cache->offset + cache->len > offset) {
2510 0 : WARN_ON(1);
2511 0 : return -EINVAL;
2512 : }
2513 :
2514 : /*
2515 : * Only merges fiemap extents if
2516 : * 1) Their logical addresses are continuous
2517 : *
2518 : * 2) Their physical addresses are continuous
2519 : * So truly compressed (physical size smaller than logical size)
2520 : * extents won't get merged with each other
2521 : *
2522 : * 3) Share same flags
2523 : */
2524 1276678 : if (cache->offset + cache->len == offset &&
2525 1039166 : cache->phys + cache->len == phys &&
2526 305143 : cache->flags == flags) {
2527 223876 : cache->len += len;
2528 223876 : return 0;
2529 : }
2530 :
2531 : /* Not mergeable, need to submit cached one */
2532 1052802 : ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
2533 : cache->len, cache->flags);
2534 1052764 : cache->cached = false;
2535 1052764 : if (ret)
2536 : return ret;
2537 1030326 : assign:
2538 1077114 : cache->cached = true;
2539 1077114 : cache->offset = offset;
2540 1077114 : cache->phys = phys;
2541 1077114 : cache->len = len;
2542 1077114 : cache->flags = flags;
2543 :
2544 1077114 : return 0;
2545 : }
2546 :
2547 : /*
2548 : * Emit last fiemap cache
2549 : *
2550 : * The last fiemap cache may still be cached in the following case:
2551 : * 0 4k 8k
2552 : * |<- Fiemap range ->|
2553 : * |<------------ First extent ----------->|
2554 : *
2555 : * In this case, the first extent range will be cached but not emitted.
2556 : * So we must emit it before ending extent_fiemap().
2557 : */
2558 49026 : static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo,
2559 : struct fiemap_cache *cache)
2560 : {
2561 49026 : int ret;
2562 :
2563 49026 : if (!cache->cached)
2564 : return 0;
2565 :
2566 24357 : ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
2567 : cache->len, cache->flags);
2568 24356 : cache->cached = false;
2569 24356 : if (ret > 0)
2570 : ret = 0;
2571 : return ret;
2572 : }
2573 :
2574 1307331 : static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path)
2575 : {
2576 1307331 : struct extent_buffer *clone;
2577 1307331 : struct btrfs_key key;
2578 1307331 : int slot;
2579 1307331 : int ret;
2580 :
2581 1307331 : path->slots[0]++;
2582 1307331 : if (path->slots[0] < btrfs_header_nritems(path->nodes[0]))
2583 : return 0;
2584 :
2585 13629 : ret = btrfs_next_leaf(inode->root, path);
2586 13629 : if (ret != 0)
2587 : return ret;
2588 :
2589 : /*
2590 : * Don't bother with cloning if there are no more file extent items for
2591 : * our inode.
2592 : */
2593 4986 : btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2594 4986 : if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY)
2595 : return 1;
2596 :
2597 : /* See the comment at fiemap_search_slot() about why we clone. */
2598 4906 : clone = btrfs_clone_extent_buffer(path->nodes[0]);
2599 4906 : if (!clone)
2600 : return -ENOMEM;
2601 :
2602 4906 : slot = path->slots[0];
2603 4906 : btrfs_release_path(path);
2604 4906 : path->nodes[0] = clone;
2605 4906 : path->slots[0] = slot;
2606 :
2607 4906 : return 0;
2608 : }
2609 :
2610 : /*
2611 : * Search for the first file extent item that starts at a given file offset or
2612 : * the one that starts immediately before that offset.
2613 : * Returns: 0 on success, < 0 on error, 1 if not found.
2614 : */
2615 49030 : static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path,
2616 : u64 file_offset)
2617 : {
2618 49030 : const u64 ino = btrfs_ino(inode);
2619 49030 : struct btrfs_root *root = inode->root;
2620 49030 : struct extent_buffer *clone;
2621 49030 : struct btrfs_key key;
2622 49030 : int slot;
2623 49030 : int ret;
2624 :
2625 49030 : key.objectid = ino;
2626 49030 : key.type = BTRFS_EXTENT_DATA_KEY;
2627 49030 : key.offset = file_offset;
2628 :
2629 49030 : ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2630 49031 : if (ret < 0)
2631 : return ret;
2632 :
2633 49031 : if (ret > 0 && path->slots[0] > 0) {
2634 16647 : btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
2635 16644 : if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY)
2636 8157 : path->slots[0]--;
2637 : }
2638 :
2639 49028 : if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
2640 152 : ret = btrfs_next_leaf(root, path);
2641 152 : if (ret != 0)
2642 : return ret;
2643 :
2644 75 : btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2645 75 : if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2646 : return 1;
2647 : }
2648 :
2649 : /*
2650 : * We clone the leaf and use it during fiemap. This is because while
2651 : * using the leaf we do expensive things like checking if an extent is
2652 : * shared, which can take a long time. In order to prevent blocking
2653 : * other tasks for too long, we use a clone of the leaf. We have locked
2654 : * the file range in the inode's io tree, so we know none of our file
2655 : * extent items can change. This way we avoid blocking other tasks that
2656 : * want to insert items for other inodes in the same leaf or b+tree
2657 : * rebalance operations (triggered for example when someone is trying
2658 : * to push items into this leaf when trying to insert an item in a
2659 : * neighbour leaf).
2660 : * We also need the private clone because holding a read lock on an
2661 : * extent buffer of the subvolume's b+tree will make lockdep unhappy
2662 : * when we call fiemap_fill_next_extent(), because that may cause a page
2663 : * fault when filling the user space buffer with fiemap data.
2664 : */
2665 48943 : clone = btrfs_clone_extent_buffer(path->nodes[0]);
2666 48947 : if (!clone)
2667 : return -ENOMEM;
2668 :
2669 48947 : slot = path->slots[0];
2670 48947 : btrfs_release_path(path);
2671 48947 : path->nodes[0] = clone;
2672 48947 : path->slots[0] = slot;
2673 :
2674 48947 : return 0;
2675 : }
2676 :
2677 : /*
2678 : * Process a range which is a hole or a prealloc extent in the inode's subvolume
2679 : * btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc
2680 : * extent. The end offset (@end) is inclusive.
2681 : */
2682 445463 : static int fiemap_process_hole(struct btrfs_inode *inode,
2683 : struct fiemap_extent_info *fieinfo,
2684 : struct fiemap_cache *cache,
2685 : struct extent_state **delalloc_cached_state,
2686 : struct btrfs_backref_share_check_ctx *backref_ctx,
2687 : u64 disk_bytenr, u64 extent_offset,
2688 : u64 extent_gen,
2689 : u64 start, u64 end)
2690 : {
2691 445463 : const u64 i_size = i_size_read(&inode->vfs_inode);
2692 445463 : u64 cur_offset = start;
2693 445463 : u64 last_delalloc_end = 0;
2694 445463 : u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN;
2695 445463 : bool checked_extent_shared = false;
2696 445463 : int ret;
2697 :
2698 : /*
2699 : * There can be no delalloc past i_size, so don't waste time looking for
2700 : * it beyond i_size.
2701 : */
2702 465651 : while (cur_offset < end && cur_offset < i_size) {
2703 430270 : u64 delalloc_start;
2704 430270 : u64 delalloc_end;
2705 430270 : u64 prealloc_start;
2706 430270 : u64 prealloc_len = 0;
2707 430270 : bool delalloc;
2708 :
2709 430270 : delalloc = btrfs_find_delalloc_in_range(inode, cur_offset, end,
2710 : delalloc_cached_state,
2711 : &delalloc_start,
2712 : &delalloc_end);
2713 430296 : if (!delalloc)
2714 : break;
2715 :
2716 : /*
2717 : * If this is a prealloc extent we have to report every section
2718 : * of it that has no delalloc.
2719 : */
2720 20188 : if (disk_bytenr != 0) {
2721 19817 : if (last_delalloc_end == 0) {
2722 19817 : prealloc_start = start;
2723 19817 : prealloc_len = delalloc_start - start;
2724 : } else {
2725 0 : prealloc_start = last_delalloc_end + 1;
2726 0 : prealloc_len = delalloc_start - prealloc_start;
2727 : }
2728 : }
2729 :
2730 19817 : if (prealloc_len > 0) {
2731 9 : if (!checked_extent_shared && fieinfo->fi_extents_max) {
2732 7 : ret = btrfs_is_data_extent_shared(inode,
2733 : disk_bytenr,
2734 : extent_gen,
2735 : backref_ctx);
2736 7 : if (ret < 0)
2737 0 : return ret;
2738 7 : else if (ret > 0)
2739 2 : prealloc_flags |= FIEMAP_EXTENT_SHARED;
2740 :
2741 : checked_extent_shared = true;
2742 : }
2743 9 : ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
2744 : disk_bytenr + extent_offset,
2745 : prealloc_len, prealloc_flags);
2746 9 : if (ret)
2747 0 : return ret;
2748 9 : extent_offset += prealloc_len;
2749 : }
2750 :
2751 20188 : ret = emit_fiemap_extent(fieinfo, cache, delalloc_start, 0,
2752 20188 : delalloc_end + 1 - delalloc_start,
2753 : FIEMAP_EXTENT_DELALLOC |
2754 : FIEMAP_EXTENT_UNKNOWN);
2755 20188 : if (ret)
2756 0 : return ret;
2757 :
2758 20188 : last_delalloc_end = delalloc_end;
2759 20188 : cur_offset = delalloc_end + 1;
2760 20188 : extent_offset += cur_offset - delalloc_start;
2761 20188 : cond_resched();
2762 : }
2763 :
2764 : /*
2765 : * Either we found no delalloc for the whole prealloc extent or we have
2766 : * a prealloc extent that spans i_size or starts at or after i_size.
2767 : */
2768 445489 : if (disk_bytenr != 0 && last_delalloc_end < end) {
2769 153786 : u64 prealloc_start;
2770 153786 : u64 prealloc_len;
2771 :
2772 153786 : if (last_delalloc_end == 0) {
2773 152764 : prealloc_start = start;
2774 152764 : prealloc_len = end + 1 - start;
2775 : } else {
2776 1022 : prealloc_start = last_delalloc_end + 1;
2777 1022 : prealloc_len = end + 1 - prealloc_start;
2778 : }
2779 :
2780 153786 : if (!checked_extent_shared && fieinfo->fi_extents_max) {
2781 117661 : ret = btrfs_is_data_extent_shared(inode,
2782 : disk_bytenr,
2783 : extent_gen,
2784 : backref_ctx);
2785 117661 : if (ret < 0)
2786 : return ret;
2787 117661 : else if (ret > 0)
2788 4629 : prealloc_flags |= FIEMAP_EXTENT_SHARED;
2789 : }
2790 153786 : ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
2791 : disk_bytenr + extent_offset,
2792 : prealloc_len, prealloc_flags);
2793 153784 : if (ret)
2794 456 : return ret;
2795 : }
2796 :
2797 : return 0;
2798 : }
2799 :
2800 49030 : static int fiemap_find_last_extent_offset(struct btrfs_inode *inode,
2801 : struct btrfs_path *path,
2802 : u64 *last_extent_end_ret)
2803 : {
2804 49030 : const u64 ino = btrfs_ino(inode);
2805 49030 : struct btrfs_root *root = inode->root;
2806 49030 : struct extent_buffer *leaf;
2807 49030 : struct btrfs_file_extent_item *ei;
2808 49030 : struct btrfs_key key;
2809 49030 : u64 disk_bytenr;
2810 49030 : int ret;
2811 :
2812 : /*
2813 : * Lookup the last file extent. We're not using i_size here because
2814 : * there might be preallocation past i_size.
2815 : */
2816 49030 : ret = btrfs_lookup_file_extent(NULL, root, path, ino, (u64)-1, 0);
2817 : /* There can't be a file extent item at offset (u64)-1 */
2818 49028 : ASSERT(ret != 0);
2819 49028 : if (ret < 0)
2820 : return ret;
2821 :
2822 : /*
2823 : * For a non-existing key, btrfs_search_slot() always leaves us at a
2824 : * slot > 0, except if the btree is empty, which is impossible because
2825 : * at least it has the inode item for this inode and all the items for
2826 : * the root inode 256.
2827 : */
2828 49028 : ASSERT(path->slots[0] > 0);
2829 49028 : path->slots[0]--;
2830 49028 : leaf = path->nodes[0];
2831 49028 : btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2832 49026 : if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) {
2833 : /* No file extent items in the subvolume tree. */
2834 752 : *last_extent_end_ret = 0;
2835 752 : return 0;
2836 : }
2837 :
2838 : /*
2839 : * For an inline extent, the disk_bytenr is where inline data starts at,
2840 : * so first check if we have an inline extent item before checking if we
2841 : * have an implicit hole (disk_bytenr == 0).
2842 : */
2843 48274 : ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
2844 48272 : if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) {
2845 1 : *last_extent_end_ret = btrfs_file_extent_end(path);
2846 1 : return 0;
2847 : }
2848 :
2849 : /*
2850 : * Find the last file extent item that is not a hole (when NO_HOLES is
2851 : * not enabled). This should take at most 2 iterations in the worst
2852 : * case: we have one hole file extent item at slot 0 of a leaf and
2853 : * another hole file extent item as the last item in the previous leaf.
2854 : * This is because we merge file extent items that represent holes.
2855 : */
2856 48276 : disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2857 48275 : while (disk_bytenr == 0) {
2858 0 : ret = btrfs_previous_item(root, path, ino, BTRFS_EXTENT_DATA_KEY);
2859 0 : if (ret < 0) {
2860 0 : return ret;
2861 0 : } else if (ret > 0) {
2862 : /* No file extent items that are not holes. */
2863 0 : *last_extent_end_ret = 0;
2864 0 : return 0;
2865 : }
2866 0 : leaf = path->nodes[0];
2867 0 : ei = btrfs_item_ptr(leaf, path->slots[0],
2868 : struct btrfs_file_extent_item);
2869 0 : disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2870 : }
2871 :
2872 48277 : *last_extent_end_ret = btrfs_file_extent_end(path);
2873 48276 : return 0;
2874 : }
2875 :
2876 49030 : int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
2877 : u64 start, u64 len)
2878 : {
2879 49030 : const u64 ino = btrfs_ino(inode);
2880 49030 : struct extent_state *cached_state = NULL;
2881 49030 : struct extent_state *delalloc_cached_state = NULL;
2882 49030 : struct btrfs_path *path;
2883 49030 : struct fiemap_cache cache = { 0 };
2884 49030 : struct btrfs_backref_share_check_ctx *backref_ctx;
2885 49030 : u64 last_extent_end;
2886 49030 : u64 prev_extent_end;
2887 49030 : u64 lockstart;
2888 49030 : u64 lockend;
2889 49030 : bool stopped = false;
2890 49030 : int ret;
2891 :
2892 49030 : backref_ctx = btrfs_alloc_backref_share_check_ctx();
2893 49030 : path = btrfs_alloc_path();
2894 49032 : if (!backref_ctx || !path) {
2895 0 : ret = -ENOMEM;
2896 0 : goto out;
2897 : }
2898 :
2899 49032 : lockstart = round_down(start, inode->root->fs_info->sectorsize);
2900 49032 : lockend = round_up(start + len, inode->root->fs_info->sectorsize);
2901 49032 : prev_extent_end = lockstart;
2902 :
2903 49032 : btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
2904 49031 : lock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
2905 :
2906 49030 : ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end);
2907 49027 : if (ret < 0)
2908 0 : goto out_unlock;
2909 49027 : btrfs_release_path(path);
2910 :
2911 49031 : path->reada = READA_FORWARD;
2912 49031 : ret = fiemap_search_slot(inode, path, lockstart);
2913 49030 : if (ret < 0) {
2914 0 : goto out_unlock;
2915 49030 : } else if (ret > 0) {
2916 : /*
2917 : * No file extent item found, but we may have delalloc between
2918 : * the current offset and i_size. So check for that.
2919 : */
2920 85 : ret = 0;
2921 85 : goto check_eof_delalloc;
2922 : }
2923 :
2924 1347598 : while (prev_extent_end < lockend) {
2925 1343010 : struct extent_buffer *leaf = path->nodes[0];
2926 1343010 : struct btrfs_file_extent_item *ei;
2927 1343010 : struct btrfs_key key;
2928 1343010 : u64 extent_end;
2929 1343010 : u64 extent_len;
2930 1343010 : u64 extent_offset = 0;
2931 1343010 : u64 extent_gen;
2932 1343010 : u64 disk_bytenr = 0;
2933 1343010 : u64 flags = 0;
2934 1343010 : int extent_type;
2935 1343010 : u8 compression;
2936 :
2937 1343010 : btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2938 1343032 : if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2939 : break;
2940 :
2941 1333184 : extent_end = btrfs_file_extent_end(path);
2942 :
2943 : /*
2944 : * The first iteration can leave us at an extent item that ends
2945 : * before our range's start. Move to the next item.
2946 : */
2947 1333106 : if (extent_end <= lockstart)
2948 7871 : goto next_item;
2949 :
2950 1325235 : backref_ctx->curr_leaf_bytenr = leaf->start;
2951 :
2952 : /* We have in implicit hole (NO_HOLES feature enabled). */
2953 1325235 : if (prev_extent_end < key.offset) {
2954 254641 : const u64 range_end = min(key.offset, lockend) - 1;
2955 :
2956 254641 : ret = fiemap_process_hole(inode, fieinfo, &cache,
2957 : &delalloc_cached_state,
2958 : backref_ctx, 0, 0, 0,
2959 : prev_extent_end, range_end);
2960 254724 : if (ret < 0) {
2961 0 : goto out_unlock;
2962 254724 : } else if (ret > 0) {
2963 : /* fiemap_fill_next_extent() told us to stop. */
2964 : stopped = true;
2965 : break;
2966 : }
2967 :
2968 : /* We've reached the end of the fiemap range, stop. */
2969 254724 : if (key.offset >= lockend) {
2970 : stopped = true;
2971 : break;
2972 : }
2973 : }
2974 :
2975 1321970 : extent_len = extent_end - key.offset;
2976 1321970 : ei = btrfs_item_ptr(leaf, path->slots[0],
2977 : struct btrfs_file_extent_item);
2978 1322041 : compression = btrfs_file_extent_compression(leaf, ei);
2979 1322031 : extent_type = btrfs_file_extent_type(leaf, ei);
2980 1322034 : extent_gen = btrfs_file_extent_generation(leaf, ei);
2981 :
2982 1322009 : if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
2983 1322006 : disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2984 1322019 : if (compression == BTRFS_COMPRESS_NONE)
2985 643206 : extent_offset = btrfs_file_extent_offset(leaf, ei);
2986 : }
2987 :
2988 1322031 : if (compression != BTRFS_COMPRESS_NONE)
2989 678814 : flags |= FIEMAP_EXTENT_ENCODED;
2990 :
2991 1322031 : if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
2992 1 : flags |= FIEMAP_EXTENT_DATA_INLINE;
2993 1 : flags |= FIEMAP_EXTENT_NOT_ALIGNED;
2994 1 : ret = emit_fiemap_extent(fieinfo, &cache, key.offset, 0,
2995 : extent_len, flags);
2996 1322030 : } else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
2997 172580 : ret = fiemap_process_hole(inode, fieinfo, &cache,
2998 : &delalloc_cached_state,
2999 : backref_ctx,
3000 : disk_bytenr, extent_offset,
3001 : extent_gen, key.offset,
3002 : extent_end - 1);
3003 1149450 : } else if (disk_bytenr == 0) {
3004 : /* We have an explicit hole. */
3005 3 : ret = fiemap_process_hole(inode, fieinfo, &cache,
3006 : &delalloc_cached_state,
3007 : backref_ctx, 0, 0, 0,
3008 : key.offset, extent_end - 1);
3009 : } else {
3010 : /* We have a regular extent. */
3011 1149447 : if (fieinfo->fi_extents_max) {
3012 1055742 : ret = btrfs_is_data_extent_shared(inode,
3013 : disk_bytenr,
3014 : extent_gen,
3015 : backref_ctx);
3016 1055799 : if (ret < 0)
3017 0 : goto out_unlock;
3018 1055799 : else if (ret > 0)
3019 449751 : flags |= FIEMAP_EXTENT_SHARED;
3020 : }
3021 :
3022 1149504 : ret = emit_fiemap_extent(fieinfo, &cache, key.offset,
3023 : disk_bytenr + extent_offset,
3024 : extent_len, flags);
3025 : }
3026 :
3027 1322028 : if (ret < 0) {
3028 0 : goto out_unlock;
3029 1322028 : } else if (ret > 0) {
3030 : /* fiemap_fill_next_extent() told us to stop. */
3031 : stopped = true;
3032 : break;
3033 : }
3034 :
3035 : prev_extent_end = extent_end;
3036 1307461 : next_item:
3037 1307461 : if (fatal_signal_pending(current)) {
3038 0 : ret = -EINTR;
3039 0 : goto out_unlock;
3040 : }
3041 :
3042 1307335 : ret = fiemap_next_leaf_item(inode, path);
3043 1307368 : if (ret < 0) {
3044 0 : goto out_unlock;
3045 1307368 : } else if (ret > 0) {
3046 : /* No more file extent items for this inode. */
3047 : break;
3048 : }
3049 1298645 : cond_resched();
3050 : }
3051 :
3052 4588 : check_eof_delalloc:
3053 : /*
3054 : * Release (and free) the path before emitting any final entries to
3055 : * fiemap_fill_next_extent() to keep lockdep happy. This is because
3056 : * once we find no more file extent items exist, we may have a
3057 : * non-cloned leaf, and fiemap_fill_next_extent() can trigger page
3058 : * faults when copying data to the user space buffer.
3059 : */
3060 49030 : btrfs_free_path(path);
3061 49028 : path = NULL;
3062 :
3063 49028 : if (!stopped && prev_extent_end < lockend) {
3064 18170 : ret = fiemap_process_hole(inode, fieinfo, &cache,
3065 : &delalloc_cached_state, backref_ctx,
3066 : 0, 0, 0, prev_extent_end, lockend - 1);
3067 18169 : if (ret < 0)
3068 0 : goto out_unlock;
3069 : prev_extent_end = lockend;
3070 : }
3071 :
3072 49027 : if (cache.cached && cache.offset + cache.len >= last_extent_end) {
3073 16659 : const u64 i_size = i_size_read(&inode->vfs_inode);
3074 :
3075 16659 : if (prev_extent_end < i_size) {
3076 6 : u64 delalloc_start;
3077 6 : u64 delalloc_end;
3078 6 : bool delalloc;
3079 :
3080 6 : delalloc = btrfs_find_delalloc_in_range(inode,
3081 : prev_extent_end,
3082 : i_size - 1,
3083 : &delalloc_cached_state,
3084 : &delalloc_start,
3085 : &delalloc_end);
3086 6 : if (!delalloc)
3087 0 : cache.flags |= FIEMAP_EXTENT_LAST;
3088 : } else {
3089 16653 : cache.flags |= FIEMAP_EXTENT_LAST;
3090 : }
3091 : }
3092 :
3093 49027 : ret = emit_last_fiemap_cache(fieinfo, &cache);
3094 :
3095 49023 : out_unlock:
3096 49023 : unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
3097 49026 : btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
3098 49024 : out:
3099 49024 : free_extent_state(delalloc_cached_state);
3100 49026 : btrfs_free_backref_share_ctx(backref_ctx);
3101 49014 : btrfs_free_path(path);
3102 49010 : return ret;
3103 : }
3104 :
3105 : static void __free_extent_buffer(struct extent_buffer *eb)
3106 : {
3107 30371856 : kmem_cache_free(extent_buffer_cache, eb);
3108 : }
3109 :
3110 42345053 : static int extent_buffer_under_io(const struct extent_buffer *eb)
3111 : {
3112 42345053 : return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
3113 0 : test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3114 : }
3115 :
3116 0 : static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page)
3117 : {
3118 0 : struct btrfs_subpage *subpage;
3119 :
3120 0 : lockdep_assert_held(&page->mapping->private_lock);
3121 :
3122 0 : if (PagePrivate(page)) {
3123 0 : subpage = (struct btrfs_subpage *)page->private;
3124 0 : if (atomic_read(&subpage->eb_refs))
3125 : return true;
3126 : /*
3127 : * Even there is no eb refs here, we may still have
3128 : * end_page_read() call relying on page::private.
3129 : */
3130 0 : if (atomic_read(&subpage->readers))
3131 0 : return true;
3132 : }
3133 : return false;
3134 : }
3135 :
3136 120819967 : static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
3137 : {
3138 120819967 : struct btrfs_fs_info *fs_info = eb->fs_info;
3139 120819967 : const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3140 :
3141 : /*
3142 : * For mapped eb, we're going to change the page private, which should
3143 : * be done under the private_lock.
3144 : */
3145 120819967 : if (mapped)
3146 43482305 : spin_lock(&page->mapping->private_lock);
3147 :
3148 120821561 : if (!PagePrivate(page)) {
3149 0 : if (mapped)
3150 0 : spin_unlock(&page->mapping->private_lock);
3151 0 : return;
3152 : }
3153 :
3154 120821561 : if (fs_info->nodesize >= PAGE_SIZE) {
3155 : /*
3156 : * We do this since we'll remove the pages after we've
3157 : * removed the eb from the radix tree, so we could race
3158 : * and have this page now attached to the new eb. So
3159 : * only clear page_private if it's still connected to
3160 : * this eb.
3161 : */
3162 120821561 : if (PagePrivate(page) &&
3163 120821561 : page->private == (unsigned long)eb) {
3164 120821410 : BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3165 241641073 : BUG_ON(PageDirty(page));
3166 241639060 : BUG_ON(PageWriteback(page));
3167 : /*
3168 : * We need to make sure we haven't be attached
3169 : * to a new eb.
3170 : */
3171 120819397 : detach_page_private(page);
3172 : }
3173 120807336 : if (mapped)
3174 43483899 : spin_unlock(&page->mapping->private_lock);
3175 120807335 : return;
3176 : }
3177 :
3178 : /*
3179 : * For subpage, we can have dummy eb with page private. In this case,
3180 : * we can directly detach the private as such page is only attached to
3181 : * one dummy eb, no sharing.
3182 : */
3183 0 : if (!mapped) {
3184 0 : btrfs_detach_subpage(fs_info, page);
3185 0 : return;
3186 : }
3187 :
3188 0 : btrfs_page_dec_eb_refs(fs_info, page);
3189 :
3190 : /*
3191 : * We can only detach the page private if there are no other ebs in the
3192 : * page range and no unfinished IO.
3193 : */
3194 0 : if (!page_range_has_eb(fs_info, page))
3195 0 : btrfs_detach_subpage(fs_info, page);
3196 :
3197 0 : spin_unlock(&page->mapping->private_lock);
3198 : }
3199 :
3200 : /* Release all pages attached to the extent buffer */
3201 30447726 : static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
3202 : {
3203 30447726 : int i;
3204 30447726 : int num_pages;
3205 :
3206 30447726 : ASSERT(!extent_buffer_under_io(eb));
3207 :
3208 30447002 : num_pages = num_extent_pages(eb);
3209 151270653 : for (i = 0; i < num_pages; i++) {
3210 120823221 : struct page *page = eb->pages[i];
3211 :
3212 120823221 : if (!page)
3213 152 : continue;
3214 :
3215 120823069 : detach_extent_buffer_page(eb, page);
3216 :
3217 : /* One for when we allocated the page */
3218 120814132 : put_page(page);
3219 : }
3220 30447432 : }
3221 :
3222 : /*
3223 : * Helper for releasing the extent buffer.
3224 : */
3225 38 : static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
3226 : {
3227 38 : btrfs_release_extent_buffer_pages(eb);
3228 38 : btrfs_leak_debug_del_eb(eb);
3229 38 : __free_extent_buffer(eb);
3230 38 : }
3231 :
3232 : static struct extent_buffer *
3233 30443986 : __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
3234 : unsigned long len)
3235 : {
3236 30443986 : struct extent_buffer *eb = NULL;
3237 :
3238 30443986 : eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
3239 30444807 : eb->start = start;
3240 30444807 : eb->len = len;
3241 30444807 : eb->fs_info = fs_info;
3242 30444807 : init_rwsem(&eb->lock);
3243 :
3244 30445589 : btrfs_leak_debug_add_eb(eb);
3245 :
3246 30445589 : spin_lock_init(&eb->refs_lock);
3247 30444644 : atomic_set(&eb->refs, 1);
3248 :
3249 30444644 : ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE);
3250 :
3251 30444644 : return eb;
3252 : }
3253 :
3254 19331554 : struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
3255 : {
3256 19331554 : int i;
3257 19331554 : struct extent_buffer *new;
3258 19331554 : int num_pages = num_extent_pages(src);
3259 19331554 : int ret;
3260 :
3261 19331554 : new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
3262 19332309 : if (new == NULL)
3263 : return NULL;
3264 :
3265 : /*
3266 : * Set UNMAPPED before calling btrfs_release_extent_buffer(), as
3267 : * btrfs_release_extent_buffer() have different behavior for
3268 : * UNMAPPED subpage extent buffer.
3269 : */
3270 19332309 : set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
3271 :
3272 19333599 : ret = btrfs_alloc_page_array(num_pages, new->pages);
3273 19332434 : if (ret) {
3274 0 : btrfs_release_extent_buffer(new);
3275 0 : return NULL;
3276 : }
3277 :
3278 96655878 : for (i = 0; i < num_pages; i++) {
3279 77322853 : int ret;
3280 77322853 : struct page *p = new->pages[i];
3281 :
3282 77322853 : ret = attach_extent_buffer_page(new, p, NULL);
3283 77326955 : if (ret < 0) {
3284 0 : btrfs_release_extent_buffer(new);
3285 0 : return NULL;
3286 : }
3287 154649066 : WARN_ON(PageDirty(p));
3288 77322111 : copy_page(page_address(p), page_address(src->pages[i]));
3289 : }
3290 19333025 : set_extent_buffer_uptodate(new);
3291 :
3292 19333025 : return new;
3293 : }
3294 :
3295 6347 : struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3296 : u64 start, unsigned long len)
3297 : {
3298 6347 : struct extent_buffer *eb;
3299 6347 : int num_pages;
3300 6347 : int i;
3301 6347 : int ret;
3302 :
3303 6347 : eb = __alloc_extent_buffer(fs_info, start, len);
3304 6347 : if (!eb)
3305 : return NULL;
3306 :
3307 6347 : num_pages = num_extent_pages(eb);
3308 6347 : ret = btrfs_alloc_page_array(num_pages, eb->pages);
3309 6347 : if (ret)
3310 0 : goto err;
3311 :
3312 31957 : for (i = 0; i < num_pages; i++) {
3313 25610 : struct page *p = eb->pages[i];
3314 :
3315 25610 : ret = attach_extent_buffer_page(eb, p, NULL);
3316 25610 : if (ret < 0)
3317 0 : goto err;
3318 : }
3319 :
3320 6347 : set_extent_buffer_uptodate(eb);
3321 6347 : btrfs_set_header_nritems(eb, 0);
3322 6347 : set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3323 :
3324 6347 : return eb;
3325 : err:
3326 0 : for (i = 0; i < num_pages; i++) {
3327 0 : if (eb->pages[i]) {
3328 0 : detach_extent_buffer_page(eb, eb->pages[i]);
3329 0 : __free_page(eb->pages[i]);
3330 : }
3331 : }
3332 0 : __free_extent_buffer(eb);
3333 0 : return NULL;
3334 : }
3335 :
3336 6347 : struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3337 : u64 start)
3338 : {
3339 6347 : return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
3340 : }
3341 :
3342 886503318 : static void check_buffer_tree_ref(struct extent_buffer *eb)
3343 : {
3344 886503318 : int refs;
3345 : /*
3346 : * The TREE_REF bit is first set when the extent_buffer is added
3347 : * to the radix tree. It is also reset, if unset, when a new reference
3348 : * is created by find_extent_buffer.
3349 : *
3350 : * It is only cleared in two cases: freeing the last non-tree
3351 : * reference to the extent_buffer when its STALE bit is set or
3352 : * calling release_folio when the tree reference is the only reference.
3353 : *
3354 : * In both cases, care is taken to ensure that the extent_buffer's
3355 : * pages are not under io. However, release_folio can be concurrently
3356 : * called with creating new references, which is prone to race
3357 : * conditions between the calls to check_buffer_tree_ref in those
3358 : * codepaths and clearing TREE_REF in try_release_extent_buffer.
3359 : *
3360 : * The actual lifetime of the extent_buffer in the radix tree is
3361 : * adequately protected by the refcount, but the TREE_REF bit and
3362 : * its corresponding reference are not. To protect against this
3363 : * class of races, we call check_buffer_tree_ref from the codepaths
3364 : * which trigger io. Note that once io is initiated, TREE_REF can no
3365 : * longer be cleared, so that is the moment at which any such race is
3366 : * best fixed.
3367 : */
3368 886503318 : refs = atomic_read(&eb->refs);
3369 1761897932 : if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3370 : return;
3371 :
3372 11083831 : spin_lock(&eb->refs_lock);
3373 11108763 : if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3374 11108786 : atomic_inc(&eb->refs);
3375 11108797 : spin_unlock(&eb->refs_lock);
3376 : }
3377 :
3378 572577100 : static void mark_extent_buffer_accessed(struct extent_buffer *eb,
3379 : struct page *accessed)
3380 : {
3381 572577100 : int num_pages, i;
3382 :
3383 572577100 : check_buffer_tree_ref(eb);
3384 :
3385 572437893 : num_pages = num_extent_pages(eb);
3386 2846945922 : for (i = 0; i < num_pages; i++) {
3387 2274090618 : struct page *p = eb->pages[i];
3388 :
3389 2274090618 : if (p != accessed)
3390 2274090580 : mark_page_accessed(p);
3391 : }
3392 572855304 : }
3393 :
3394 585090905 : struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
3395 : u64 start)
3396 : {
3397 585090905 : struct extent_buffer *eb;
3398 :
3399 585090905 : eb = find_extent_buffer_nolock(fs_info, start);
3400 585600517 : if (!eb)
3401 : return NULL;
3402 : /*
3403 : * Lock our eb's refs_lock to avoid races with free_extent_buffer().
3404 : * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
3405 : * another task running free_extent_buffer() might have seen that flag
3406 : * set, eb->refs == 2, that the buffer isn't under IO (dirty and
3407 : * writeback flags not set) and it's still in the tree (flag
3408 : * EXTENT_BUFFER_TREE_REF set), therefore being in the process of
3409 : * decrementing the extent buffer's reference count twice. So here we
3410 : * could race and increment the eb's reference count, clear its stale
3411 : * flag, mark it as dirty and drop our reference before the other task
3412 : * finishes executing free_extent_buffer, which would later result in
3413 : * an attempt to free an extent buffer that is dirty.
3414 : */
3415 1145365536 : if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
3416 1370463 : spin_lock(&eb->refs_lock);
3417 1370461 : spin_unlock(&eb->refs_lock);
3418 : }
3419 572682765 : mark_extent_buffer_accessed(eb, NULL);
3420 572682765 : return eb;
3421 : }
3422 :
3423 : #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3424 : struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
3425 : u64 start)
3426 : {
3427 : struct extent_buffer *eb, *exists = NULL;
3428 : int ret;
3429 :
3430 : eb = find_extent_buffer(fs_info, start);
3431 : if (eb)
3432 : return eb;
3433 : eb = alloc_dummy_extent_buffer(fs_info, start);
3434 : if (!eb)
3435 : return ERR_PTR(-ENOMEM);
3436 : eb->fs_info = fs_info;
3437 : again:
3438 : ret = radix_tree_preload(GFP_NOFS);
3439 : if (ret) {
3440 : exists = ERR_PTR(ret);
3441 : goto free_eb;
3442 : }
3443 : spin_lock(&fs_info->buffer_lock);
3444 : ret = radix_tree_insert(&fs_info->buffer_radix,
3445 : start >> fs_info->sectorsize_bits, eb);
3446 : spin_unlock(&fs_info->buffer_lock);
3447 : radix_tree_preload_end();
3448 : if (ret == -EEXIST) {
3449 : exists = find_extent_buffer(fs_info, start);
3450 : if (exists)
3451 : goto free_eb;
3452 : else
3453 : goto again;
3454 : }
3455 : check_buffer_tree_ref(eb);
3456 : set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
3457 :
3458 : return eb;
3459 : free_eb:
3460 : btrfs_release_extent_buffer(eb);
3461 : return exists;
3462 : }
3463 : #endif
3464 :
3465 43483956 : static struct extent_buffer *grab_extent_buffer(
3466 : struct btrfs_fs_info *fs_info, struct page *page)
3467 : {
3468 43483956 : struct extent_buffer *exists;
3469 :
3470 : /*
3471 : * For subpage case, we completely rely on radix tree to ensure we
3472 : * don't try to insert two ebs for the same bytenr. So here we always
3473 : * return NULL and just continue.
3474 : */
3475 43483956 : if (fs_info->nodesize < PAGE_SIZE)
3476 : return NULL;
3477 :
3478 : /* Page not yet attached to an extent buffer */
3479 43483956 : if (!PagePrivate(page))
3480 : return NULL;
3481 :
3482 : /*
3483 : * We could have already allocated an eb for this page and attached one
3484 : * so lets see if we can get a ref on the existing eb, and if we can we
3485 : * know it's good and we can just return that one, else we know we can
3486 : * just overwrite page->private.
3487 : */
3488 41 : exists = (struct extent_buffer *)page->private;
3489 82 : if (atomic_inc_not_zero(&exists->refs))
3490 : return exists;
3491 :
3492 6 : WARN_ON(PageDirty(page));
3493 3 : detach_page_private(page);
3494 3 : return NULL;
3495 : }
3496 :
3497 45662988 : static int check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
3498 : {
3499 45662988 : if (!IS_ALIGNED(start, fs_info->sectorsize)) {
3500 0 : btrfs_err(fs_info, "bad tree block start %llu", start);
3501 0 : return -EINVAL;
3502 : }
3503 :
3504 45662988 : if (fs_info->nodesize < PAGE_SIZE &&
3505 0 : offset_in_page(start) + fs_info->nodesize > PAGE_SIZE) {
3506 0 : btrfs_err(fs_info,
3507 : "tree block crosses page boundary, start %llu nodesize %u",
3508 : start, fs_info->nodesize);
3509 0 : return -EINVAL;
3510 : }
3511 45662988 : if (fs_info->nodesize >= PAGE_SIZE &&
3512 45662988 : !PAGE_ALIGNED(start)) {
3513 0 : btrfs_err(fs_info,
3514 : "tree block is not page aligned, start %llu nodesize %u",
3515 : start, fs_info->nodesize);
3516 0 : return -EINVAL;
3517 : }
3518 : return 0;
3519 : }
3520 :
3521 45662783 : struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
3522 : u64 start, u64 owner_root, int level)
3523 : {
3524 45662783 : unsigned long len = fs_info->nodesize;
3525 45662783 : int num_pages;
3526 45662783 : int i;
3527 45662783 : unsigned long index = start >> PAGE_SHIFT;
3528 45662783 : struct extent_buffer *eb;
3529 45662783 : struct extent_buffer *exists = NULL;
3530 45662783 : struct page *p;
3531 45662783 : struct address_space *mapping = fs_info->btree_inode->i_mapping;
3532 45662783 : u64 lockdep_owner = owner_root;
3533 45662783 : int uptodate = 1;
3534 45662783 : int ret;
3535 :
3536 45662783 : if (check_eb_alignment(fs_info, start))
3537 : return ERR_PTR(-EINVAL);
3538 :
3539 : #if BITS_PER_LONG == 32
3540 : if (start >= MAX_LFS_FILESIZE) {
3541 : btrfs_err_rl(fs_info,
3542 : "extent buffer %llu is beyond 32bit page cache limit", start);
3543 : btrfs_err_32bit_limit(fs_info);
3544 : return ERR_PTR(-EOVERFLOW);
3545 : }
3546 : if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3547 : btrfs_warn_32bit_limit(fs_info);
3548 : #endif
3549 :
3550 45662802 : eb = find_extent_buffer(fs_info, start);
3551 45663577 : if (eb)
3552 : return eb;
3553 :
3554 11108359 : eb = __alloc_extent_buffer(fs_info, start, len);
3555 11108486 : if (!eb)
3556 : return ERR_PTR(-ENOMEM);
3557 :
3558 : /*
3559 : * The reloc trees are just snapshots, so we need them to appear to be
3560 : * just like any other fs tree WRT lockdep.
3561 : */
3562 11108486 : if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3563 : lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3564 :
3565 11108486 : btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level);
3566 :
3567 11108486 : num_pages = num_extent_pages(eb);
3568 54591696 : for (i = 0; i < num_pages; i++, index++) {
3569 43482967 : struct btrfs_subpage *prealloc = NULL;
3570 :
3571 43482967 : p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
3572 43481587 : if (!p) {
3573 0 : exists = ERR_PTR(-ENOMEM);
3574 0 : goto free_eb;
3575 : }
3576 :
3577 : /*
3578 : * Preallocate page->private for subpage case, so that we won't
3579 : * allocate memory with private_lock hold. The memory will be
3580 : * freed by attach_extent_buffer_page() or freed manually if
3581 : * we exit earlier.
3582 : *
3583 : * Although we have ensured one subpage eb can only have one
3584 : * page, but it may change in the future for 16K page size
3585 : * support, so we still preallocate the memory in the loop.
3586 : */
3587 43481587 : if (fs_info->nodesize < PAGE_SIZE) {
3588 0 : prealloc = btrfs_alloc_subpage(fs_info, BTRFS_SUBPAGE_METADATA);
3589 0 : if (IS_ERR(prealloc)) {
3590 0 : ret = PTR_ERR(prealloc);
3591 0 : unlock_page(p);
3592 0 : put_page(p);
3593 0 : exists = ERR_PTR(ret);
3594 0 : goto free_eb;
3595 : }
3596 : }
3597 :
3598 43481587 : spin_lock(&mapping->private_lock);
3599 43483957 : exists = grab_extent_buffer(fs_info, p);
3600 43483956 : if (exists) {
3601 38 : spin_unlock(&mapping->private_lock);
3602 38 : unlock_page(p);
3603 38 : put_page(p);
3604 38 : mark_extent_buffer_accessed(exists, p);
3605 38 : btrfs_free_subpage(prealloc);
3606 38 : goto free_eb;
3607 : }
3608 : /* Should not fail, as we have preallocated the memory */
3609 43483918 : ret = attach_extent_buffer_page(eb, p, prealloc);
3610 43483918 : ASSERT(!ret);
3611 : /*
3612 : * To inform we have extra eb under allocation, so that
3613 : * detach_extent_buffer_page() won't release the page private
3614 : * when the eb hasn't yet been inserted into radix tree.
3615 : *
3616 : * The ref will be decreased when the eb released the page, in
3617 : * detach_extent_buffer_page().
3618 : * Thus needs no special handling in error path.
3619 : */
3620 43483918 : btrfs_page_inc_eb_refs(fs_info, p);
3621 43483919 : spin_unlock(&mapping->private_lock);
3622 :
3623 43483916 : WARN_ON(btrfs_page_test_dirty(fs_info, p, eb->start, eb->len));
3624 43483732 : eb->pages[i] = p;
3625 43483732 : if (!btrfs_page_test_uptodate(fs_info, p, eb->start, eb->len))
3626 12007937 : uptodate = 0;
3627 :
3628 : /*
3629 : * We can't unlock the pages just yet since the extent buffer
3630 : * hasn't been properly inserted in the radix tree, this
3631 : * opens a race with btree_release_folio which can free a page
3632 : * while we are still filling in all pages for the buffer and
3633 : * we could crash.
3634 : */
3635 : }
3636 11108729 : if (uptodate)
3637 8061239 : set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3638 3047490 : again:
3639 11108788 : ret = radix_tree_preload(GFP_NOFS);
3640 11108246 : if (ret) {
3641 0 : exists = ERR_PTR(ret);
3642 0 : goto free_eb;
3643 : }
3644 :
3645 11108246 : spin_lock(&fs_info->buffer_lock);
3646 11108825 : ret = radix_tree_insert(&fs_info->buffer_radix,
3647 11108825 : start >> fs_info->sectorsize_bits, eb);
3648 11108824 : spin_unlock(&fs_info->buffer_lock);
3649 11108823 : radix_tree_preload_end();
3650 11108741 : if (ret == -EEXIST) {
3651 0 : exists = find_extent_buffer(fs_info, start);
3652 0 : if (exists)
3653 0 : goto free_eb;
3654 : else
3655 0 : goto again;
3656 : }
3657 : /* add one reference for the tree */
3658 11108741 : check_buffer_tree_ref(eb);
3659 11108690 : set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
3660 :
3661 : /*
3662 : * Now it's safe to unlock the pages because any calls to
3663 : * btree_release_folio will correctly detect that a page belongs to a
3664 : * live buffer and won't free them prematurely.
3665 : */
3666 65699508 : for (i = 0; i < num_pages; i++)
3667 43482131 : unlock_page(eb->pages[i]);
3668 : return eb;
3669 :
3670 38 : free_eb:
3671 38 : WARN_ON(!atomic_dec_and_test(&eb->refs));
3672 190 : for (i = 0; i < num_pages; i++) {
3673 152 : if (eb->pages[i])
3674 0 : unlock_page(eb->pages[i]);
3675 : }
3676 :
3677 38 : btrfs_release_extent_buffer(eb);
3678 38 : return exists;
3679 : }
3680 :
3681 30371818 : static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3682 : {
3683 30371818 : struct extent_buffer *eb =
3684 30371818 : container_of(head, struct extent_buffer, rcu_head);
3685 :
3686 30371818 : __free_extent_buffer(eb);
3687 30391989 : }
3688 :
3689 1005602583 : static int release_extent_buffer(struct extent_buffer *eb)
3690 : __releases(&eb->refs_lock)
3691 : {
3692 1005602583 : lockdep_assert_held(&eb->refs_lock);
3693 :
3694 1005602583 : WARN_ON(atomic_read(&eb->refs) == 0);
3695 1005602583 : if (atomic_dec_and_test(&eb->refs)) {
3696 30447291 : if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
3697 11108757 : struct btrfs_fs_info *fs_info = eb->fs_info;
3698 :
3699 11108757 : spin_unlock(&eb->refs_lock);
3700 :
3701 11108761 : spin_lock(&fs_info->buffer_lock);
3702 11108820 : radix_tree_delete(&fs_info->buffer_radix,
3703 11108820 : eb->start >> fs_info->sectorsize_bits);
3704 11108820 : spin_unlock(&fs_info->buffer_lock);
3705 : } else {
3706 19339496 : spin_unlock(&eb->refs_lock);
3707 : }
3708 :
3709 30448587 : btrfs_leak_debug_del_eb(eb);
3710 : /* Should be safe to release our pages at this point */
3711 30448587 : btrfs_release_extent_buffer_pages(eb);
3712 : #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3713 : if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
3714 : __free_extent_buffer(eb);
3715 : return 1;
3716 : }
3717 : #endif
3718 30446943 : call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
3719 30446943 : return 1;
3720 : }
3721 975519224 : spin_unlock(&eb->refs_lock);
3722 :
3723 975519224 : return 0;
3724 : }
3725 :
3726 1134739575 : void free_extent_buffer(struct extent_buffer *eb)
3727 : {
3728 1134739575 : int refs;
3729 1134739575 : if (!eb)
3730 : return;
3731 :
3732 1134591802 : refs = atomic_read(&eb->refs);
3733 1134902817 : while (1) {
3734 1134902817 : if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
3735 160943088 : || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
3736 : refs == 1))
3737 : break;
3738 283211720 : if (atomic_try_cmpxchg(&eb->refs, &refs, refs - 1))
3739 : return;
3740 : }
3741 :
3742 993293456 : spin_lock(&eb->refs_lock);
3743 1539237183 : if (atomic_read(&eb->refs) == 2 &&
3744 1111245 : test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
3745 2035017 : !extent_buffer_under_io(eb) &&
3746 : test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3747 923769 : atomic_dec(&eb->refs);
3748 :
3749 : /*
3750 : * I know this is terrible, but it's temporary until we stop tracking
3751 : * the uptodate bits and such for the extent buffers.
3752 : */
3753 994118785 : release_extent_buffer(eb);
3754 : }
3755 :
3756 8753170 : void free_extent_buffer_stale(struct extent_buffer *eb)
3757 : {
3758 8753170 : if (!eb)
3759 : return;
3760 :
3761 8753170 : spin_lock(&eb->refs_lock);
3762 8753183 : set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
3763 :
3764 15946635 : if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
3765 : test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3766 7193464 : atomic_dec(&eb->refs);
3767 8753179 : release_extent_buffer(eb);
3768 : }
3769 :
3770 498741 : static void btree_clear_page_dirty(struct page *page)
3771 : {
3772 498741 : ASSERT(PageDirty(page));
3773 498741 : ASSERT(PageLocked(page));
3774 498741 : clear_page_dirty_for_io(page);
3775 498742 : xa_lock_irq(&page->mapping->i_pages);
3776 997484 : if (!PageDirty(page))
3777 498742 : __xa_clear_mark(&page->mapping->i_pages,
3778 : page_index(page), PAGECACHE_TAG_DIRTY);
3779 498742 : xa_unlock_irq(&page->mapping->i_pages);
3780 498742 : }
3781 :
3782 0 : static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb)
3783 : {
3784 0 : struct btrfs_fs_info *fs_info = eb->fs_info;
3785 0 : struct page *page = eb->pages[0];
3786 0 : bool last;
3787 :
3788 : /* btree_clear_page_dirty() needs page locked */
3789 0 : lock_page(page);
3790 0 : last = btrfs_subpage_clear_and_test_dirty(fs_info, page, eb->start,
3791 0 : eb->len);
3792 0 : if (last)
3793 0 : btree_clear_page_dirty(page);
3794 0 : unlock_page(page);
3795 0 : WARN_ON(atomic_read(&eb->refs) == 0);
3796 0 : }
3797 :
3798 17769346 : void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
3799 : struct extent_buffer *eb)
3800 : {
3801 17769346 : struct btrfs_fs_info *fs_info = eb->fs_info;
3802 17769346 : int i;
3803 17769346 : int num_pages;
3804 17769346 : struct page *page;
3805 :
3806 17769346 : btrfs_assert_tree_write_locked(eb);
3807 :
3808 17769346 : if (trans && btrfs_header_generation(eb) != trans->transid)
3809 : return;
3810 :
3811 10787519 : if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
3812 : return;
3813 :
3814 125416 : percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len,
3815 : fs_info->dirty_metadata_batch);
3816 :
3817 125416 : if (eb->fs_info->nodesize < PAGE_SIZE)
3818 0 : return clear_subpage_extent_buffer_dirty(eb);
3819 :
3820 125416 : num_pages = num_extent_pages(eb);
3821 :
3822 624158 : for (i = 0; i < num_pages; i++) {
3823 498742 : page = eb->pages[i];
3824 997484 : if (!PageDirty(page))
3825 0 : continue;
3826 498742 : lock_page(page);
3827 498741 : btree_clear_page_dirty(page);
3828 498742 : unlock_page(page);
3829 : }
3830 125416 : WARN_ON(atomic_read(&eb->refs) == 0);
3831 : }
3832 :
3833 303650605 : void set_extent_buffer_dirty(struct extent_buffer *eb)
3834 : {
3835 303650605 : int i;
3836 303650605 : int num_pages;
3837 303650605 : bool was_dirty;
3838 :
3839 303650605 : check_buffer_tree_ref(eb);
3840 :
3841 303639122 : was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
3842 :
3843 303722884 : num_pages = num_extent_pages(eb);
3844 303722884 : WARN_ON(atomic_read(&eb->refs) == 0);
3845 303722884 : WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
3846 :
3847 303722884 : if (!was_dirty) {
3848 9007410 : bool subpage = eb->fs_info->nodesize < PAGE_SIZE;
3849 :
3850 : /*
3851 : * For subpage case, we can have other extent buffers in the
3852 : * same page, and in clear_subpage_extent_buffer_dirty() we
3853 : * have to clear page dirty without subpage lock held.
3854 : * This can cause race where our page gets dirty cleared after
3855 : * we just set it.
3856 : *
3857 : * Thankfully, clear_subpage_extent_buffer_dirty() has locked
3858 : * its page for other reasons, we can use page lock to prevent
3859 : * the above race.
3860 : */
3861 9007410 : if (subpage)
3862 0 : lock_page(eb->pages[0]);
3863 44057432 : for (i = 0; i < num_pages; i++)
3864 35049911 : btrfs_page_set_dirty(eb->fs_info, eb->pages[i],
3865 35049911 : eb->start, eb->len);
3866 9007521 : if (subpage)
3867 0 : unlock_page(eb->pages[0]);
3868 9007521 : percpu_counter_add_batch(&eb->fs_info->dirty_metadata_bytes,
3869 9007521 : eb->len,
3870 9007521 : eb->fs_info->dirty_metadata_batch);
3871 : }
3872 : #ifdef CONFIG_BTRFS_DEBUG
3873 : for (i = 0; i < num_pages; i++)
3874 : ASSERT(PageDirty(eb->pages[i]));
3875 : #endif
3876 303722992 : }
3877 :
3878 3217 : void clear_extent_buffer_uptodate(struct extent_buffer *eb)
3879 : {
3880 3217 : struct btrfs_fs_info *fs_info = eb->fs_info;
3881 3217 : struct page *page;
3882 3217 : int num_pages;
3883 3217 : int i;
3884 :
3885 3217 : clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3886 3217 : num_pages = num_extent_pages(eb);
3887 16307 : for (i = 0; i < num_pages; i++) {
3888 13090 : page = eb->pages[i];
3889 13090 : if (!page)
3890 0 : continue;
3891 :
3892 : /*
3893 : * This is special handling for metadata subpage, as regular
3894 : * btrfs_is_subpage() can not handle cloned/dummy metadata.
3895 : */
3896 13090 : if (fs_info->nodesize >= PAGE_SIZE)
3897 13090 : ClearPageUptodate(page);
3898 : else
3899 0 : btrfs_subpage_clear_uptodate(fs_info, page, eb->start,
3900 0 : eb->len);
3901 : }
3902 3217 : }
3903 :
3904 28448346 : void set_extent_buffer_uptodate(struct extent_buffer *eb)
3905 : {
3906 28448346 : struct btrfs_fs_info *fs_info = eb->fs_info;
3907 28448346 : struct page *page;
3908 28448346 : int num_pages;
3909 28448346 : int i;
3910 :
3911 28448346 : set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3912 28449562 : num_pages = num_extent_pages(eb);
3913 141259048 : for (i = 0; i < num_pages; i++) {
3914 112809728 : page = eb->pages[i];
3915 :
3916 : /*
3917 : * This is special handling for metadata subpage, as regular
3918 : * btrfs_is_subpage() can not handle cloned/dummy metadata.
3919 : */
3920 112809728 : if (fs_info->nodesize >= PAGE_SIZE)
3921 112809728 : SetPageUptodate(page);
3922 : else
3923 0 : btrfs_subpage_set_uptodate(fs_info, page, eb->start,
3924 0 : eb->len);
3925 : }
3926 28449320 : }
3927 :
3928 99360 : static void extent_buffer_read_end_io(struct btrfs_bio *bbio)
3929 : {
3930 99360 : struct extent_buffer *eb = bbio->private;
3931 99360 : struct btrfs_fs_info *fs_info = eb->fs_info;
3932 99360 : bool uptodate = !bbio->bio.bi_status;
3933 99360 : struct bvec_iter_all iter_all;
3934 99360 : struct bio_vec *bvec;
3935 99360 : u32 bio_offset = 0;
3936 :
3937 99360 : eb->read_mirror = bbio->mirror_num;
3938 :
3939 198719 : if (uptodate &&
3940 99360 : btrfs_validate_extent_buffer(eb, &bbio->parent_check) < 0)
3941 : uptodate = false;
3942 :
3943 99358 : if (uptodate) {
3944 99358 : set_extent_buffer_uptodate(eb);
3945 : } else {
3946 1 : clear_extent_buffer_uptodate(eb);
3947 1 : set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
3948 : }
3949 :
3950 499499 : bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
3951 400137 : u64 start = eb->start + bio_offset;
3952 400137 : struct page *page = bvec->bv_page;
3953 400137 : u32 len = bvec->bv_len;
3954 :
3955 400137 : if (uptodate)
3956 400133 : btrfs_page_set_uptodate(fs_info, page, start, len);
3957 : else
3958 4 : btrfs_page_clear_uptodate(fs_info, page, start, len);
3959 :
3960 400140 : bio_offset += len;
3961 : }
3962 :
3963 99359 : clear_bit(EXTENT_BUFFER_READING, &eb->bflags);
3964 99360 : smp_mb__after_atomic();
3965 99360 : wake_up_bit(&eb->bflags, EXTENT_BUFFER_READING);
3966 99359 : free_extent_buffer(eb);
3967 :
3968 99358 : bio_put(&bbio->bio);
3969 99360 : }
3970 :
3971 23671101 : int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num,
3972 : struct btrfs_tree_parent_check *check)
3973 : {
3974 23671101 : int num_pages = num_extent_pages(eb), i;
3975 23671101 : struct btrfs_bio *bbio;
3976 :
3977 47342202 : if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3978 : return 0;
3979 :
3980 : /*
3981 : * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
3982 : * operation, which could potentially still be in flight. In this case
3983 : * we simply want to return an error.
3984 : */
3985 99532 : if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
3986 : return -EIO;
3987 :
3988 : /* Someone else is already reading the buffer, just wait for it. */
3989 99529 : if (test_and_set_bit(EXTENT_BUFFER_READING, &eb->bflags))
3990 169 : goto done;
3991 :
3992 99360 : clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
3993 99360 : eb->read_mirror = 0;
3994 99360 : check_buffer_tree_ref(eb);
3995 99360 : atomic_inc(&eb->refs);
3996 :
3997 99360 : bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
3998 : REQ_OP_READ | REQ_META, eb->fs_info,
3999 : extent_buffer_read_end_io, eb);
4000 99360 : bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
4001 99360 : bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
4002 99360 : bbio->file_offset = eb->start;
4003 198720 : memcpy(&bbio->parent_check, check, sizeof(*check));
4004 99360 : if (eb->fs_info->nodesize < PAGE_SIZE) {
4005 0 : __bio_add_page(&bbio->bio, eb->pages[0], eb->len,
4006 0 : eb->start - page_offset(eb->pages[0]));
4007 : } else {
4008 499504 : for (i = 0; i < num_pages; i++)
4009 400145 : __bio_add_page(&bbio->bio, eb->pages[i], PAGE_SIZE, 0);
4010 : }
4011 99359 : btrfs_submit_bio(bbio, mirror_num);
4012 :
4013 99529 : done:
4014 99529 : if (wait == WAIT_COMPLETE) {
4015 90257 : wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
4016 90257 : if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4017 1 : return -EIO;
4018 : }
4019 :
4020 : return 0;
4021 : }
4022 :
4023 0 : static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
4024 : unsigned long len)
4025 : {
4026 0 : btrfs_warn(eb->fs_info,
4027 : "access to eb bytenr %llu len %lu out of range start %lu len %lu",
4028 : eb->start, eb->len, start, len);
4029 0 : WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4030 :
4031 0 : return true;
4032 : }
4033 :
4034 : /*
4035 : * Check if the [start, start + len) range is valid before reading/writing
4036 : * the eb.
4037 : * NOTE: @start and @len are offset inside the eb, not logical address.
4038 : *
4039 : * Caller should not touch the dst/src memory if this function returns error.
4040 : */
4041 11222595110 : static inline int check_eb_range(const struct extent_buffer *eb,
4042 : unsigned long start, unsigned long len)
4043 : {
4044 11222595110 : unsigned long offset;
4045 :
4046 : /* start, start + len should not go beyond eb->len nor overflow */
4047 11222595110 : if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
4048 0 : return report_eb_range(eb, start, len);
4049 :
4050 : return false;
4051 : }
4052 :
4053 10518548333 : void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
4054 : unsigned long start, unsigned long len)
4055 : {
4056 10518548333 : size_t cur;
4057 10518548333 : size_t offset;
4058 10518548333 : struct page *page;
4059 10518548333 : char *kaddr;
4060 10518548333 : char *dst = (char *)dstv;
4061 10518548333 : unsigned long i = get_eb_page_index(start);
4062 :
4063 10518548333 : if (check_eb_range(eb, start, len))
4064 : return;
4065 :
4066 10528573157 : offset = get_eb_offset_in_page(eb, start);
4067 :
4068 20741553141 : while (len > 0) {
4069 10227564153 : page = eb->pages[i];
4070 :
4071 10227564153 : cur = min(len, (PAGE_SIZE - offset));
4072 10227564153 : kaddr = page_address(page);
4073 20455128306 : memcpy(dst, kaddr + offset, cur);
4074 :
4075 10212979984 : dst += cur;
4076 10212979984 : len -= cur;
4077 10212979984 : offset = 0;
4078 10212979984 : i++;
4079 : }
4080 : }
4081 :
4082 7093 : int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
4083 : void __user *dstv,
4084 : unsigned long start, unsigned long len)
4085 : {
4086 7093 : size_t cur;
4087 7093 : size_t offset;
4088 7093 : struct page *page;
4089 7093 : char *kaddr;
4090 7093 : char __user *dst = (char __user *)dstv;
4091 7093 : unsigned long i = get_eb_page_index(start);
4092 7093 : int ret = 0;
4093 :
4094 7093 : WARN_ON(start > eb->len);
4095 7093 : WARN_ON(start + len > eb->start + eb->len);
4096 :
4097 7093 : offset = get_eb_offset_in_page(eb, start);
4098 :
4099 14214 : while (len > 0) {
4100 7121 : page = eb->pages[i];
4101 :
4102 7121 : cur = min(len, (PAGE_SIZE - offset));
4103 7121 : kaddr = page_address(page);
4104 7121 : if (copy_to_user_nofault(dst, kaddr + offset, cur)) {
4105 : ret = -EFAULT;
4106 : break;
4107 : }
4108 :
4109 7121 : dst += cur;
4110 7121 : len -= cur;
4111 7121 : offset = 0;
4112 7121 : i++;
4113 : }
4114 :
4115 7093 : return ret;
4116 : }
4117 :
4118 22060248 : int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
4119 : unsigned long start, unsigned long len)
4120 : {
4121 22060248 : size_t cur;
4122 22060248 : size_t offset;
4123 22060248 : struct page *page;
4124 22060248 : char *kaddr;
4125 22060248 : char *ptr = (char *)ptrv;
4126 22060248 : unsigned long i = get_eb_page_index(start);
4127 22060248 : int ret = 0;
4128 :
4129 22060248 : if (check_eb_range(eb, start, len))
4130 : return -EINVAL;
4131 :
4132 22060154 : offset = get_eb_offset_in_page(eb, start);
4133 :
4134 36438261 : while (len > 0) {
4135 22086258 : page = eb->pages[i];
4136 :
4137 22086258 : cur = min(len, (PAGE_SIZE - offset));
4138 :
4139 22086258 : kaddr = page_address(page);
4140 22086258 : ret = memcmp(ptr, kaddr + offset, cur);
4141 22086240 : if (ret)
4142 : break;
4143 :
4144 14378107 : ptr += cur;
4145 14378107 : len -= cur;
4146 14378107 : offset = 0;
4147 14378107 : i++;
4148 : }
4149 : return ret;
4150 : }
4151 :
4152 : /*
4153 : * Check that the extent buffer is uptodate.
4154 : *
4155 : * For regular sector size == PAGE_SIZE case, check if @page is uptodate.
4156 : * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
4157 : */
4158 315782864 : static void assert_eb_page_uptodate(const struct extent_buffer *eb,
4159 : struct page *page)
4160 : {
4161 315782864 : struct btrfs_fs_info *fs_info = eb->fs_info;
4162 :
4163 : /*
4164 : * If we are using the commit root we could potentially clear a page
4165 : * Uptodate while we're using the extent buffer that we've previously
4166 : * looked up. We don't want to complain in this case, as the page was
4167 : * valid before, we just didn't write it out. Instead we want to catch
4168 : * the case where we didn't actually read the block properly, which
4169 : * would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
4170 : */
4171 631565728 : if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
4172 : return;
4173 :
4174 315782864 : if (fs_info->nodesize < PAGE_SIZE) {
4175 0 : if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, page,
4176 : eb->start, eb->len)))
4177 0 : btrfs_subpage_dump_bitmap(fs_info, page, eb->start, eb->len);
4178 : } else {
4179 315782864 : WARN_ON(!PageUptodate(page));
4180 : }
4181 : }
4182 :
4183 9006455 : void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb,
4184 : const void *srcv)
4185 : {
4186 9006455 : char *kaddr;
4187 :
4188 9006455 : assert_eb_page_uptodate(eb, eb->pages[0]);
4189 9006324 : kaddr = page_address(eb->pages[0]) +
4190 : get_eb_offset_in_page(eb, offsetof(struct btrfs_header,
4191 : chunk_tree_uuid));
4192 18012648 : memcpy(kaddr, srcv, BTRFS_FSID_SIZE);
4193 9006324 : }
4194 :
4195 17638500 : void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv)
4196 : {
4197 17638500 : char *kaddr;
4198 :
4199 17638500 : assert_eb_page_uptodate(eb, eb->pages[0]);
4200 17638214 : kaddr = page_address(eb->pages[0]) +
4201 : get_eb_offset_in_page(eb, offsetof(struct btrfs_header, fsid));
4202 35276428 : memcpy(kaddr, srcv, BTRFS_FSID_SIZE);
4203 17638214 : }
4204 :
4205 149666746 : void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
4206 : unsigned long start, unsigned long len)
4207 : {
4208 149666746 : size_t cur;
4209 149666746 : size_t offset;
4210 149666746 : struct page *page;
4211 149666746 : char *kaddr;
4212 149666746 : char *src = (char *)srcv;
4213 149666746 : unsigned long i = get_eb_page_index(start);
4214 :
4215 149666746 : WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags));
4216 :
4217 149666746 : if (check_eb_range(eb, start, len))
4218 : return;
4219 :
4220 149654914 : offset = get_eb_offset_in_page(eb, start);
4221 :
4222 300112724 : while (len > 0) {
4223 150472177 : page = eb->pages[i];
4224 150472177 : assert_eb_page_uptodate(eb, page);
4225 :
4226 150457810 : cur = min(len, PAGE_SIZE - offset);
4227 150457810 : kaddr = page_address(page);
4228 300915620 : memcpy(kaddr + offset, src, cur);
4229 :
4230 150457810 : src += cur;
4231 150457810 : len -= cur;
4232 150457810 : offset = 0;
4233 150457810 : i++;
4234 : }
4235 : }
4236 :
4237 21338702 : void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
4238 : unsigned long len)
4239 : {
4240 21338702 : size_t cur;
4241 21338702 : size_t offset;
4242 21338702 : struct page *page;
4243 21338702 : char *kaddr;
4244 21338702 : unsigned long i = get_eb_page_index(start);
4245 :
4246 21338702 : if (check_eb_range(eb, start, len))
4247 : return;
4248 :
4249 21338484 : offset = get_eb_offset_in_page(eb, start);
4250 :
4251 52583805 : while (len > 0) {
4252 31247363 : page = eb->pages[i];
4253 31247363 : assert_eb_page_uptodate(eb, page);
4254 :
4255 31245321 : cur = min(len, PAGE_SIZE - offset);
4256 31245321 : kaddr = page_address(page);
4257 31245321 : memset(kaddr + offset, 0, cur);
4258 :
4259 31245321 : len -= cur;
4260 31245321 : offset = 0;
4261 31245321 : i++;
4262 : }
4263 : }
4264 :
4265 8632188 : void copy_extent_buffer_full(const struct extent_buffer *dst,
4266 : const struct extent_buffer *src)
4267 : {
4268 8632188 : int i;
4269 8632188 : int num_pages;
4270 :
4271 8632188 : ASSERT(dst->len == src->len);
4272 :
4273 8632188 : if (dst->fs_info->nodesize >= PAGE_SIZE) {
4274 8632188 : num_pages = num_extent_pages(dst);
4275 42217106 : for (i = 0; i < num_pages; i++)
4276 33584959 : copy_page(page_address(dst->pages[i]),
4277 33584959 : page_address(src->pages[i]));
4278 : } else {
4279 0 : size_t src_offset = get_eb_offset_in_page(src, 0);
4280 0 : size_t dst_offset = get_eb_offset_in_page(dst, 0);
4281 :
4282 0 : ASSERT(src->fs_info->nodesize < PAGE_SIZE);
4283 0 : memcpy(page_address(dst->pages[0]) + dst_offset,
4284 : page_address(src->pages[0]) + src_offset,
4285 : src->len);
4286 : }
4287 8632147 : }
4288 :
4289 9706861 : void copy_extent_buffer(const struct extent_buffer *dst,
4290 : const struct extent_buffer *src,
4291 : unsigned long dst_offset, unsigned long src_offset,
4292 : unsigned long len)
4293 : {
4294 9706861 : u64 dst_len = dst->len;
4295 9706861 : size_t cur;
4296 9706861 : size_t offset;
4297 9706861 : struct page *page;
4298 9706861 : char *kaddr;
4299 9706861 : unsigned long i = get_eb_page_index(dst_offset);
4300 :
4301 19413671 : if (check_eb_range(dst, dst_offset, len) ||
4302 9706837 : check_eb_range(src, src_offset, len))
4303 0 : return;
4304 :
4305 9706810 : WARN_ON(src->len != dst_len);
4306 :
4307 9706810 : offset = get_eb_offset_in_page(dst, dst_offset);
4308 :
4309 20733104 : while (len > 0) {
4310 11026224 : page = dst->pages[i];
4311 11026224 : assert_eb_page_uptodate(dst, page);
4312 :
4313 11026204 : cur = min(len, (unsigned long)(PAGE_SIZE - offset));
4314 :
4315 11026204 : kaddr = page_address(page);
4316 11026204 : read_extent_buffer(src, kaddr + offset, src_offset, cur);
4317 :
4318 11026294 : src_offset += cur;
4319 11026294 : len -= cur;
4320 11026294 : offset = 0;
4321 11026294 : i++;
4322 : }
4323 : }
4324 :
4325 : /*
4326 : * eb_bitmap_offset() - calculate the page and offset of the byte containing the
4327 : * given bit number
4328 : * @eb: the extent buffer
4329 : * @start: offset of the bitmap item in the extent buffer
4330 : * @nr: bit number
4331 : * @page_index: return index of the page in the extent buffer that contains the
4332 : * given bit number
4333 : * @page_offset: return offset into the page given by page_index
4334 : *
4335 : * This helper hides the ugliness of finding the byte in an extent buffer which
4336 : * contains a given bit.
4337 : */
4338 : static inline void eb_bitmap_offset(const struct extent_buffer *eb,
4339 : unsigned long start, unsigned long nr,
4340 : unsigned long *page_index,
4341 : size_t *page_offset)
4342 : {
4343 96447401 : size_t byte_offset = BIT_BYTE(nr);
4344 96447401 : size_t offset;
4345 :
4346 : /*
4347 : * The byte we want is the offset of the extent buffer + the offset of
4348 : * the bitmap item in the extent buffer + the offset of the byte in the
4349 : * bitmap item.
4350 : */
4351 96447401 : offset = start + offset_in_page(eb->start) + byte_offset;
4352 :
4353 96447401 : *page_index = offset >> PAGE_SHIFT;
4354 96447401 : *page_offset = offset_in_page(offset);
4355 : }
4356 :
4357 : /*
4358 : * Determine whether a bit in a bitmap item is set.
4359 : *
4360 : * @eb: the extent buffer
4361 : * @start: offset of the bitmap item in the extent buffer
4362 : * @nr: bit number to test
4363 : */
4364 86121656 : int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
4365 : unsigned long nr)
4366 : {
4367 86121656 : u8 *kaddr;
4368 86121656 : struct page *page;
4369 86121656 : unsigned long i;
4370 86121656 : size_t offset;
4371 :
4372 86121656 : eb_bitmap_offset(eb, start, nr, &i, &offset);
4373 86121656 : page = eb->pages[i];
4374 86121656 : assert_eb_page_uptodate(eb, page);
4375 86121659 : kaddr = page_address(page);
4376 86121659 : return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
4377 : }
4378 :
4379 : /*
4380 : * Set an area of a bitmap to 1.
4381 : *
4382 : * @eb: the extent buffer
4383 : * @start: offset of the bitmap item in the extent buffer
4384 : * @pos: bit number of the first bit
4385 : * @len: number of bits to set
4386 : */
4387 5062098 : void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
4388 : unsigned long pos, unsigned long len)
4389 : {
4390 5062098 : u8 *kaddr;
4391 5062098 : struct page *page;
4392 5062098 : unsigned long i;
4393 5062098 : size_t offset;
4394 5062098 : const unsigned int size = pos + len;
4395 5062098 : int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
4396 5062098 : u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(pos);
4397 :
4398 5062098 : eb_bitmap_offset(eb, start, pos, &i, &offset);
4399 5062098 : page = eb->pages[i];
4400 5062098 : assert_eb_page_uptodate(eb, page);
4401 5062099 : kaddr = page_address(page);
4402 :
4403 7493795 : while (len >= bits_to_set) {
4404 2431696 : kaddr[offset] |= mask_to_set;
4405 2431696 : len -= bits_to_set;
4406 2431696 : bits_to_set = BITS_PER_BYTE;
4407 2431696 : mask_to_set = ~0;
4408 2431696 : if (++offset >= PAGE_SIZE && len > 0) {
4409 1 : offset = 0;
4410 1 : page = eb->pages[++i];
4411 1 : assert_eb_page_uptodate(eb, page);
4412 1 : kaddr = page_address(page);
4413 : }
4414 : }
4415 5062099 : if (len) {
4416 2774676 : mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
4417 2774676 : kaddr[offset] |= mask_to_set;
4418 : }
4419 5062099 : }
4420 :
4421 :
4422 : /*
4423 : * Clear an area of a bitmap.
4424 : *
4425 : * @eb: the extent buffer
4426 : * @start: offset of the bitmap item in the extent buffer
4427 : * @pos: bit number of the first bit
4428 : * @len: number of bits to clear
4429 : */
4430 5263647 : void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
4431 : unsigned long start, unsigned long pos,
4432 : unsigned long len)
4433 : {
4434 5263647 : u8 *kaddr;
4435 5263647 : struct page *page;
4436 5263647 : unsigned long i;
4437 5263647 : size_t offset;
4438 5263647 : const unsigned int size = pos + len;
4439 5263647 : int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
4440 5263647 : u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos);
4441 :
4442 5263647 : eb_bitmap_offset(eb, start, pos, &i, &offset);
4443 5263647 : page = eb->pages[i];
4444 5263647 : assert_eb_page_uptodate(eb, page);
4445 5263647 : kaddr = page_address(page);
4446 :
4447 7775006 : while (len >= bits_to_clear) {
4448 2511359 : kaddr[offset] &= ~mask_to_clear;
4449 2511359 : len -= bits_to_clear;
4450 2511359 : bits_to_clear = BITS_PER_BYTE;
4451 2511359 : mask_to_clear = ~0;
4452 2511359 : if (++offset >= PAGE_SIZE && len > 0) {
4453 0 : offset = 0;
4454 0 : page = eb->pages[++i];
4455 0 : assert_eb_page_uptodate(eb, page);
4456 0 : kaddr = page_address(page);
4457 : }
4458 : }
4459 5263647 : if (len) {
4460 2943529 : mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
4461 2943529 : kaddr[offset] &= ~mask_to_clear;
4462 : }
4463 5263647 : }
4464 :
4465 : static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4466 : {
4467 297038660 : unsigned long distance = (src > dst) ? src - dst : dst - src;
4468 297038660 : return distance < len;
4469 : }
4470 :
4471 414762908 : static void copy_pages(struct page *dst_page, struct page *src_page,
4472 : unsigned long dst_off, unsigned long src_off,
4473 : unsigned long len)
4474 : {
4475 414762908 : char *dst_kaddr = page_address(dst_page);
4476 414762908 : char *src_kaddr;
4477 414762908 : int must_memmove = 0;
4478 :
4479 414762908 : if (dst_page != src_page) {
4480 117724248 : src_kaddr = page_address(src_page);
4481 : } else {
4482 297038660 : src_kaddr = dst_kaddr;
4483 594077320 : if (areas_overlap(src_off, dst_off, len))
4484 275448509 : must_memmove = 1;
4485 : }
4486 :
4487 117724248 : if (must_memmove)
4488 550897018 : memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
4489 : else
4490 278628798 : memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
4491 414762908 : }
4492 :
4493 87589357 : void memcpy_extent_buffer(const struct extent_buffer *dst,
4494 : unsigned long dst_offset, unsigned long src_offset,
4495 : unsigned long len)
4496 : {
4497 87589357 : size_t cur;
4498 87589357 : size_t dst_off_in_page;
4499 87589357 : size_t src_off_in_page;
4500 87589357 : unsigned long dst_i;
4501 87589357 : unsigned long src_i;
4502 :
4503 175176556 : if (check_eb_range(dst, dst_offset, len) ||
4504 87588088 : check_eb_range(dst, src_offset, len))
4505 0 : return;
4506 :
4507 296560484 : while (len > 0) {
4508 208967762 : dst_off_in_page = get_eb_offset_in_page(dst, dst_offset);
4509 208967762 : src_off_in_page = get_eb_offset_in_page(dst, src_offset);
4510 :
4511 208967762 : dst_i = get_eb_page_index(dst_offset);
4512 208967762 : src_i = get_eb_page_index(src_offset);
4513 :
4514 208967762 : cur = min(len, (unsigned long)(PAGE_SIZE -
4515 : src_off_in_page));
4516 208967762 : cur = min_t(unsigned long, cur,
4517 : (unsigned long)(PAGE_SIZE - dst_off_in_page));
4518 :
4519 208967762 : copy_pages(dst->pages[dst_i], dst->pages[src_i],
4520 : dst_off_in_page, src_off_in_page, cur);
4521 :
4522 208973285 : src_offset += cur;
4523 208973285 : dst_offset += cur;
4524 208973285 : len -= cur;
4525 : }
4526 : }
4527 :
4528 186084597 : void memmove_extent_buffer(const struct extent_buffer *dst,
4529 : unsigned long dst_offset, unsigned long src_offset,
4530 : unsigned long len)
4531 : {
4532 186084597 : size_t cur;
4533 186084597 : size_t dst_off_in_page;
4534 186084597 : size_t src_off_in_page;
4535 186084597 : unsigned long dst_end = dst_offset + len - 1;
4536 186084597 : unsigned long src_end = src_offset + len - 1;
4537 186084597 : unsigned long dst_i;
4538 186084597 : unsigned long src_i;
4539 :
4540 372160782 : if (check_eb_range(dst, dst_offset, len) ||
4541 186078855 : check_eb_range(dst, src_offset, len))
4542 0 : return;
4543 186076185 : if (dst_offset < src_offset) {
4544 84976587 : memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4545 84976587 : return;
4546 : }
4547 306884292 : while (len > 0) {
4548 205768451 : dst_i = get_eb_page_index(dst_end);
4549 205768451 : src_i = get_eb_page_index(src_end);
4550 :
4551 205768451 : dst_off_in_page = get_eb_offset_in_page(dst, dst_end);
4552 205768451 : src_off_in_page = get_eb_offset_in_page(dst, src_end);
4553 :
4554 205768451 : cur = min_t(unsigned long, len, src_off_in_page + 1);
4555 205768451 : cur = min(cur, dst_off_in_page + 1);
4556 205768451 : copy_pages(dst->pages[dst_i], dst->pages[src_i],
4557 205768451 : dst_off_in_page - cur + 1,
4558 205768451 : src_off_in_page - cur + 1, cur);
4559 :
4560 205784694 : dst_end -= cur;
4561 205784694 : src_end -= cur;
4562 205784694 : len -= cur;
4563 : }
4564 : }
4565 :
4566 : #define GANG_LOOKUP_SIZE 16
4567 0 : static struct extent_buffer *get_next_extent_buffer(
4568 : struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
4569 : {
4570 0 : struct extent_buffer *gang[GANG_LOOKUP_SIZE];
4571 0 : struct extent_buffer *found = NULL;
4572 0 : u64 page_start = page_offset(page);
4573 0 : u64 cur = page_start;
4574 :
4575 0 : ASSERT(in_range(bytenr, page_start, PAGE_SIZE));
4576 : lockdep_assert_held(&fs_info->buffer_lock);
4577 :
4578 0 : while (cur < page_start + PAGE_SIZE) {
4579 0 : int ret;
4580 0 : int i;
4581 :
4582 0 : ret = radix_tree_gang_lookup(&fs_info->buffer_radix,
4583 0 : (void **)gang, cur >> fs_info->sectorsize_bits,
4584 0 : min_t(unsigned int, GANG_LOOKUP_SIZE,
4585 : PAGE_SIZE / fs_info->nodesize));
4586 0 : if (ret == 0)
4587 0 : goto out;
4588 0 : for (i = 0; i < ret; i++) {
4589 : /* Already beyond page end */
4590 0 : if (gang[i]->start >= page_start + PAGE_SIZE)
4591 0 : goto out;
4592 : /* Found one */
4593 0 : if (gang[i]->start >= bytenr) {
4594 0 : found = gang[i];
4595 0 : goto out;
4596 : }
4597 : }
4598 0 : cur = gang[ret - 1]->start + gang[ret - 1]->len;
4599 : }
4600 0 : out:
4601 0 : return found;
4602 : }
4603 :
4604 0 : static int try_release_subpage_extent_buffer(struct page *page)
4605 : {
4606 0 : struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
4607 0 : u64 cur = page_offset(page);
4608 0 : const u64 end = page_offset(page) + PAGE_SIZE;
4609 0 : int ret;
4610 :
4611 0 : while (cur < end) {
4612 0 : struct extent_buffer *eb = NULL;
4613 :
4614 : /*
4615 : * Unlike try_release_extent_buffer() which uses page->private
4616 : * to grab buffer, for subpage case we rely on radix tree, thus
4617 : * we need to ensure radix tree consistency.
4618 : *
4619 : * We also want an atomic snapshot of the radix tree, thus go
4620 : * with spinlock rather than RCU.
4621 : */
4622 0 : spin_lock(&fs_info->buffer_lock);
4623 0 : eb = get_next_extent_buffer(fs_info, page, cur);
4624 0 : if (!eb) {
4625 : /* No more eb in the page range after or at cur */
4626 0 : spin_unlock(&fs_info->buffer_lock);
4627 : break;
4628 : }
4629 0 : cur = eb->start + eb->len;
4630 :
4631 : /*
4632 : * The same as try_release_extent_buffer(), to ensure the eb
4633 : * won't disappear out from under us.
4634 : */
4635 0 : spin_lock(&eb->refs_lock);
4636 0 : if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4637 0 : spin_unlock(&eb->refs_lock);
4638 0 : spin_unlock(&fs_info->buffer_lock);
4639 : break;
4640 : }
4641 0 : spin_unlock(&fs_info->buffer_lock);
4642 :
4643 : /*
4644 : * If tree ref isn't set then we know the ref on this eb is a
4645 : * real ref, so just return, this eb will likely be freed soon
4646 : * anyway.
4647 : */
4648 0 : if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4649 0 : spin_unlock(&eb->refs_lock);
4650 : break;
4651 : }
4652 :
4653 : /*
4654 : * Here we don't care about the return value, we will always
4655 : * check the page private at the end. And
4656 : * release_extent_buffer() will release the refs_lock.
4657 : */
4658 0 : release_extent_buffer(eb);
4659 : }
4660 : /*
4661 : * Finally to check if we have cleared page private, as if we have
4662 : * released all ebs in the page, the page private should be cleared now.
4663 : */
4664 0 : spin_lock(&page->mapping->private_lock);
4665 0 : if (!PagePrivate(page))
4666 : ret = 1;
4667 : else
4668 0 : ret = 0;
4669 0 : spin_unlock(&page->mapping->private_lock);
4670 0 : return ret;
4671 :
4672 : }
4673 :
4674 3237570 : int try_release_extent_buffer(struct page *page)
4675 : {
4676 3237570 : struct extent_buffer *eb;
4677 :
4678 3237570 : if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
4679 0 : return try_release_subpage_extent_buffer(page);
4680 :
4681 : /*
4682 : * We need to make sure nobody is changing page->private, as we rely on
4683 : * page->private as the pointer to extent buffer.
4684 : */
4685 3237570 : spin_lock(&page->mapping->private_lock);
4686 3237570 : if (!PagePrivate(page)) {
4687 0 : spin_unlock(&page->mapping->private_lock);
4688 0 : return 1;
4689 : }
4690 :
4691 3237570 : eb = (struct extent_buffer *)page->private;
4692 3237570 : BUG_ON(!eb);
4693 :
4694 : /*
4695 : * This is a little awful but should be ok, we need to make sure that
4696 : * the eb doesn't disappear out from under us while we're looking at
4697 : * this page.
4698 : */
4699 3237570 : spin_lock(&eb->refs_lock);
4700 3237570 : if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4701 245995 : spin_unlock(&eb->refs_lock);
4702 245995 : spin_unlock(&page->mapping->private_lock);
4703 245995 : return 0;
4704 : }
4705 2991575 : spin_unlock(&page->mapping->private_lock);
4706 :
4707 : /*
4708 : * If tree ref isn't set then we know the ref on this eb is a real ref,
4709 : * so just return, this page will likely be freed soon anyway.
4710 : */
4711 2991575 : if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4712 0 : spin_unlock(&eb->refs_lock);
4713 0 : return 0;
4714 : }
4715 :
4716 2991575 : return release_extent_buffer(eb);
4717 : }
4718 :
4719 : /*
4720 : * btrfs_readahead_tree_block - attempt to readahead a child block
4721 : * @fs_info: the fs_info
4722 : * @bytenr: bytenr to read
4723 : * @owner_root: objectid of the root that owns this eb
4724 : * @gen: generation for the uptodate check, can be 0
4725 : * @level: level for the eb
4726 : *
4727 : * Attempt to readahead a tree block at @bytenr. If @gen is 0 then we do a
4728 : * normal uptodate check of the eb, without checking the generation. If we have
4729 : * to read the block we will not block on anything.
4730 : */
4731 11373462 : void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
4732 : u64 bytenr, u64 owner_root, u64 gen, int level)
4733 : {
4734 11373462 : struct btrfs_tree_parent_check check = {
4735 : .has_first_key = 0,
4736 : .level = level,
4737 : .transid = gen
4738 : };
4739 11373462 : struct extent_buffer *eb;
4740 11373462 : int ret;
4741 :
4742 11373462 : eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
4743 11373435 : if (IS_ERR(eb))
4744 11364413 : return;
4745 :
4746 11373435 : if (btrfs_buffer_uptodate(eb, gen, 1)) {
4747 11364033 : free_extent_buffer(eb);
4748 11364033 : return;
4749 : }
4750 :
4751 9272 : ret = read_extent_buffer_pages(eb, WAIT_NONE, 0, &check);
4752 9272 : if (ret < 0)
4753 0 : free_extent_buffer_stale(eb);
4754 : else
4755 9272 : free_extent_buffer(eb);
4756 : }
4757 :
4758 : /*
4759 : * btrfs_readahead_node_child - readahead a node's child block
4760 : * @node: parent node we're reading from
4761 : * @slot: slot in the parent node for the child we want to read
4762 : *
4763 : * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
4764 : * the slot in the node provided.
4765 : */
4766 9419588 : void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
4767 : {
4768 18839077 : btrfs_readahead_tree_block(node->fs_info,
4769 : btrfs_node_blockptr(node, slot),
4770 : btrfs_header_owner(node),
4771 : btrfs_node_ptr_generation(node, slot),
4772 9419588 : btrfs_header_level(node) - 1);
4773 9419626 : }
|
