Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * fs/mpage.c
4 : *
5 : * Copyright (C) 2002, Linus Torvalds.
6 : *
7 : * Contains functions related to preparing and submitting BIOs which contain
8 : * multiple pagecache pages.
9 : *
10 : * 15May2002 Andrew Morton
11 : * Initial version
12 : * 27Jun2002 axboe@suse.de
13 : * use bio_add_page() to build bio's just the right size
14 : */
15 :
16 : #include <linux/kernel.h>
17 : #include <linux/export.h>
18 : #include <linux/mm.h>
19 : #include <linux/kdev_t.h>
20 : #include <linux/gfp.h>
21 : #include <linux/bio.h>
22 : #include <linux/fs.h>
23 : #include <linux/buffer_head.h>
24 : #include <linux/blkdev.h>
25 : #include <linux/highmem.h>
26 : #include <linux/prefetch.h>
27 : #include <linux/mpage.h>
28 : #include <linux/mm_inline.h>
29 : #include <linux/writeback.h>
30 : #include <linux/backing-dev.h>
31 : #include <linux/pagevec.h>
32 : #include "internal.h"
33 :
34 : /*
35 : * I/O completion handler for multipage BIOs.
36 : *
37 : * The mpage code never puts partial pages into a BIO (except for end-of-file).
38 : * If a page does not map to a contiguous run of blocks then it simply falls
39 : * back to block_read_full_folio().
40 : *
41 : * Why is this? If a page's completion depends on a number of different BIOs
42 : * which can complete in any order (or at the same time) then determining the
43 : * status of that page is hard. See end_buffer_async_read() for the details.
44 : * There is no point in duplicating all that complexity.
45 : */
46 796833 : static void mpage_read_end_io(struct bio *bio)
47 : {
48 796833 : struct folio_iter fi;
49 796833 : int err = blk_status_to_errno(bio->bi_status);
50 :
51 2643695 : bio_for_each_folio_all(fi, bio) {
52 1846862 : if (err)
53 6051 : folio_set_error(fi.folio);
54 : else
55 1840811 : folio_mark_uptodate(fi.folio);
56 1846862 : folio_unlock(fi.folio);
57 : }
58 :
59 796833 : bio_put(bio);
60 796833 : }
61 :
62 950 : static void mpage_write_end_io(struct bio *bio)
63 : {
64 950 : struct folio_iter fi;
65 950 : int err = blk_status_to_errno(bio->bi_status);
66 :
67 2571 : bio_for_each_folio_all(fi, bio) {
68 1621 : if (err) {
69 0 : folio_set_error(fi.folio);
70 0 : mapping_set_error(fi.folio->mapping, err);
71 : }
72 1621 : folio_end_writeback(fi.folio);
73 : }
74 :
75 950 : bio_put(bio);
76 950 : }
77 :
78 795317 : static struct bio *mpage_bio_submit_read(struct bio *bio)
79 : {
80 796818 : bio->bi_end_io = mpage_read_end_io;
81 795317 : guard_bio_eod(bio);
82 796802 : submit_bio(bio);
83 796772 : return NULL;
84 : }
85 :
86 24 : static struct bio *mpage_bio_submit_write(struct bio *bio)
87 : {
88 950 : bio->bi_end_io = mpage_write_end_io;
89 24 : guard_bio_eod(bio);
90 899 : submit_bio(bio);
91 51 : return NULL;
92 : }
93 :
94 : /*
95 : * support function for mpage_readahead. The fs supplied get_block might
96 : * return an up to date buffer. This is used to map that buffer into
97 : * the page, which allows read_folio to avoid triggering a duplicate call
98 : * to get_block.
99 : *
100 : * The idea is to avoid adding buffers to pages that don't already have
101 : * them. So when the buffer is up to date and the page size == block size,
102 : * this marks the page up to date instead of adding new buffers.
103 : */
104 0 : static void map_buffer_to_folio(struct folio *folio, struct buffer_head *bh,
105 : int page_block)
106 : {
107 0 : struct inode *inode = folio->mapping->host;
108 0 : struct buffer_head *page_bh, *head;
109 0 : int block = 0;
110 :
111 0 : head = folio_buffers(folio);
112 0 : if (!head) {
113 : /*
114 : * don't make any buffers if there is only one buffer on
115 : * the folio and the folio just needs to be set up to date
116 : */
117 0 : if (inode->i_blkbits == PAGE_SHIFT &&
118 : buffer_uptodate(bh)) {
119 0 : folio_mark_uptodate(folio);
120 0 : return;
121 : }
122 0 : create_empty_buffers(&folio->page, i_blocksize(inode), 0);
123 0 : head = folio_buffers(folio);
124 : }
125 :
126 0 : page_bh = head;
127 0 : do {
128 0 : if (block == page_block) {
129 0 : page_bh->b_state = bh->b_state;
130 0 : page_bh->b_bdev = bh->b_bdev;
131 0 : page_bh->b_blocknr = bh->b_blocknr;
132 0 : break;
133 : }
134 0 : page_bh = page_bh->b_this_page;
135 0 : block++;
136 0 : } while (page_bh != head);
137 : }
138 :
139 : struct mpage_readpage_args {
140 : struct bio *bio;
141 : struct folio *folio;
142 : unsigned int nr_pages;
143 : bool is_readahead;
144 : sector_t last_block_in_bio;
145 : struct buffer_head map_bh;
146 : unsigned long first_logical_block;
147 : get_block_t *get_block;
148 : };
149 :
150 : /*
151 : * This is the worker routine which does all the work of mapping the disk
152 : * blocks and constructs largest possible bios, submits them for IO if the
153 : * blocks are not contiguous on the disk.
154 : *
155 : * We pass a buffer_head back and forth and use its buffer_mapped() flag to
156 : * represent the validity of its disk mapping and to decide when to do the next
157 : * get_block() call.
158 : */
159 1847117 : static struct bio *do_mpage_readpage(struct mpage_readpage_args *args)
160 : {
161 1847117 : struct folio *folio = args->folio;
162 1847117 : struct inode *inode = folio->mapping->host;
163 1847117 : const unsigned blkbits = inode->i_blkbits;
164 1847117 : const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
165 1847117 : const unsigned blocksize = 1 << blkbits;
166 1847117 : struct buffer_head *map_bh = &args->map_bh;
167 1847117 : sector_t block_in_file;
168 1847117 : sector_t last_block;
169 1847117 : sector_t last_block_in_file;
170 1847117 : sector_t blocks[MAX_BUF_PER_PAGE];
171 1847117 : unsigned page_block;
172 1847117 : unsigned first_hole = blocks_per_page;
173 1847117 : struct block_device *bdev = NULL;
174 1847117 : int length;
175 1847117 : int fully_mapped = 1;
176 1847117 : blk_opf_t opf = REQ_OP_READ;
177 1847117 : unsigned nblocks;
178 1847117 : unsigned relative_block;
179 1847117 : gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
180 :
181 : /* MAX_BUF_PER_PAGE, for example */
182 1847117 : VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
183 :
184 1847117 : if (args->is_readahead) {
185 1847027 : opf |= REQ_RAHEAD;
186 1847027 : gfp |= __GFP_NORETRY | __GFP_NOWARN;
187 : }
188 :
189 1847117 : if (folio_buffers(folio))
190 81 : goto confused;
191 :
192 1847036 : block_in_file = (sector_t)folio->index << (PAGE_SHIFT - blkbits);
193 1847036 : last_block = block_in_file + args->nr_pages * blocks_per_page;
194 1847036 : last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
195 1847036 : if (last_block > last_block_in_file)
196 : last_block = last_block_in_file;
197 1847036 : page_block = 0;
198 :
199 : /*
200 : * Map blocks using the result from the previous get_blocks call first.
201 : */
202 1847036 : nblocks = map_bh->b_size >> blkbits;
203 3694072 : if (buffer_mapped(map_bh) &&
204 1050031 : block_in_file > args->first_logical_block &&
205 1050031 : block_in_file < (args->first_logical_block + nblocks)) {
206 1050027 : unsigned map_offset = block_in_file - args->first_logical_block;
207 1050027 : unsigned last = nblocks - map_offset;
208 :
209 1050027 : for (relative_block = 0; ; relative_block++) {
210 7890265 : if (relative_block == last) {
211 119489 : clear_buffer_mapped(map_bh);
212 : break;
213 : }
214 7770776 : if (page_block == blocks_per_page)
215 : break;
216 6840238 : blocks[page_block] = map_bh->b_blocknr + map_offset +
217 : relative_block;
218 6840238 : page_block++;
219 6840238 : block_in_file++;
220 : }
221 1050027 : bdev = map_bh->b_bdev;
222 : }
223 :
224 : /*
225 : * Then do more get_blocks calls until we are done with this folio.
226 : */
227 1847036 : map_bh->b_folio = folio;
228 2668874 : while (page_block < blocks_per_page) {
229 821857 : map_bh->b_state = 0;
230 821857 : map_bh->b_size = 0;
231 :
232 821857 : if (block_in_file < last_block) {
233 808719 : map_bh->b_size = (last_block-block_in_file) << blkbits;
234 808719 : if (args->get_block(inode, block_in_file, map_bh, 0))
235 0 : goto confused;
236 808696 : args->first_logical_block = block_in_file;
237 : }
238 :
239 1643668 : if (!buffer_mapped(map_bh)) {
240 25018 : fully_mapped = 0;
241 25018 : if (first_hole == blocks_per_page)
242 1691 : first_hole = page_block;
243 25018 : page_block++;
244 25018 : block_in_file++;
245 25018 : continue;
246 : }
247 :
248 : /* some filesystems will copy data into the page during
249 : * the get_block call, in which case we don't want to
250 : * read it again. map_buffer_to_folio copies the data
251 : * we just collected from get_block into the folio's buffers
252 : * so read_folio doesn't have to repeat the get_block call
253 : */
254 1593629 : if (buffer_uptodate(map_bh)) {
255 0 : map_buffer_to_folio(folio, map_bh, page_block);
256 0 : goto confused;
257 : }
258 :
259 796813 : if (first_hole != blocks_per_page)
260 12 : goto confused; /* hole -> non-hole */
261 :
262 : /* Contiguous blocks? */
263 796801 : if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
264 5 : goto confused;
265 796796 : nblocks = map_bh->b_size >> blkbits;
266 796796 : for (relative_block = 0; ; relative_block++) {
267 5045566 : if (relative_block == nblocks) {
268 677315 : clear_buffer_mapped(map_bh);
269 : break;
270 4368251 : } else if (page_block == blocks_per_page)
271 : break;
272 4248770 : blocks[page_block] = map_bh->b_blocknr+relative_block;
273 4248770 : page_block++;
274 4248770 : block_in_file++;
275 : }
276 796820 : bdev = map_bh->b_bdev;
277 : }
278 :
279 1847017 : if (first_hole != blocks_per_page) {
280 1679 : folio_zero_segment(folio, first_hole << blkbits, PAGE_SIZE);
281 1679 : if (first_hole == 0) {
282 180 : folio_mark_uptodate(folio);
283 180 : folio_unlock(folio);
284 180 : goto out;
285 : }
286 1845338 : } else if (fully_mapped) {
287 1845341 : folio_set_mappedtodisk(folio);
288 : }
289 :
290 : /*
291 : * This folio will go to BIO. Do we need to send this BIO off first?
292 : */
293 1846813 : if (args->bio && (args->last_block_in_bio != blocks[0] - 1))
294 0 : args->bio = mpage_bio_submit_read(args->bio);
295 :
296 1846813 : alloc_new:
297 1846813 : if (args->bio == NULL) {
298 796786 : args->bio = bio_alloc(bdev, bio_max_segs(args->nr_pages), opf,
299 : gfp);
300 796816 : if (args->bio == NULL)
301 0 : goto confused;
302 796816 : args->bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9);
303 : }
304 :
305 1846843 : length = first_hole << blkbits;
306 1846843 : if (!bio_add_folio(args->bio, folio, length, 0)) {
307 0 : args->bio = mpage_bio_submit_read(args->bio);
308 0 : goto alloc_new;
309 : }
310 :
311 1846843 : relative_block = block_in_file - args->first_logical_block;
312 1846843 : nblocks = map_bh->b_size >> blkbits;
313 3693686 : if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
314 : (first_hole != blocks_per_page))
315 3002 : args->bio = mpage_bio_submit_read(args->bio);
316 : else
317 1845342 : args->last_block_in_bio = blocks[blocks_per_page - 1];
318 1847119 : out:
319 1847119 : return args->bio;
320 :
321 98 : confused:
322 98 : if (args->bio)
323 0 : args->bio = mpage_bio_submit_read(args->bio);
324 98 : if (!folio_test_uptodate(folio))
325 98 : block_read_full_folio(folio, args->get_block);
326 : else
327 0 : folio_unlock(folio);
328 98 : goto out;
329 : }
330 :
331 : /**
332 : * mpage_readahead - start reads against pages
333 : * @rac: Describes which pages to read.
334 : * @get_block: The filesystem's block mapper function.
335 : *
336 : * This function walks the pages and the blocks within each page, building and
337 : * emitting large BIOs.
338 : *
339 : * If anything unusual happens, such as:
340 : *
341 : * - encountering a page which has buffers
342 : * - encountering a page which has a non-hole after a hole
343 : * - encountering a page with non-contiguous blocks
344 : *
345 : * then this code just gives up and calls the buffer_head-based read function.
346 : * It does handle a page which has holes at the end - that is a common case:
347 : * the end-of-file on blocksize < PAGE_SIZE setups.
348 : *
349 : * BH_Boundary explanation:
350 : *
351 : * There is a problem. The mpage read code assembles several pages, gets all
352 : * their disk mappings, and then submits them all. That's fine, but obtaining
353 : * the disk mappings may require I/O. Reads of indirect blocks, for example.
354 : *
355 : * So an mpage read of the first 16 blocks of an ext2 file will cause I/O to be
356 : * submitted in the following order:
357 : *
358 : * 12 0 1 2 3 4 5 6 7 8 9 10 11 13 14 15 16
359 : *
360 : * because the indirect block has to be read to get the mappings of blocks
361 : * 13,14,15,16. Obviously, this impacts performance.
362 : *
363 : * So what we do it to allow the filesystem's get_block() function to set
364 : * BH_Boundary when it maps block 11. BH_Boundary says: mapping of the block
365 : * after this one will require I/O against a block which is probably close to
366 : * this one. So you should push what I/O you have currently accumulated.
367 : *
368 : * This all causes the disk requests to be issued in the correct order.
369 : */
370 796926 : void mpage_readahead(struct readahead_control *rac, get_block_t get_block)
371 : {
372 796926 : struct folio *folio;
373 796926 : struct mpage_readpage_args args = {
374 : .get_block = get_block,
375 : .is_readahead = true,
376 : };
377 :
378 2643958 : while ((folio = readahead_folio(rac))) {
379 1847032 : prefetchw(&folio->flags);
380 1847033 : args.folio = folio;
381 1847033 : args.nr_pages = readahead_count(rac);
382 1847033 : args.bio = do_mpage_readpage(&args);
383 : }
384 796928 : if (args.bio)
385 795321 : mpage_bio_submit_read(args.bio);
386 796877 : }
387 : EXPORT_SYMBOL(mpage_readahead);
388 :
389 : /*
390 : * This isn't called much at all
391 : */
392 85 : int mpage_read_folio(struct folio *folio, get_block_t get_block)
393 : {
394 85 : struct mpage_readpage_args args = {
395 : .folio = folio,
396 : .nr_pages = 1,
397 : .get_block = get_block,
398 : };
399 :
400 85 : args.bio = do_mpage_readpage(&args);
401 85 : if (args.bio)
402 0 : mpage_bio_submit_read(args.bio);
403 85 : return 0;
404 : }
405 : EXPORT_SYMBOL(mpage_read_folio);
406 :
407 : /*
408 : * Writing is not so simple.
409 : *
410 : * If the page has buffers then they will be used for obtaining the disk
411 : * mapping. We only support pages which are fully mapped-and-dirty, with a
412 : * special case for pages which are unmapped at the end: end-of-file.
413 : *
414 : * If the page has no buffers (preferred) then the page is mapped here.
415 : *
416 : * If all blocks are found to be contiguous then the page can go into the
417 : * BIO. Otherwise fall back to the mapping's writepage().
418 : *
419 : * FIXME: This code wants an estimate of how many pages are still to be
420 : * written, so it can intelligently allocate a suitably-sized BIO. For now,
421 : * just allocate full-size (16-page) BIOs.
422 : */
423 :
424 : struct mpage_data {
425 : struct bio *bio;
426 : sector_t last_block_in_bio;
427 : get_block_t *get_block;
428 : };
429 :
430 : /*
431 : * We have our BIO, so we can now mark the buffers clean. Make
432 : * sure to only clean buffers which we know we'll be writing.
433 : */
434 1621 : static void clean_buffers(struct page *page, unsigned first_unmapped)
435 : {
436 1621 : unsigned buffer_counter = 0;
437 1621 : struct buffer_head *bh, *head;
438 1621 : if (!page_has_buffers(page))
439 : return;
440 1611 : head = page_buffers(page);
441 1611 : bh = head;
442 :
443 20148 : do {
444 20148 : if (buffer_counter++ == first_unmapped)
445 : break;
446 19273 : clear_buffer_dirty(bh);
447 19273 : bh = bh->b_this_page;
448 19273 : } while (bh != head);
449 :
450 : /*
451 : * we cannot drop the bh if the page is not uptodate or a concurrent
452 : * read_folio would fail to serialize with the bh and it would read from
453 : * disk before we reach the platter.
454 : */
455 1611 : if (buffer_heads_over_limit && PageUptodate(page))
456 0 : try_to_free_buffers(page_folio(page));
457 : }
458 :
459 : /*
460 : * For situations where we want to clean all buffers attached to a page.
461 : * We don't need to calculate how many buffers are attached to the page,
462 : * we just need to specify a number larger than the maximum number of buffers.
463 : */
464 0 : void clean_page_buffers(struct page *page)
465 : {
466 0 : clean_buffers(page, ~0U);
467 0 : }
468 :
469 2845 : static int __mpage_writepage(struct folio *folio, struct writeback_control *wbc,
470 : void *data)
471 : {
472 2845 : struct mpage_data *mpd = data;
473 2845 : struct bio *bio = mpd->bio;
474 2845 : struct address_space *mapping = folio->mapping;
475 2845 : struct inode *inode = mapping->host;
476 2845 : const unsigned blkbits = inode->i_blkbits;
477 2845 : const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
478 2845 : sector_t last_block;
479 2845 : sector_t block_in_file;
480 2845 : sector_t blocks[MAX_BUF_PER_PAGE];
481 2845 : unsigned page_block;
482 2845 : unsigned first_unmapped = blocks_per_page;
483 2845 : struct block_device *bdev = NULL;
484 2845 : int boundary = 0;
485 2845 : sector_t boundary_block = 0;
486 2845 : struct block_device *boundary_bdev = NULL;
487 2845 : size_t length;
488 2845 : struct buffer_head map_bh;
489 2845 : loff_t i_size = i_size_read(inode);
490 2845 : int ret = 0;
491 2845 : struct buffer_head *head = folio_buffers(folio);
492 :
493 2845 : if (head) {
494 : struct buffer_head *bh = head;
495 :
496 : /* If they're all mapped and dirty, do it */
497 : page_block = 0;
498 75484 : do {
499 150968 : BUG_ON(buffer_locked(bh));
500 150968 : if (!buffer_mapped(bh)) {
501 : /*
502 : * unmapped dirty buffers are created by
503 : * block_dirty_folio -> mmapped data
504 : */
505 105724 : if (buffer_dirty(bh))
506 4 : goto confused;
507 52858 : if (first_unmapped == blocks_per_page)
508 1933 : first_unmapped = page_block;
509 52858 : continue;
510 : }
511 :
512 22622 : if (first_unmapped != blocks_per_page)
513 1058 : goto confused; /* hole -> non-hole */
514 :
515 64674 : if (!buffer_dirty(bh) || !buffer_uptodate(bh))
516 18 : goto confused;
517 21546 : if (page_block) {
518 19782 : if (bh->b_blocknr != blocks[page_block-1] + 1)
519 142 : goto confused;
520 : }
521 21404 : blocks[page_block++] = bh->b_blocknr;
522 21404 : boundary = buffer_boundary(bh);
523 21404 : if (boundary) {
524 60 : boundary_block = bh->b_blocknr;
525 60 : boundary_bdev = bh->b_bdev;
526 : }
527 21404 : bdev = bh->b_bdev;
528 74262 : } while ((bh = bh->b_this_page) != head);
529 :
530 1611 : if (first_unmapped)
531 1611 : goto page_is_mapped;
532 :
533 : /*
534 : * Page has buffers, but they are all unmapped. The page was
535 : * created by pagein or read over a hole which was handled by
536 : * block_read_full_folio(). If this address_space is also
537 : * using mpage_readahead then this can rarely happen.
538 : */
539 0 : goto confused;
540 : }
541 :
542 : /*
543 : * The page has no buffers: map it to disk
544 : */
545 24 : BUG_ON(!folio_test_uptodate(folio));
546 12 : block_in_file = (sector_t)folio->index << (PAGE_SHIFT - blkbits);
547 : /*
548 : * Whole page beyond EOF? Skip allocating blocks to avoid leaking
549 : * space.
550 : */
551 12 : if (block_in_file >= (i_size + (1 << blkbits) - 1) >> blkbits)
552 0 : goto page_is_mapped;
553 12 : last_block = (i_size - 1) >> blkbits;
554 12 : map_bh.b_folio = folio;
555 667 : for (page_block = 0; page_block < blocks_per_page; ) {
556 :
557 657 : map_bh.b_state = 0;
558 657 : map_bh.b_size = 1 << blkbits;
559 657 : if (mpd->get_block(inode, block_in_file, &map_bh, 1))
560 0 : goto confused;
561 1314 : if (!buffer_mapped(&map_bh))
562 0 : goto confused;
563 1314 : if (buffer_new(&map_bh))
564 657 : clean_bdev_bh_alias(&map_bh);
565 1314 : if (buffer_boundary(&map_bh)) {
566 3 : boundary_block = map_bh.b_blocknr;
567 3 : boundary_bdev = map_bh.b_bdev;
568 : }
569 657 : if (page_block) {
570 645 : if (map_bh.b_blocknr != blocks[page_block-1] + 1)
571 2 : goto confused;
572 : }
573 655 : blocks[page_block++] = map_bh.b_blocknr;
574 655 : boundary = buffer_boundary(&map_bh);
575 655 : bdev = map_bh.b_bdev;
576 655 : if (block_in_file == last_block)
577 : break;
578 655 : block_in_file++;
579 : }
580 10 : BUG_ON(page_block == 0);
581 :
582 : first_unmapped = page_block;
583 :
584 1621 : page_is_mapped:
585 : /* Don't bother writing beyond EOF, truncate will discard the folio */
586 1621 : if (folio_pos(folio) >= i_size)
587 0 : goto confused;
588 1621 : length = folio_size(folio);
589 1621 : if (folio_pos(folio) + length > i_size) {
590 : /*
591 : * The page straddles i_size. It must be zeroed out on each
592 : * and every writepage invocation because it may be mmapped.
593 : * "A file is mapped in multiples of the page size. For a file
594 : * that is not a multiple of the page size, the remaining memory
595 : * is zeroed when mapped, and writes to that region are not
596 : * written out to the file."
597 : */
598 547 : length = i_size - folio_pos(folio);
599 547 : folio_zero_segment(folio, length, folio_size(folio));
600 : }
601 :
602 : /*
603 : * This page will go to BIO. Do we need to send this BIO off first?
604 : */
605 1621 : if (bio && mpd->last_block_in_bio != blocks[0] - 1)
606 5 : bio = mpage_bio_submit_write(bio);
607 :
608 1616 : alloc_new:
609 1621 : if (bio == NULL) {
610 1063 : bio = bio_alloc(bdev, BIO_MAX_VECS,
611 : REQ_OP_WRITE | wbc_to_write_flags(wbc),
612 : GFP_NOFS);
613 950 : bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9);
614 950 : wbc_init_bio(wbc, bio);
615 : }
616 :
617 : /*
618 : * Must try to add the page before marking the buffer clean or
619 : * the confused fail path above (OOM) will be very confused when
620 : * it finds all bh marked clean (i.e. it will not write anything)
621 : */
622 1621 : wbc_account_cgroup_owner(wbc, &folio->page, folio_size(folio));
623 1621 : length = first_unmapped << blkbits;
624 1621 : if (!bio_add_folio(bio, folio, length, 0)) {
625 0 : bio = mpage_bio_submit_write(bio);
626 0 : goto alloc_new;
627 : }
628 :
629 1621 : clean_buffers(&folio->page, first_unmapped);
630 :
631 1621 : BUG_ON(folio_test_writeback(folio));
632 1621 : folio_start_writeback(folio);
633 1621 : folio_unlock(folio);
634 1621 : if (boundary || (first_unmapped != blocks_per_page)) {
635 875 : bio = mpage_bio_submit_write(bio);
636 875 : if (boundary_block) {
637 8 : write_boundary_block(boundary_bdev,
638 8 : boundary_block, 1 << blkbits);
639 : }
640 : } else {
641 746 : mpd->last_block_in_bio = blocks[blocks_per_page - 1];
642 : }
643 1621 : goto out;
644 :
645 1224 : confused:
646 1224 : if (bio)
647 46 : bio = mpage_bio_submit_write(bio);
648 :
649 : /*
650 : * The caller has a ref on the inode, so *mapping is stable
651 : */
652 1224 : ret = block_write_full_page(&folio->page, mpd->get_block, wbc);
653 1224 : mapping_set_error(mapping, ret);
654 2845 : out:
655 2845 : mpd->bio = bio;
656 2845 : return ret;
657 : }
658 :
659 : /**
660 : * mpage_writepages - walk the list of dirty pages of the given address space & writepage() all of them
661 : * @mapping: address space structure to write
662 : * @wbc: subtract the number of written pages from *@wbc->nr_to_write
663 : * @get_block: the filesystem's block mapper function.
664 : *
665 : * This is a library function, which implements the writepages()
666 : * address_space_operation.
667 : */
668 : int
669 1816 : mpage_writepages(struct address_space *mapping,
670 : struct writeback_control *wbc, get_block_t get_block)
671 : {
672 1816 : struct mpage_data mpd = {
673 : .get_block = get_block,
674 : };
675 1816 : struct blk_plug plug;
676 1816 : int ret;
677 :
678 1816 : blk_start_plug(&plug);
679 1816 : ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
680 1816 : if (mpd.bio)
681 24 : mpage_bio_submit_write(mpd.bio);
682 1816 : blk_finish_plug(&plug);
683 1816 : return ret;
684 : }
685 : EXPORT_SYMBOL(mpage_writepages);
|
