Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * fs/mpage.c
4 : *
5 : * Copyright (C) 2002, Linus Torvalds.
6 : *
7 : * Contains functions related to preparing and submitting BIOs which contain
8 : * multiple pagecache pages.
9 : *
10 : * 15May2002 Andrew Morton
11 : * Initial version
12 : * 27Jun2002 axboe@suse.de
13 : * use bio_add_page() to build bio's just the right size
14 : */
15 :
16 : #include <linux/kernel.h>
17 : #include <linux/export.h>
18 : #include <linux/mm.h>
19 : #include <linux/kdev_t.h>
20 : #include <linux/gfp.h>
21 : #include <linux/bio.h>
22 : #include <linux/fs.h>
23 : #include <linux/buffer_head.h>
24 : #include <linux/blkdev.h>
25 : #include <linux/highmem.h>
26 : #include <linux/prefetch.h>
27 : #include <linux/mpage.h>
28 : #include <linux/mm_inline.h>
29 : #include <linux/writeback.h>
30 : #include <linux/backing-dev.h>
31 : #include <linux/pagevec.h>
32 : #include "internal.h"
33 :
34 : /*
35 : * I/O completion handler for multipage BIOs.
36 : *
37 : * The mpage code never puts partial pages into a BIO (except for end-of-file).
38 : * If a page does not map to a contiguous run of blocks then it simply falls
39 : * back to block_read_full_folio().
40 : *
41 : * Why is this? If a page's completion depends on a number of different BIOs
42 : * which can complete in any order (or at the same time) then determining the
43 : * status of that page is hard. See end_buffer_async_read() for the details.
44 : * There is no point in duplicating all that complexity.
45 : */
46 11139718 : static void mpage_read_end_io(struct bio *bio)
47 : {
48 11139718 : struct folio_iter fi;
49 11139718 : int err = blk_status_to_errno(bio->bi_status);
50 :
51 152982376 : bio_for_each_folio_all(fi, bio) {
52 141842655 : if (err)
53 38484 : folio_set_error(fi.folio);
54 : else
55 141804171 : folio_mark_uptodate(fi.folio);
56 141842657 : folio_unlock(fi.folio);
57 : }
58 :
59 11139721 : bio_put(bio);
60 11139721 : }
61 :
62 4940 : static void mpage_write_end_io(struct bio *bio)
63 : {
64 4940 : struct folio_iter fi;
65 4940 : int err = blk_status_to_errno(bio->bi_status);
66 :
67 63233 : bio_for_each_folio_all(fi, bio) {
68 58293 : if (err) {
69 0 : folio_set_error(fi.folio);
70 0 : mapping_set_error(fi.folio->mapping, err);
71 : }
72 58293 : folio_end_writeback(fi.folio);
73 : }
74 :
75 4940 : bio_put(bio);
76 4940 : }
77 :
78 11113202 : static struct bio *mpage_bio_submit_read(struct bio *bio)
79 : {
80 11132627 : bio->bi_end_io = mpage_read_end_io;
81 11113202 : guard_bio_eod(bio);
82 11130359 : submit_bio(bio);
83 11129412 : return NULL;
84 : }
85 :
86 2202 : static struct bio *mpage_bio_submit_write(struct bio *bio)
87 : {
88 4940 : bio->bi_end_io = mpage_write_end_io;
89 2202 : guard_bio_eod(bio);
90 4053 : submit_bio(bio);
91 887 : return NULL;
92 : }
93 :
94 : /*
95 : * support function for mpage_readahead. The fs supplied get_block might
96 : * return an up to date buffer. This is used to map that buffer into
97 : * the page, which allows read_folio to avoid triggering a duplicate call
98 : * to get_block.
99 : *
100 : * The idea is to avoid adding buffers to pages that don't already have
101 : * them. So when the buffer is up to date and the page size == block size,
102 : * this marks the page up to date instead of adding new buffers.
103 : */
104 0 : static void map_buffer_to_folio(struct folio *folio, struct buffer_head *bh,
105 : int page_block)
106 : {
107 0 : struct inode *inode = folio->mapping->host;
108 0 : struct buffer_head *page_bh, *head;
109 0 : int block = 0;
110 :
111 0 : head = folio_buffers(folio);
112 0 : if (!head) {
113 : /*
114 : * don't make any buffers if there is only one buffer on
115 : * the folio and the folio just needs to be set up to date
116 : */
117 0 : if (inode->i_blkbits == PAGE_SHIFT &&
118 : buffer_uptodate(bh)) {
119 0 : folio_mark_uptodate(folio);
120 0 : return;
121 : }
122 0 : create_empty_buffers(&folio->page, i_blocksize(inode), 0);
123 0 : head = folio_buffers(folio);
124 : }
125 :
126 0 : page_bh = head;
127 0 : do {
128 0 : if (block == page_block) {
129 0 : page_bh->b_state = bh->b_state;
130 0 : page_bh->b_bdev = bh->b_bdev;
131 0 : page_bh->b_blocknr = bh->b_blocknr;
132 0 : break;
133 : }
134 0 : page_bh = page_bh->b_this_page;
135 0 : block++;
136 0 : } while (page_bh != head);
137 : }
138 :
139 : struct mpage_readpage_args {
140 : struct bio *bio;
141 : struct folio *folio;
142 : unsigned int nr_pages;
143 : bool is_readahead;
144 : sector_t last_block_in_bio;
145 : struct buffer_head map_bh;
146 : unsigned long first_logical_block;
147 : get_block_t *get_block;
148 : };
149 :
150 : /*
151 : * This is the worker routine which does all the work of mapping the disk
152 : * blocks and constructs largest possible bios, submits them for IO if the
153 : * blocks are not contiguous on the disk.
154 : *
155 : * We pass a buffer_head back and forth and use its buffer_mapped() flag to
156 : * represent the validity of its disk mapping and to decide when to do the next
157 : * get_block() call.
158 : */
159 141847255 : static struct bio *do_mpage_readpage(struct mpage_readpage_args *args)
160 : {
161 141847255 : struct folio *folio = args->folio;
162 141847255 : struct inode *inode = folio->mapping->host;
163 141847255 : const unsigned blkbits = inode->i_blkbits;
164 141847255 : const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
165 141847255 : const unsigned blocksize = 1 << blkbits;
166 141847255 : struct buffer_head *map_bh = &args->map_bh;
167 141847255 : sector_t block_in_file;
168 141847255 : sector_t last_block;
169 141847255 : sector_t last_block_in_file;
170 141847255 : sector_t blocks[MAX_BUF_PER_PAGE];
171 141847255 : unsigned page_block;
172 141847255 : unsigned first_hole = blocks_per_page;
173 141847255 : struct block_device *bdev = NULL;
174 141847255 : int length;
175 141847255 : int fully_mapped = 1;
176 141847255 : blk_opf_t opf = REQ_OP_READ;
177 141847255 : unsigned nblocks;
178 141847255 : unsigned relative_block;
179 141847255 : gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
180 :
181 : /* MAX_BUF_PER_PAGE, for example */
182 141847255 : VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
183 :
184 141847255 : if (args->is_readahead) {
185 141844481 : opf |= REQ_RAHEAD;
186 141844481 : gfp |= __GFP_NORETRY | __GFP_NOWARN;
187 : }
188 :
189 141847255 : if (folio_buffers(folio))
190 183 : goto confused;
191 :
192 141847072 : block_in_file = (sector_t)folio->index << (PAGE_SHIFT - blkbits);
193 141847072 : last_block = block_in_file + args->nr_pages * blocks_per_page;
194 141847072 : last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
195 141847072 : if (last_block > last_block_in_file)
196 : last_block = last_block_in_file;
197 141847072 : page_block = 0;
198 :
199 : /*
200 : * Map blocks using the result from the previous get_blocks call first.
201 : */
202 141847072 : nblocks = map_bh->b_size >> blkbits;
203 283694144 : if (buffer_mapped(map_bh) &&
204 130718230 : block_in_file > args->first_logical_block &&
205 130718201 : block_in_file < (args->first_logical_block + nblocks)) {
206 130718108 : unsigned map_offset = block_in_file - args->first_logical_block;
207 130718108 : unsigned last = nblocks - map_offset;
208 :
209 130718108 : for (relative_block = 0; ; relative_block++) {
210 292728216 : if (relative_block == last) {
211 3905196 : clear_buffer_mapped(map_bh);
212 : break;
213 : }
214 288823020 : if (page_block == blocks_per_page)
215 : break;
216 162010728 : blocks[page_block] = map_bh->b_blocknr + map_offset +
217 : relative_block;
218 162010108 : page_block++;
219 162010108 : block_in_file++;
220 : }
221 130717494 : bdev = map_bh->b_bdev;
222 : }
223 :
224 : /*
225 : * Then do more get_blocks calls until we are done with this folio.
226 : */
227 141846458 : map_bh->b_folio = folio;
228 153003520 : while (page_block < blocks_per_page) {
229 11157839 : map_bh->b_state = 0;
230 11157839 : map_bh->b_size = 0;
231 :
232 11157839 : if (block_in_file < last_block) {
233 11153376 : map_bh->b_size = (last_block-block_in_file) << blkbits;
234 11153376 : if (args->get_block(inode, block_in_file, map_bh, 0))
235 0 : goto confused;
236 11147711 : args->first_logical_block = block_in_file;
237 : }
238 :
239 22304348 : if (!buffer_mapped(map_bh)) {
240 34689 : fully_mapped = 0;
241 34689 : if (first_hole == blocks_per_page)
242 8839 : first_hole = page_block;
243 34689 : page_block++;
244 34689 : block_in_file++;
245 34689 : continue;
246 : }
247 :
248 : /* some filesystems will copy data into the page during
249 : * the get_block call, in which case we don't want to
250 : * read it again. map_buffer_to_folio copies the data
251 : * we just collected from get_block into the folio's buffers
252 : * so read_folio doesn't have to repeat the get_block call
253 : */
254 22231914 : if (buffer_uptodate(map_bh)) {
255 0 : map_buffer_to_folio(folio, map_bh, page_block);
256 0 : goto confused;
257 : }
258 :
259 11114429 : if (first_hole != blocks_per_page)
260 19 : goto confused; /* hole -> non-hole */
261 :
262 : /* Contiguous blocks? */
263 11114410 : if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
264 8 : goto confused;
265 11114402 : nblocks = map_bh->b_size >> blkbits;
266 11114402 : for (relative_block = 0; ; relative_block++) {
267 24061566 : if (relative_block == nblocks) {
268 7208850 : clear_buffer_mapped(map_bh);
269 : break;
270 16852716 : } else if (page_block == blocks_per_page)
271 : break;
272 12947608 : blocks[page_block] = map_bh->b_blocknr+relative_block;
273 12947164 : page_block++;
274 12947164 : block_in_file++;
275 : }
276 11122373 : bdev = map_bh->b_bdev;
277 : }
278 :
279 141845681 : if (first_hole != blocks_per_page) {
280 8820 : folio_zero_segment(folio, first_hole << blkbits, PAGE_SIZE);
281 8635 : if (first_hole == 0) {
282 8432 : folio_mark_uptodate(folio);
283 8432 : folio_unlock(folio);
284 8432 : goto out;
285 : }
286 141836861 : } else if (fully_mapped) {
287 141832968 : folio_set_mappedtodisk(folio);
288 : }
289 :
290 : /*
291 : * This folio will go to BIO. Do we need to send this BIO off first?
292 : */
293 141840923 : if (args->bio && (args->last_block_in_bio != blocks[0] - 1))
294 132 : args->bio = mpage_bio_submit_read(args->bio);
295 :
296 141840857 : alloc_new:
297 141857259 : if (args->bio == NULL) {
298 11137974 : args->bio = bio_alloc(bdev, bio_max_segs(args->nr_pages), opf,
299 : gfp);
300 11136547 : if (args->bio == NULL)
301 0 : goto confused;
302 11136547 : args->bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9);
303 : }
304 :
305 141855832 : length = first_hole << blkbits;
306 141855832 : if (!bio_add_folio(args->bio, folio, length, 0)) {
307 16336 : args->bio = mpage_bio_submit_read(args->bio);
308 16336 : goto alloc_new;
309 : }
310 :
311 141829544 : relative_block = block_in_file - args->first_logical_block;
312 141829544 : nblocks = map_bh->b_size >> blkbits;
313 283659088 : if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
314 : (first_hole != blocks_per_page))
315 3418 : args->bio = mpage_bio_submit_read(args->bio);
316 : else
317 141826526 : args->last_block_in_bio = blocks[blocks_per_page - 1];
318 141837164 : out:
319 141837164 : return args->bio;
320 :
321 210 : confused:
322 210 : if (args->bio)
323 10 : args->bio = mpage_bio_submit_read(args->bio);
324 210 : if (!folio_test_uptodate(folio))
325 210 : block_read_full_folio(folio, args->get_block);
326 : else
327 0 : folio_unlock(folio);
328 210 : goto out;
329 : }
330 :
331 : /**
332 : * mpage_readahead - start reads against pages
333 : * @rac: Describes which pages to read.
334 : * @get_block: The filesystem's block mapper function.
335 : *
336 : * This function walks the pages and the blocks within each page, building and
337 : * emitting large BIOs.
338 : *
339 : * If anything unusual happens, such as:
340 : *
341 : * - encountering a page which has buffers
342 : * - encountering a page which has a non-hole after a hole
343 : * - encountering a page with non-contiguous blocks
344 : *
345 : * then this code just gives up and calls the buffer_head-based read function.
346 : * It does handle a page which has holes at the end - that is a common case:
347 : * the end-of-file on blocksize < PAGE_SIZE setups.
348 : *
349 : * BH_Boundary explanation:
350 : *
351 : * There is a problem. The mpage read code assembles several pages, gets all
352 : * their disk mappings, and then submits them all. That's fine, but obtaining
353 : * the disk mappings may require I/O. Reads of indirect blocks, for example.
354 : *
355 : * So an mpage read of the first 16 blocks of an ext2 file will cause I/O to be
356 : * submitted in the following order:
357 : *
358 : * 12 0 1 2 3 4 5 6 7 8 9 10 11 13 14 15 16
359 : *
360 : * because the indirect block has to be read to get the mappings of blocks
361 : * 13,14,15,16. Obviously, this impacts performance.
362 : *
363 : * So what we do it to allow the filesystem's get_block() function to set
364 : * BH_Boundary when it maps block 11. BH_Boundary says: mapping of the block
365 : * after this one will require I/O against a block which is probably close to
366 : * this one. So you should push what I/O you have currently accumulated.
367 : *
368 : * This all causes the disk requests to be issued in the correct order.
369 : */
370 11118226 : void mpage_readahead(struct readahead_control *rac, get_block_t get_block)
371 : {
372 11118226 : struct folio *folio;
373 11118226 : struct mpage_readpage_args args = {
374 : .get_block = get_block,
375 : .is_readahead = true,
376 : };
377 :
378 152958695 : while ((folio = readahead_folio(rac))) {
379 141847325 : prefetchw(&folio->flags);
380 141847255 : args.folio = folio;
381 141847255 : args.nr_pages = readahead_count(rac);
382 141847255 : args.bio = do_mpage_readpage(&args);
383 : }
384 11112899 : if (args.bio)
385 11112026 : mpage_bio_submit_read(args.bio);
386 11110108 : }
387 : EXPORT_SYMBOL(mpage_readahead);
388 :
389 : /*
390 : * This isn't called much at all
391 : */
392 193 : int mpage_read_folio(struct folio *folio, get_block_t get_block)
393 : {
394 193 : struct mpage_readpage_args args = {
395 : .folio = folio,
396 : .nr_pages = 1,
397 : .get_block = get_block,
398 : };
399 :
400 193 : args.bio = do_mpage_readpage(&args);
401 193 : if (args.bio)
402 5 : mpage_bio_submit_read(args.bio);
403 193 : return 0;
404 : }
405 : EXPORT_SYMBOL(mpage_read_folio);
406 :
407 : /*
408 : * Writing is not so simple.
409 : *
410 : * If the page has buffers then they will be used for obtaining the disk
411 : * mapping. We only support pages which are fully mapped-and-dirty, with a
412 : * special case for pages which are unmapped at the end: end-of-file.
413 : *
414 : * If the page has no buffers (preferred) then the page is mapped here.
415 : *
416 : * If all blocks are found to be contiguous then the page can go into the
417 : * BIO. Otherwise fall back to the mapping's writepage().
418 : *
419 : * FIXME: This code wants an estimate of how many pages are still to be
420 : * written, so it can intelligently allocate a suitably-sized BIO. For now,
421 : * just allocate full-size (16-page) BIOs.
422 : */
423 :
424 : struct mpage_data {
425 : struct bio *bio;
426 : sector_t last_block_in_bio;
427 : get_block_t *get_block;
428 : };
429 :
430 : /*
431 : * We have our BIO, so we can now mark the buffers clean. Make
432 : * sure to only clean buffers which we know we'll be writing.
433 : */
434 58293 : static void clean_buffers(struct page *page, unsigned first_unmapped)
435 : {
436 58293 : unsigned buffer_counter = 0;
437 58293 : struct buffer_head *bh, *head;
438 58293 : if (!page_has_buffers(page))
439 : return;
440 58004 : head = page_buffers(page);
441 58004 : bh = head;
442 :
443 91927 : do {
444 91927 : if (buffer_counter++ == first_unmapped)
445 : break;
446 90325 : clear_buffer_dirty(bh);
447 90325 : bh = bh->b_this_page;
448 90325 : } while (bh != head);
449 :
450 : /*
451 : * we cannot drop the bh if the page is not uptodate or a concurrent
452 : * read_folio would fail to serialize with the bh and it would read from
453 : * disk before we reach the platter.
454 : */
455 58004 : if (buffer_heads_over_limit && PageUptodate(page))
456 0 : try_to_free_buffers(page_folio(page));
457 : }
458 :
459 : /*
460 : * For situations where we want to clean all buffers attached to a page.
461 : * We don't need to calculate how many buffers are attached to the page,
462 : * we just need to specify a number larger than the maximum number of buffers.
463 : */
464 0 : void clean_page_buffers(struct page *page)
465 : {
466 0 : clean_buffers(page, ~0U);
467 0 : }
468 :
469 59101 : static int __mpage_writepage(struct folio *folio, struct writeback_control *wbc,
470 : void *data)
471 : {
472 59101 : struct mpage_data *mpd = data;
473 59101 : struct bio *bio = mpd->bio;
474 59101 : struct address_space *mapping = folio->mapping;
475 59101 : struct inode *inode = mapping->host;
476 59101 : const unsigned blkbits = inode->i_blkbits;
477 59101 : const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
478 59101 : sector_t last_block;
479 59101 : sector_t block_in_file;
480 59101 : sector_t blocks[MAX_BUF_PER_PAGE];
481 59101 : unsigned page_block;
482 59101 : unsigned first_unmapped = blocks_per_page;
483 59101 : struct block_device *bdev = NULL;
484 59101 : int boundary = 0;
485 59101 : sector_t boundary_block = 0;
486 59101 : struct block_device *boundary_bdev = NULL;
487 59101 : size_t length;
488 59101 : struct buffer_head map_bh;
489 59101 : loff_t i_size = i_size_read(inode);
490 59101 : int ret = 0;
491 59101 : struct buffer_head *head = folio_buffers(folio);
492 :
493 59101 : if (head) {
494 : struct buffer_head *bh = head;
495 :
496 : /* If they're all mapped and dirty, do it */
497 : page_block = 0;
498 96988 : do {
499 193976 : BUG_ON(buffer_locked(bh));
500 193976 : if (!buffer_mapped(bh)) {
501 : /*
502 : * unmapped dirty buffers are created by
503 : * block_dirty_folio -> mmapped data
504 : */
505 10678 : if (buffer_dirty(bh))
506 9 : goto confused;
507 5330 : if (first_unmapped == blocks_per_page)
508 2106 : first_unmapped = page_block;
509 5330 : continue;
510 : }
511 :
512 91649 : if (first_unmapped != blocks_per_page)
513 504 : goto confused; /* hole -> non-hole */
514 :
515 273392 : if (!buffer_dirty(bh) || !buffer_uptodate(bh))
516 43 : goto confused;
517 91102 : if (page_block) {
518 32832 : if (bh->b_blocknr != blocks[page_block-1] + 1)
519 243 : goto confused;
520 : }
521 90859 : blocks[page_block++] = bh->b_blocknr;
522 90859 : boundary = buffer_boundary(bh);
523 90859 : if (boundary) {
524 206 : boundary_block = bh->b_blocknr;
525 206 : boundary_bdev = bh->b_bdev;
526 : }
527 90859 : bdev = bh->b_bdev;
528 96189 : } while ((bh = bh->b_this_page) != head);
529 :
530 58004 : if (first_unmapped)
531 58004 : goto page_is_mapped;
532 :
533 : /*
534 : * Page has buffers, but they are all unmapped. The page was
535 : * created by pagein or read over a hole which was handled by
536 : * block_read_full_folio(). If this address_space is also
537 : * using mpage_readahead then this can rarely happen.
538 : */
539 0 : goto confused;
540 : }
541 :
542 : /*
543 : * The page has no buffers: map it to disk
544 : */
545 596 : BUG_ON(!folio_test_uptodate(folio));
546 298 : block_in_file = (sector_t)folio->index << (PAGE_SHIFT - blkbits);
547 : /*
548 : * Whole page beyond EOF? Skip allocating blocks to avoid leaking
549 : * space.
550 : */
551 298 : if (block_in_file >= (i_size + (1 << blkbits) - 1) >> blkbits)
552 0 : goto page_is_mapped;
553 298 : last_block = (i_size - 1) >> blkbits;
554 298 : map_bh.b_folio = folio;
555 1466 : for (page_block = 0; page_block < blocks_per_page; ) {
556 :
557 1177 : map_bh.b_state = 0;
558 1177 : map_bh.b_size = 1 << blkbits;
559 1177 : if (mpd->get_block(inode, block_in_file, &map_bh, 1))
560 5 : goto confused;
561 2344 : if (!buffer_mapped(&map_bh))
562 0 : goto confused;
563 2344 : if (buffer_new(&map_bh))
564 1100 : clean_bdev_bh_alias(&map_bh);
565 2344 : if (buffer_boundary(&map_bh)) {
566 4 : boundary_block = map_bh.b_blocknr;
567 4 : boundary_bdev = map_bh.b_bdev;
568 : }
569 1172 : if (page_block) {
570 879 : if (map_bh.b_blocknr != blocks[page_block-1] + 1)
571 4 : goto confused;
572 : }
573 1168 : blocks[page_block++] = map_bh.b_blocknr;
574 1168 : boundary = buffer_boundary(&map_bh);
575 1168 : bdev = map_bh.b_bdev;
576 1168 : if (block_in_file == last_block)
577 : break;
578 1168 : block_in_file++;
579 : }
580 289 : BUG_ON(page_block == 0);
581 :
582 : first_unmapped = page_block;
583 :
584 58293 : page_is_mapped:
585 : /* Don't bother writing beyond EOF, truncate will discard the folio */
586 58293 : if (folio_pos(folio) >= i_size)
587 0 : goto confused;
588 58293 : length = folio_size(folio);
589 58293 : if (folio_pos(folio) + length > i_size) {
590 : /*
591 : * The page straddles i_size. It must be zeroed out on each
592 : * and every writepage invocation because it may be mmapped.
593 : * "A file is mapped in multiples of the page size. For a file
594 : * that is not a multiple of the page size, the remaining memory
595 : * is zeroed when mapped, and writes to that region are not
596 : * written out to the file."
597 : */
598 1338 : length = i_size - folio_pos(folio);
599 1338 : folio_zero_segment(folio, length, folio_size(folio));
600 : }
601 :
602 : /*
603 : * This page will go to BIO. Do we need to send this BIO off first?
604 : */
605 58293 : if (bio && mpd->last_block_in_bio != blocks[0] - 1)
606 614 : bio = mpage_bio_submit_write(bio);
607 :
608 57679 : alloc_new:
609 58293 : if (bio == NULL) {
610 5755 : bio = bio_alloc(bdev, BIO_MAX_VECS,
611 : REQ_OP_WRITE | wbc_to_write_flags(wbc),
612 : GFP_NOFS);
613 4940 : bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9);
614 4940 : wbc_init_bio(wbc, bio);
615 : }
616 :
617 : /*
618 : * Must try to add the page before marking the buffer clean or
619 : * the confused fail path above (OOM) will be very confused when
620 : * it finds all bh marked clean (i.e. it will not write anything)
621 : */
622 58332 : wbc_account_cgroup_owner(wbc, &folio->page, folio_size(folio));
623 58332 : length = first_unmapped << blkbits;
624 58332 : if (!bio_add_folio(bio, folio, length, 0)) {
625 39 : bio = mpage_bio_submit_write(bio);
626 39 : goto alloc_new;
627 : }
628 :
629 58293 : clean_buffers(&folio->page, first_unmapped);
630 :
631 58293 : BUG_ON(folio_test_writeback(folio));
632 58293 : folio_start_writeback(folio);
633 58293 : folio_unlock(folio);
634 58293 : if (boundary || (first_unmapped != blocks_per_page)) {
635 1812 : bio = mpage_bio_submit_write(bio);
636 1812 : if (boundary_block) {
637 210 : write_boundary_block(boundary_bdev,
638 210 : boundary_block, 1 << blkbits);
639 : }
640 : } else {
641 56481 : mpd->last_block_in_bio = blocks[blocks_per_page - 1];
642 : }
643 58293 : goto out;
644 :
645 808 : confused:
646 808 : if (bio)
647 273 : bio = mpage_bio_submit_write(bio);
648 :
649 : /*
650 : * The caller has a ref on the inode, so *mapping is stable
651 : */
652 808 : ret = block_write_full_page(&folio->page, mpd->get_block, wbc);
653 808 : mapping_set_error(mapping, ret);
654 59101 : out:
655 59101 : mpd->bio = bio;
656 59101 : return ret;
657 : }
658 :
659 : /**
660 : * mpage_writepages - walk the list of dirty pages of the given address space & writepage() all of them
661 : * @mapping: address space structure to write
662 : * @wbc: subtract the number of written pages from *@wbc->nr_to_write
663 : * @get_block: the filesystem's block mapper function.
664 : *
665 : * This is a library function, which implements the writepages()
666 : * address_space_operation.
667 : */
668 : int
669 4325 : mpage_writepages(struct address_space *mapping,
670 : struct writeback_control *wbc, get_block_t get_block)
671 : {
672 4325 : struct mpage_data mpd = {
673 : .get_block = get_block,
674 : };
675 4325 : struct blk_plug plug;
676 4325 : int ret;
677 :
678 4325 : blk_start_plug(&plug);
679 4325 : ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
680 4325 : if (mpd.bio)
681 2202 : mpage_bio_submit_write(mpd.bio);
682 4325 : blk_finish_plug(&plug);
683 4325 : return ret;
684 : }
685 : EXPORT_SYMBOL(mpage_writepages);
|
