Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (C) 2010 Red Hat, Inc.
4 : * Copyright (C) 2016-2019 Christoph Hellwig.
5 : */
6 : #include <linux/module.h>
7 : #include <linux/compiler.h>
8 : #include <linux/fs.h>
9 : #include <linux/iomap.h>
10 : #include <linux/pagemap.h>
11 : #include <linux/uio.h>
12 : #include <linux/buffer_head.h>
13 : #include <linux/dax.h>
14 : #include <linux/writeback.h>
15 : #include <linux/list_sort.h>
16 : #include <linux/swap.h>
17 : #include <linux/bio.h>
18 : #include <linux/sched/signal.h>
19 : #include <linux/migrate.h>
20 : #include "trace.h"
21 :
22 : #include "../internal.h"
23 :
24 : #define IOEND_BATCH_SIZE 4096
25 :
26 : typedef int (*iomap_punch_t)(struct inode *inode, loff_t offset, loff_t length);
27 : /*
28 : * Structure allocated for each folio to track per-block uptodate, dirty state
29 : * and I/O completions.
30 : */
31 : struct iomap_folio_state {
32 : atomic_t read_bytes_pending;
33 : atomic_t write_bytes_pending;
34 : spinlock_t state_lock;
35 :
36 : /*
37 : * Each block has two bits in this bitmap:
38 : * Bits [0..blocks_per_folio) has the uptodate status.
39 : * Bits [b_p_f...(2*b_p_f)) has the dirty status.
40 : */
41 : unsigned long state[];
42 : };
43 :
44 : static struct bio_set iomap_ioend_bioset;
45 :
46 304929893 : static inline bool ifs_is_fully_uptodate(struct folio *folio,
47 : struct iomap_folio_state *ifs)
48 : {
49 304929893 : struct inode *inode = folio->mapping->host;
50 :
51 304929893 : return bitmap_full(ifs->state, i_blocks_per_folio(inode, folio));
52 : }
53 :
54 739433984 : static inline bool ifs_block_is_uptodate(struct iomap_folio_state *ifs,
55 : unsigned int block)
56 : {
57 739433984 : return test_bit(block, ifs->state);
58 : }
59 :
60 159353985 : static void ifs_set_range_uptodate(struct folio *folio,
61 : struct iomap_folio_state *ifs, size_t off, size_t len)
62 : {
63 159353985 : struct inode *inode = folio->mapping->host;
64 159353985 : unsigned int first_blk = off >> inode->i_blkbits;
65 159353985 : unsigned int last_blk = (off + len - 1) >> inode->i_blkbits;
66 159353985 : unsigned int nr_blks = last_blk - first_blk + 1;
67 159353985 : unsigned long flags;
68 :
69 159353985 : spin_lock_irqsave(&ifs->state_lock, flags);
70 159359757 : bitmap_set(ifs->state, first_blk, nr_blks);
71 159355147 : if (ifs_is_fully_uptodate(folio, ifs))
72 134753731 : folio_mark_uptodate(folio);
73 159354416 : spin_unlock_irqrestore(&ifs->state_lock, flags);
74 159357506 : }
75 :
76 2468665740 : static void iomap_set_range_uptodate(struct folio *folio, size_t off,
77 : size_t len)
78 : {
79 2468665740 : struct iomap_folio_state *ifs = folio->private;
80 :
81 2468665740 : if (ifs)
82 159354499 : ifs_set_range_uptodate(folio, ifs, off, len);
83 : else
84 2309311241 : folio_mark_uptodate(folio);
85 2470091632 : }
86 :
87 436642029 : static inline bool ifs_block_is_dirty(struct folio *folio,
88 : struct iomap_folio_state *ifs, int block)
89 : {
90 436642029 : struct inode *inode = folio->mapping->host;
91 436642029 : unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);
92 :
93 436636179 : return test_bit(block + blks_per_folio, ifs->state);
94 : }
95 :
96 47043189 : static void ifs_clear_range_dirty(struct folio *folio,
97 : struct iomap_folio_state *ifs, size_t off, size_t len)
98 : {
99 47043189 : struct inode *inode = folio->mapping->host;
100 47043189 : unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);
101 47043038 : unsigned int first_blk = (off >> inode->i_blkbits);
102 47043038 : unsigned int last_blk = (off + len - 1) >> inode->i_blkbits;
103 47043038 : unsigned int nr_blks = last_blk - first_blk + 1;
104 47043038 : unsigned long flags;
105 :
106 47043038 : spin_lock_irqsave(&ifs->state_lock, flags);
107 47043461 : bitmap_clear(ifs->state, first_blk + blks_per_folio, nr_blks);
108 47043352 : spin_unlock_irqrestore(&ifs->state_lock, flags);
109 47043304 : }
110 :
111 261829107 : static void iomap_clear_range_dirty(struct folio *folio, size_t off, size_t len)
112 : {
113 261829107 : struct iomap_folio_state *ifs = folio->private;
114 :
115 261829107 : if (ifs)
116 47043203 : ifs_clear_range_dirty(folio, ifs, off, len);
117 261829021 : }
118 :
119 181956045 : static void ifs_set_range_dirty(struct folio *folio,
120 : struct iomap_folio_state *ifs, size_t off, size_t len)
121 : {
122 181956045 : struct inode *inode = folio->mapping->host;
123 181956045 : unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);
124 181684968 : unsigned int first_blk = (off >> inode->i_blkbits);
125 181684968 : unsigned int last_blk = (off + len - 1) >> inode->i_blkbits;
126 181684968 : unsigned int nr_blks = last_blk - first_blk + 1;
127 181684968 : unsigned long flags;
128 :
129 181684968 : spin_lock_irqsave(&ifs->state_lock, flags);
130 182550863 : bitmap_set(ifs->state, first_blk + blks_per_folio, nr_blks);
131 182089648 : spin_unlock_irqrestore(&ifs->state_lock, flags);
132 182470330 : }
133 :
134 691409982 : static void iomap_set_range_dirty(struct folio *folio, size_t off, size_t len)
135 : {
136 691409982 : struct iomap_folio_state *ifs = folio->private;
137 :
138 691409982 : if (ifs)
139 181981780 : ifs_set_range_dirty(folio, ifs, off, len);
140 691893140 : }
141 :
142 2458257833 : static struct iomap_folio_state *ifs_alloc(struct inode *inode,
143 : struct folio *folio, unsigned int flags)
144 : {
145 2458257833 : struct iomap_folio_state *ifs = folio->private;
146 2458257833 : unsigned int nr_blocks = i_blocks_per_folio(inode, folio);
147 2458179512 : gfp_t gfp;
148 :
149 2458179512 : if (ifs || nr_blocks <= 1)
150 : return ifs;
151 :
152 145558493 : if (flags & IOMAP_NOWAIT)
153 : gfp = GFP_NOWAIT;
154 : else
155 145559486 : gfp = GFP_NOFS | __GFP_NOFAIL;
156 :
157 : /*
158 : * ifs->state tracks two sets of state flags when the
159 : * filesystem block size is smaller than the folio size.
160 : * The first state tracks per-block uptodate and the
161 : * second tracks per-block dirty state.
162 : */
163 145558493 : ifs = kzalloc(struct_size(ifs, state,
164 : BITS_TO_LONGS(2 * nr_blocks)), gfp);
165 145560955 : if (!ifs)
166 : return ifs;
167 :
168 145560955 : spin_lock_init(&ifs->state_lock);
169 145561759 : if (folio_test_uptodate(folio))
170 31067136 : bitmap_set(ifs->state, 0, nr_blocks);
171 145558291 : if (folio_test_dirty(folio))
172 229435 : bitmap_set(ifs->state, nr_blocks, nr_blocks);
173 145560003 : folio_attach_private(folio, ifs);
174 :
175 145560003 : return ifs;
176 : }
177 :
178 145580157 : static void ifs_free(struct folio *folio)
179 : {
180 145580157 : struct iomap_folio_state *ifs = folio_detach_private(folio);
181 :
182 145581784 : if (!ifs)
183 : return;
184 145581784 : WARN_ON_ONCE(atomic_read(&ifs->read_bytes_pending));
185 145581784 : WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending));
186 145581784 : WARN_ON_ONCE(ifs_is_fully_uptodate(folio, ifs) !=
187 : folio_test_uptodate(folio));
188 145576209 : kfree(ifs);
189 : }
190 :
191 : /*
192 : * Calculate the range inside the folio that we actually need to read.
193 : */
194 2056667807 : static void iomap_adjust_read_range(struct inode *inode, struct folio *folio,
195 : loff_t *pos, loff_t length, size_t *offp, size_t *lenp)
196 : {
197 2056667807 : struct iomap_folio_state *ifs = folio->private;
198 2056667807 : loff_t orig_pos = *pos;
199 2056667807 : loff_t isize = i_size_read(inode);
200 2056667807 : unsigned block_bits = inode->i_blkbits;
201 2056667807 : unsigned block_size = (1 << block_bits);
202 2056667807 : size_t poff = offset_in_folio(folio, *pos);
203 2055713919 : size_t plen = min_t(loff_t, folio_size(folio) - poff, length);
204 2055784659 : unsigned first = poff >> block_bits;
205 2055784659 : unsigned last = (poff + plen - 1) >> block_bits;
206 :
207 : /*
208 : * If the block size is smaller than the page size, we need to check the
209 : * per-block uptodate status and adjust the offset and length if needed
210 : * to avoid reading in already uptodate ranges.
211 : */
212 2055784659 : if (ifs) {
213 : unsigned int i;
214 :
215 : /* move forward for each leading block marked uptodate */
216 153459690 : for (i = first; i <= last; i++) {
217 139658837 : if (!ifs_block_is_uptodate(ifs, i))
218 : break;
219 13990785 : *pos += block_size;
220 13990785 : poff += block_size;
221 13990785 : plen -= block_size;
222 13990785 : first++;
223 : }
224 :
225 : /* truncate len if we find any trailing uptodate block(s) */
226 739267643 : for ( ; i <= last; i++) {
227 599798046 : if (ifs_block_is_uptodate(ifs, i)) {
228 2 : plen -= (last - i + 1) * block_size;
229 2 : last = i - 1;
230 2 : break;
231 : }
232 : }
233 : }
234 :
235 : /*
236 : * If the extent spans the block that contains the i_size, we need to
237 : * handle both halves separately so that we properly zero data in the
238 : * page cache for blocks that are entirely outside of i_size.
239 : */
240 2055785353 : if (orig_pos <= isize && orig_pos + length > isize) {
241 104484305 : unsigned end = offset_in_folio(folio, isize - 1) >> block_bits;
242 :
243 104471674 : if (first <= end && last > end)
244 4867145 : plen -= (last - end) * block_size;
245 : }
246 :
247 2055772722 : *offp = poff;
248 2055772722 : *lenp = plen;
249 2055772722 : }
250 :
251 70619514 : static void iomap_finish_folio_read(struct folio *folio, size_t offset,
252 : size_t len, int error)
253 : {
254 70619514 : struct iomap_folio_state *ifs = folio->private;
255 :
256 70619514 : if (unlikely(error)) {
257 6678 : folio_clear_uptodate(folio);
258 6678 : folio_set_error(folio);
259 : } else {
260 70612836 : iomap_set_range_uptodate(folio, offset, len);
261 : }
262 :
263 70619514 : if (!ifs || atomic_sub_and_test(len, &ifs->read_bytes_pending))
264 68633115 : folio_unlock(folio);
265 70619514 : }
266 :
267 24661933 : static void iomap_read_end_io(struct bio *bio)
268 : {
269 24661933 : int error = blk_status_to_errno(bio->bi_status);
270 24661933 : struct folio_iter fi;
271 :
272 95281447 : bio_for_each_folio_all(fi, bio)
273 70619514 : iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);
274 24661933 : bio_put(bio);
275 24661933 : }
276 :
277 : struct iomap_readpage_ctx {
278 : struct folio *cur_folio;
279 : bool cur_folio_in_bio;
280 : struct bio *bio;
281 : struct readahead_control *rac;
282 : };
283 :
284 : /**
285 : * iomap_read_inline_data - copy inline data into the page cache
286 : * @iter: iteration structure
287 : * @folio: folio to copy to
288 : *
289 : * Copy the inline data in @iter into @folio and zero out the rest of the folio.
290 : * Only a single IOMAP_INLINE extent is allowed at the end of each file.
291 : * Returns zero for success to complete the read, or the usual negative errno.
292 : */
293 0 : static int iomap_read_inline_data(const struct iomap_iter *iter,
294 : struct folio *folio)
295 : {
296 0 : const struct iomap *iomap = iomap_iter_srcmap(iter);
297 0 : size_t size = i_size_read(iter->inode) - iomap->offset;
298 0 : size_t poff = offset_in_page(iomap->offset);
299 0 : size_t offset = offset_in_folio(folio, iomap->offset);
300 0 : void *addr;
301 :
302 0 : if (folio_test_uptodate(folio))
303 : return 0;
304 :
305 0 : if (WARN_ON_ONCE(size > PAGE_SIZE - poff))
306 : return -EIO;
307 0 : if (WARN_ON_ONCE(size > PAGE_SIZE -
308 : offset_in_page(iomap->inline_data)))
309 : return -EIO;
310 0 : if (WARN_ON_ONCE(size > iomap->length))
311 : return -EIO;
312 0 : if (offset > 0)
313 0 : ifs_alloc(iter->inode, folio, iter->flags);
314 :
315 0 : addr = kmap_local_folio(folio, offset);
316 0 : memcpy(addr, iomap->inline_data, size);
317 0 : memset(addr + size, 0, PAGE_SIZE - poff - size);
318 0 : kunmap_local(addr);
319 0 : iomap_set_range_uptodate(folio, offset, PAGE_SIZE - poff);
320 0 : return 0;
321 : }
322 :
323 2041744006 : static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter,
324 : loff_t pos)
325 : {
326 2041744006 : const struct iomap *srcmap = iomap_iter_srcmap(iter);
327 :
328 2119373215 : return srcmap->type != IOMAP_MAPPED ||
329 2041744006 : (srcmap->flags & IOMAP_F_NEW) ||
330 77628554 : pos >= i_size_read(iter->inode);
331 : }
332 :
333 1972233331 : static loff_t iomap_readpage_iter(const struct iomap_iter *iter,
334 : struct iomap_readpage_ctx *ctx, loff_t offset)
335 : {
336 1972233331 : const struct iomap *iomap = &iter->iomap;
337 1972233331 : loff_t pos = iter->pos + offset;
338 1972233331 : loff_t length = iomap_length(iter) - offset;
339 1972233331 : struct folio *folio = ctx->cur_folio;
340 1972233331 : struct iomap_folio_state *ifs;
341 1972233331 : loff_t orig_pos = pos;
342 1972233331 : size_t poff, plen;
343 1972233331 : sector_t sector;
344 :
345 1972233331 : if (iomap->type == IOMAP_INLINE)
346 0 : return iomap_read_inline_data(iter, folio);
347 :
348 : /* zero post-eof blocks as the page may be mapped */
349 1972233331 : ifs = ifs_alloc(iter->inode, folio, iter->flags);
350 1972284091 : iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen);
351 1971426504 : if (plen == 0)
352 182 : goto done;
353 :
354 1971426322 : if (iomap_block_needs_zeroing(iter, pos)) {
355 1900224385 : folio_zero_range(folio, poff, plen);
356 1901033995 : iomap_set_range_uptodate(folio, poff, plen);
357 1901948988 : goto done;
358 : }
359 :
360 71201937 : ctx->cur_folio_in_bio = true;
361 71201937 : if (ifs)
362 10876654 : atomic_add(plen, &ifs->read_bytes_pending);
363 :
364 71201961 : sector = iomap_sector(iomap, pos);
365 71201961 : if (!ctx->bio ||
366 99835437 : bio_end_sector(ctx->bio) != sector ||
367 46665609 : !bio_add_folio(ctx->bio, folio, plen, poff)) {
368 24653522 : gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
369 24653522 : gfp_t orig_gfp = gfp;
370 24653522 : unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
371 :
372 24653522 : if (ctx->bio)
373 6620560 : submit_bio(ctx->bio);
374 :
375 24654335 : if (ctx->rac) /* same as readahead_gfp_mask */
376 12125820 : gfp |= __GFP_NORETRY | __GFP_NOWARN;
377 24654335 : ctx->bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs),
378 : REQ_OP_READ, gfp);
379 : /*
380 : * If the bio_alloc fails, try it again for a single page to
381 : * avoid having to deal with partial page reads. This emulates
382 : * what do_mpage_read_folio does.
383 : */
384 24657676 : if (!ctx->bio) {
385 0 : ctx->bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ,
386 : orig_gfp);
387 : }
388 24657676 : if (ctx->rac)
389 12126182 : ctx->bio->bi_opf |= REQ_RAHEAD;
390 24657676 : ctx->bio->bi_iter.bi_sector = sector;
391 24657676 : ctx->bio->bi_end_io = iomap_read_end_io;
392 24657676 : bio_add_folio_nofail(ctx->bio, folio, plen, poff);
393 : }
394 :
395 46548348 : done:
396 : /*
397 : * Move the caller beyond our range so that it keeps making progress.
398 : * For that, we have to include any leading non-uptodate ranges, but
399 : * we can skip trailing ones as they will be handled in the next
400 : * iteration.
401 : */
402 1973152900 : return pos - orig_pos + plen;
403 : }
404 :
405 1201416694 : int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops)
406 : {
407 2402360126 : struct iomap_iter iter = {
408 1201416694 : .inode = folio->mapping->host,
409 : .pos = folio_pos(folio),
410 1201416694 : .len = folio_size(folio),
411 : };
412 1200943432 : struct iomap_readpage_ctx ctx = {
413 : .cur_folio = folio,
414 : };
415 1200943432 : int ret;
416 :
417 1200943432 : trace_iomap_readpage(iter.inode, 1);
418 :
419 2402591072 : while ((ret = iomap_iter(&iter, ops)) > 0)
420 1200800249 : iter.processed = iomap_readpage_iter(&iter, &ctx, 0);
421 :
422 1201005645 : if (ret < 0)
423 324 : folio_set_error(folio);
424 :
425 1201005645 : if (ctx.bio) {
426 12528704 : submit_bio(ctx.bio);
427 12535524 : WARN_ON_ONCE(!ctx.cur_folio_in_bio);
428 : } else {
429 1188476941 : WARN_ON_ONCE(ctx.cur_folio_in_bio);
430 1188476941 : folio_unlock(folio);
431 : }
432 :
433 : /*
434 : * Just like mpage_readahead and block_read_full_folio, we always
435 : * return 0 and just set the folio error flag on errors. This
436 : * should be cleaned up throughout the stack eventually.
437 : */
438 1201533319 : return 0;
439 : }
440 : EXPORT_SYMBOL_GPL(iomap_read_folio);
441 :
442 52156349 : static loff_t iomap_readahead_iter(const struct iomap_iter *iter,
443 : struct iomap_readpage_ctx *ctx)
444 : {
445 52156349 : loff_t length = iomap_length(iter);
446 52156349 : loff_t done, ret;
447 :
448 823793837 : for (done = 0; done < length; done += ret) {
449 771636913 : if (ctx->cur_folio &&
450 739637954 : offset_in_folio(ctx->cur_folio, iter->pos + done) == 0) {
451 729025435 : if (!ctx->cur_folio_in_bio)
452 676581930 : folio_unlock(ctx->cur_folio);
453 729079421 : ctx->cur_folio = NULL;
454 : }
455 771632998 : if (!ctx->cur_folio) {
456 761075089 : ctx->cur_folio = readahead_folio(ctx->rac);
457 761067377 : ctx->cur_folio_in_bio = false;
458 : }
459 771625286 : ret = iomap_readpage_iter(iter, ctx, done);
460 771637488 : if (ret <= 0)
461 0 : return ret;
462 : }
463 :
464 : return done;
465 : }
466 :
467 : /**
468 : * iomap_readahead - Attempt to read pages from a file.
469 : * @rac: Describes the pages to be read.
470 : * @ops: The operations vector for the filesystem.
471 : *
472 : * This function is for filesystems to call to implement their readahead
473 : * address_space operation.
474 : *
475 : * Context: The @ops callbacks may submit I/O (eg to read the addresses of
476 : * blocks from disc), and may wait for it. The caller may be trying to
477 : * access a different page, and so sleeping excessively should be avoided.
478 : * It may allocate memory, but should avoid costly allocations. This
479 : * function is called with memalloc_nofs set, so allocations will not cause
480 : * the filesystem to be reentered.
481 : */
482 31999691 : void iomap_readahead(struct readahead_control *rac, const struct iomap_ops *ops)
483 : {
484 31999691 : struct iomap_iter iter = {
485 31999691 : .inode = rac->mapping->host,
486 : .pos = readahead_pos(rac),
487 : .len = readahead_length(rac),
488 : };
489 31999691 : struct iomap_readpage_ctx ctx = {
490 : .rac = rac,
491 : };
492 :
493 31999691 : trace_iomap_readahead(rac->mapping->host, readahead_count(rac));
494 :
495 84156541 : while (iomap_iter(&iter, ops) > 0)
496 52156393 : iter.processed = iomap_readahead_iter(&iter, &ctx);
497 :
498 31999521 : if (ctx.bio)
499 5505359 : submit_bio(ctx.bio);
500 31999596 : if (ctx.cur_folio) {
501 31999460 : if (!ctx.cur_folio_in_bio)
502 28347031 : folio_unlock(ctx.cur_folio);
503 : }
504 31999653 : }
505 : EXPORT_SYMBOL_GPL(iomap_readahead);
506 :
507 : /*
508 : * iomap_is_partially_uptodate checks whether blocks within a folio are
509 : * uptodate or not.
510 : *
511 : * Returns true if all blocks which correspond to the specified part
512 : * of the folio are uptodate.
513 : */
514 46582 : bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
515 : {
516 46582 : struct iomap_folio_state *ifs = folio->private;
517 46582 : struct inode *inode = folio->mapping->host;
518 46582 : unsigned first, last, i;
519 :
520 46582 : if (!ifs)
521 : return false;
522 :
523 : /* Caller's range may extend past the end of this folio */
524 43 : count = min(folio_size(folio) - from, count);
525 :
526 : /* First and last blocks in range within folio */
527 43 : first = from >> inode->i_blkbits;
528 43 : last = (from + count - 1) >> inode->i_blkbits;
529 :
530 152 : for (i = first; i <= last; i++)
531 151 : if (!ifs_block_is_uptodate(ifs, i))
532 : return false;
533 : return true;
534 : }
535 : EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate);
536 :
537 : /**
538 : * iomap_get_folio - get a folio reference for writing
539 : * @iter: iteration structure
540 : * @pos: start offset of write
541 : * @len: Suggested size of folio to create.
542 : *
543 : * Returns a locked reference to the folio at @pos, or an error pointer if the
544 : * folio could not be obtained.
545 : */
546 426950046 : struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len)
547 : {
548 426950046 : fgf_t fgp = FGP_WRITEBEGIN | FGP_NOFS;
549 :
550 426950046 : if (iter->flags & IOMAP_NOWAIT)
551 0 : fgp |= FGP_NOWAIT;
552 426950046 : fgp |= fgf_set_order(len);
553 :
554 854407217 : return __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT,
555 426950046 : fgp, mapping_gfp_mask(iter->inode->i_mapping));
556 : }
557 : EXPORT_SYMBOL_GPL(iomap_get_folio);
558 :
559 26304575 : bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags)
560 : {
561 26304562 : trace_iomap_release_folio(folio->mapping->host, folio_pos(folio),
562 26304575 : folio_size(folio));
563 :
564 : /*
565 : * If the folio is dirty, we refuse to release our metadata because
566 : * it may be partially dirty. Once we track per-block dirty state,
567 : * we can release the metadata if every block is dirty.
568 : */
569 26304542 : if (folio_test_dirty(folio))
570 : return false;
571 26119273 : ifs_free(folio);
572 26119273 : return true;
573 : }
574 : EXPORT_SYMBOL_GPL(iomap_release_folio);
575 :
576 126935550 : void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len)
577 : {
578 126935550 : trace_iomap_invalidate_folio(folio->mapping->host,
579 126935550 : folio_pos(folio) + offset, len);
580 :
581 : /*
582 : * If we're invalidating the entire folio, clear the dirty state
583 : * from it and release it to avoid unnecessary buildup of the LRU.
584 : */
585 126935032 : if (offset == 0 && len == folio_size(folio)) {
586 119461065 : WARN_ON_ONCE(folio_test_writeback(folio));
587 119461134 : folio_cancel_dirty(folio);
588 119461173 : ifs_free(folio);
589 : }
590 126937769 : }
591 : EXPORT_SYMBOL_GPL(iomap_invalidate_folio);
592 :
593 232863592 : bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio)
594 : {
595 232863592 : struct inode *inode = mapping->host;
596 232863592 : size_t len = folio_size(folio);
597 :
598 232763262 : ifs_alloc(inode, folio, 0);
599 232806480 : iomap_set_range_dirty(folio, 0, len);
600 233305514 : return filemap_dirty_folio(mapping, folio);
601 : }
602 : EXPORT_SYMBOL_GPL(iomap_dirty_folio);
603 :
604 : static void
605 112867 : iomap_write_failed(struct inode *inode, loff_t pos, unsigned len)
606 : {
607 112867 : loff_t i_size = i_size_read(inode);
608 :
609 : /*
610 : * Only truncate newly allocated pages beyoned EOF, even if the
611 : * write started inside the existing inode size.
612 : */
613 112867 : if (pos + len > i_size)
614 39664 : truncate_pagecache_range(inode, max(pos, i_size),
615 : pos + len - 1);
616 112867 : }
617 :
618 6412126 : static int iomap_read_folio_sync(loff_t block_start, struct folio *folio,
619 : size_t poff, size_t plen, const struct iomap *iomap)
620 : {
621 6412126 : struct bio_vec bvec;
622 6412126 : struct bio bio;
623 :
624 6412126 : bio_init(&bio, iomap->bdev, &bvec, 1, REQ_OP_READ);
625 6412140 : bio.bi_iter.bi_sector = iomap_sector(iomap, block_start);
626 6412140 : bio_add_folio_nofail(&bio, folio, plen, poff);
627 6412093 : return submit_bio_wait(&bio);
628 : }
629 :
630 427235073 : static int __iomap_write_begin(const struct iomap_iter *iter, loff_t pos,
631 : size_t len, struct folio *folio)
632 : {
633 427235073 : const struct iomap *srcmap = iomap_iter_srcmap(iter);
634 427235073 : struct iomap_folio_state *ifs;
635 427235073 : loff_t block_size = i_blocksize(iter->inode);
636 427263567 : loff_t block_start = round_down(pos, block_size);
637 427263567 : loff_t block_end = round_up(pos + len, block_size);
638 427263567 : unsigned int nr_blocks = i_blocks_per_folio(iter->inode, folio);
639 427315988 : size_t from = offset_in_folio(folio, pos), to = from + len;
640 427476837 : size_t poff, plen;
641 :
642 : /*
643 : * If the write completely overlaps the current folio, then
644 : * entire folio will be dirtied so there is no need for
645 : * per-block state tracking structures to be attached to this folio.
646 : */
647 427476837 : if (pos <= folio_pos(folio) &&
648 256705111 : pos + len >= folio_pos(folio) + folio_size(folio))
649 : return 0;
650 :
651 222791588 : ifs = ifs_alloc(iter->inode, folio, iter->flags);
652 222484941 : if ((iter->flags & IOMAP_NOWAIT) && !ifs && nr_blocks > 1)
653 : return -EAGAIN;
654 :
655 222484941 : if (folio_test_uptodate(folio))
656 : return 0;
657 83978659 : folio_clear_error(folio);
658 :
659 84048601 : do {
660 84048601 : iomap_adjust_read_range(iter->inode, folio, &block_start,
661 : block_end - block_start, &poff, &plen);
662 83955297 : if (plen == 0)
663 : break;
664 :
665 70155583 : if (!(iter->flags & IOMAP_UNSHARE) &&
666 70200938 : (from <= poff || from >= poff + plen) &&
667 48171963 : (to <= poff || to >= poff + plen))
668 546805 : continue;
669 :
670 69608778 : if (iomap_block_needs_zeroing(iter, block_start)) {
671 63196662 : if (WARN_ON_ONCE(iter->flags & IOMAP_UNSHARE))
672 : return -EIO;
673 63196662 : folio_zero_segments(folio, poff, from, to, poff + plen);
674 : } else {
675 6412116 : int status;
676 :
677 6412116 : if (iter->flags & IOMAP_NOWAIT)
678 : return -EAGAIN;
679 :
680 6412116 : status = iomap_read_folio_sync(block_start, folio,
681 : poff, plen, srcmap);
682 6412023 : if (status)
683 1812 : return status;
684 : }
685 69712995 : iomap_set_range_uptodate(folio, poff, plen);
686 70283443 : } while ((block_start += plen) < block_end);
687 :
688 : return 0;
689 : }
690 :
691 426908931 : static struct folio *__iomap_get_folio(struct iomap_iter *iter, loff_t pos,
692 : size_t len)
693 : {
694 426908931 : const struct iomap_folio_ops *folio_ops = iter->iomap.folio_ops;
695 :
696 426908931 : if (folio_ops && folio_ops->get_folio)
697 0 : return folio_ops->get_folio(iter, pos, len);
698 : else
699 426908931 : return iomap_get_folio(iter, pos, len);
700 : }
701 :
702 427748999 : static void __iomap_put_folio(struct iomap_iter *iter, loff_t pos, size_t ret,
703 : struct folio *folio)
704 : {
705 427748999 : const struct iomap_folio_ops *folio_ops = iter->iomap.folio_ops;
706 :
707 427748999 : if (folio_ops && folio_ops->put_folio) {
708 0 : folio_ops->put_folio(iter->inode, pos, ret, folio);
709 : } else {
710 427748999 : folio_unlock(folio);
711 427847033 : folio_put(folio);
712 : }
713 428042870 : }
714 :
715 0 : static int iomap_write_begin_inline(const struct iomap_iter *iter,
716 : struct folio *folio)
717 : {
718 : /* needs more work for the tailpacking case; disable for now */
719 0 : if (WARN_ON_ONCE(iomap_iter_srcmap(iter)->offset != 0))
720 : return -EIO;
721 0 : return iomap_read_inline_data(iter, folio);
722 : }
723 :
724 427374909 : static int iomap_write_begin(struct iomap_iter *iter, loff_t pos,
725 : size_t len, struct folio **foliop)
726 : {
727 427374909 : const struct iomap_folio_ops *folio_ops = iter->iomap.folio_ops;
728 427374909 : const struct iomap *srcmap = iomap_iter_srcmap(iter);
729 427374909 : struct folio *folio;
730 427374909 : int status = 0;
731 :
732 427374909 : BUG_ON(pos + len > iter->iomap.offset + iter->iomap.length);
733 427374909 : if (srcmap != &iter->iomap)
734 8870469 : BUG_ON(pos + len > srcmap->offset + srcmap->length);
735 :
736 427374909 : if (fatal_signal_pending(current))
737 : return -EINTR;
738 :
739 853566600 : if (!mapping_large_folio_support(iter->inode->i_mapping))
740 0 : len = min_t(size_t, len, PAGE_SIZE - offset_in_page(pos));
741 :
742 426783300 : folio = __iomap_get_folio(iter, pos, len);
743 427339302 : if (IS_ERR(folio))
744 0 : return PTR_ERR(folio);
745 :
746 : /*
747 : * Now we have a locked folio, before we do anything with it we need to
748 : * check that the iomap we have cached is not stale. The inode extent
749 : * mapping can change due to concurrent IO in flight (e.g.
750 : * IOMAP_UNWRITTEN state can change and memory reclaim could have
751 : * reclaimed a previously partially written page at this index after IO
752 : * completion before this write reaches this file offset) and hence we
753 : * could do the wrong thing here (zero a page range incorrectly or fail
754 : * to zero) and corrupt data.
755 : */
756 427339302 : if (folio_ops && folio_ops->iomap_valid) {
757 427339302 : bool iomap_valid = folio_ops->iomap_valid(iter->inode,
758 : &iter->iomap);
759 427245573 : if (!iomap_valid) {
760 111055 : iter->iomap.flags |= IOMAP_F_STALE;
761 111055 : status = 0;
762 111055 : goto out_unlock;
763 : }
764 : }
765 :
766 427134518 : if (pos + len > folio_pos(folio) + folio_size(folio))
767 140960542 : len = folio_pos(folio) + folio_size(folio) - pos;
768 :
769 427214617 : if (srcmap->type == IOMAP_INLINE)
770 0 : status = iomap_write_begin_inline(iter, folio);
771 427214617 : else if (srcmap->flags & IOMAP_F_BUFFER_HEAD)
772 0 : status = __block_write_begin_int(folio, pos, len, NULL, srcmap);
773 : else
774 427214617 : status = __iomap_write_begin(iter, pos, len, folio);
775 :
776 427365441 : if (unlikely(status))
777 1812 : goto out_unlock;
778 :
779 427363629 : *foliop = folio;
780 427363629 : return 0;
781 :
782 112867 : out_unlock:
783 112867 : __iomap_put_folio(iter, pos, 0, folio);
784 112867 : iomap_write_failed(iter->inode, pos, len);
785 :
786 112867 : return status;
787 : }
788 :
789 427393782 : static size_t __iomap_write_end(struct inode *inode, loff_t pos, size_t len,
790 : size_t copied, struct folio *folio)
791 : {
792 427393782 : flush_dcache_folio(folio);
793 :
794 : /*
795 : * The blocks that were entirely written will now be uptodate, so we
796 : * don't have to worry about a read_folio reading them and overwriting a
797 : * partial write. However, if we've encountered a short write and only
798 : * partially written into a block, it will not be marked uptodate, so a
799 : * read_folio might come in and destroy our partial write.
800 : *
801 : * Do the simplest thing and just treat any short write to a
802 : * non-uptodate page as a zero-length write, and force the caller to
803 : * redo the whole thing.
804 : */
805 427393782 : if (unlikely(copied < len && !folio_test_uptodate(folio)))
806 : return 0;
807 427393782 : iomap_set_range_uptodate(folio, offset_in_folio(folio, pos), len);
808 427695147 : iomap_set_range_dirty(folio, offset_in_folio(folio, pos), copied);
809 427587004 : filemap_dirty_folio(inode->i_mapping, folio);
810 427587004 : return copied;
811 : }
812 :
813 0 : static size_t iomap_write_end_inline(const struct iomap_iter *iter,
814 : struct folio *folio, loff_t pos, size_t copied)
815 : {
816 0 : const struct iomap *iomap = &iter->iomap;
817 0 : void *addr;
818 :
819 0 : WARN_ON_ONCE(!folio_test_uptodate(folio));
820 0 : BUG_ON(!iomap_inline_data_valid(iomap));
821 :
822 0 : flush_dcache_folio(folio);
823 0 : addr = kmap_local_folio(folio, pos);
824 0 : memcpy(iomap_inline_data(iomap, pos), addr, copied);
825 0 : kunmap_local(addr);
826 :
827 0 : mark_inode_dirty(iter->inode);
828 0 : return copied;
829 : }
830 :
831 : /* Returns the number of bytes copied. May be 0. Cannot be an errno. */
832 427897101 : static size_t iomap_write_end(struct iomap_iter *iter, loff_t pos, size_t len,
833 : size_t copied, struct folio *folio)
834 : {
835 427897101 : const struct iomap *srcmap = iomap_iter_srcmap(iter);
836 427897101 : loff_t old_size = iter->inode->i_size;
837 427897101 : size_t ret;
838 :
839 427897101 : if (srcmap->type == IOMAP_INLINE) {
840 0 : ret = iomap_write_end_inline(iter, folio, pos, copied);
841 427897101 : } else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) {
842 0 : ret = block_write_end(NULL, iter->inode->i_mapping, pos, len,
843 : copied, &folio->page, NULL);
844 : } else {
845 427897101 : ret = __iomap_write_end(iter->inode, pos, len, copied, folio);
846 : }
847 :
848 : /*
849 : * Update the in-memory inode size after copying the data into the page
850 : * cache. It's up to the file system to write the updated size to disk,
851 : * preferably after I/O completion so that no stale data is exposed.
852 : */
853 427727396 : if (pos + ret > old_size) {
854 284645688 : i_size_write(iter->inode, pos + ret);
855 284645688 : iter->iomap.flags |= IOMAP_F_SIZE_CHANGED;
856 : }
857 427727396 : __iomap_put_folio(iter, pos, ret, folio);
858 :
859 427880558 : if (old_size < pos)
860 36951955 : pagecache_isize_extended(iter->inode, old_size, pos);
861 427878467 : if (ret < len)
862 0 : iomap_write_failed(iter->inode, pos + ret, len - ret);
863 427878467 : return ret;
864 : }
865 :
866 245034927 : static loff_t iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i)
867 : {
868 245034927 : loff_t length = iomap_length(iter);
869 245034927 : size_t chunk = PAGE_SIZE << MAX_PAGECACHE_ORDER;
870 245034927 : loff_t pos = iter->pos;
871 245034927 : ssize_t written = 0;
872 245034927 : long status = 0;
873 245034927 : struct address_space *mapping = iter->inode->i_mapping;
874 245034927 : unsigned int bdp_flags = (iter->flags & IOMAP_NOWAIT) ? BDP_ASYNC : 0;
875 :
876 387883274 : do {
877 387883274 : struct folio *folio;
878 387883274 : size_t offset; /* Offset into folio */
879 387883274 : size_t bytes; /* Bytes to write to folio */
880 387883274 : size_t copied; /* Bytes copied from user */
881 :
882 387883274 : offset = pos & (chunk - 1);
883 387883274 : bytes = min(chunk - offset, iov_iter_count(i));
884 387883274 : status = balance_dirty_pages_ratelimited_flags(mapping,
885 : bdp_flags);
886 387924451 : if (unlikely(status))
887 : break;
888 :
889 387924451 : if (bytes > length)
890 : bytes = length;
891 :
892 : /*
893 : * Bring in the user page that we'll copy from _first_.
894 : * Otherwise there's a nasty deadlock on copying from the
895 : * same page as we're writing to, without it being marked
896 : * up-to-date.
897 : *
898 : * For async buffered writes the assumption is that the user
899 : * page has already been faulted in. This can be optimized by
900 : * faulting the user page.
901 : */
902 387924451 : if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) {
903 : status = -EFAULT;
904 : break;
905 : }
906 :
907 388253483 : status = iomap_write_begin(iter, pos, bytes, &folio);
908 388225191 : if (unlikely(status))
909 : break;
910 388224141 : if (iter->iomap.flags & IOMAP_F_STALE)
911 : break;
912 :
913 388114321 : offset = offset_in_folio(folio, pos);
914 388090671 : if (bytes > folio_size(folio) - offset)
915 138546644 : bytes = folio_size(folio) - offset;
916 :
917 387920898 : if (mapping_writably_mapped(mapping))
918 : flush_dcache_folio(folio);
919 :
920 387920898 : copied = copy_folio_from_iter_atomic(folio, offset, bytes, i);
921 388566744 : status = iomap_write_end(iter, pos, bytes, copied, folio);
922 :
923 388416187 : if (unlikely(copied != status))
924 0 : iov_iter_revert(i, copied - status);
925 :
926 388416187 : cond_resched();
927 388382742 : if (unlikely(status == 0)) {
928 : /*
929 : * A short copy made iomap_write_end() reject the
930 : * thing entirely. Might be memory poisoning
931 : * halfway through, might be a race with munmap,
932 : * might be severe memory pressure.
933 : */
934 0 : if (copied)
935 : bytes = copied;
936 0 : if (chunk > PAGE_SIZE)
937 0 : chunk /= 2;
938 : } else {
939 388382742 : pos += status;
940 388382742 : written += status;
941 388382742 : length -= status;
942 : }
943 388382742 : } while (iov_iter_count(i) && length);
944 :
945 245645265 : if (status == -EAGAIN) {
946 0 : iov_iter_revert(i, written);
947 0 : return -EAGAIN;
948 : }
949 245645265 : return written ? written : status;
950 : }
951 :
952 : ssize_t
953 236951500 : iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i,
954 : const struct iomap_ops *ops)
955 : {
956 236951500 : struct iomap_iter iter = {
957 236951500 : .inode = iocb->ki_filp->f_mapping->host,
958 236951500 : .pos = iocb->ki_pos,
959 : .len = iov_iter_count(i),
960 : .flags = IOMAP_WRITE,
961 : };
962 236951500 : ssize_t ret;
963 :
964 236951500 : if (iocb->ki_flags & IOCB_NOWAIT)
965 0 : iter.flags |= IOMAP_NOWAIT;
966 :
967 482550557 : while ((ret = iomap_iter(&iter, ops)) > 0)
968 245048924 : iter.processed = iomap_write_iter(&iter, i);
969 :
970 237134025 : if (unlikely(iter.pos == iocb->ki_pos))
971 : return ret;
972 234953964 : ret = iter.pos - iocb->ki_pos;
973 234953964 : iocb->ki_pos = iter.pos;
974 234953964 : return ret;
975 : }
976 : EXPORT_SYMBOL_GPL(iomap_file_buffered_write);
977 :
978 154 : static int iomap_write_delalloc_ifs_punch(struct inode *inode,
979 : struct folio *folio, loff_t start_byte, loff_t end_byte,
980 : iomap_punch_t punch)
981 : {
982 154 : unsigned int first_blk, last_blk, i;
983 154 : loff_t last_byte;
984 154 : u8 blkbits = inode->i_blkbits;
985 154 : struct iomap_folio_state *ifs;
986 154 : int ret = 0;
987 :
988 : /*
989 : * When we have per-block dirty tracking, there can be
990 : * blocks within a folio which are marked uptodate
991 : * but not dirty. In that case it is necessary to punch
992 : * out such blocks to avoid leaking any delalloc blocks.
993 : */
994 154 : ifs = folio->private;
995 154 : if (!ifs)
996 : return ret;
997 :
998 116 : last_byte = min_t(loff_t, end_byte - 1,
999 : folio_pos(folio) + folio_size(folio) - 1);
1000 116 : first_blk = offset_in_folio(folio, start_byte) >> blkbits;
1001 116 : last_blk = offset_in_folio(folio, last_byte) >> blkbits;
1002 234 : for (i = first_blk; i <= last_blk; i++) {
1003 118 : if (!ifs_block_is_dirty(folio, ifs, i)) {
1004 10 : ret = punch(inode, folio_pos(folio) + (i << blkbits),
1005 5 : 1 << blkbits);
1006 5 : if (ret)
1007 0 : return ret;
1008 : }
1009 : }
1010 :
1011 : return ret;
1012 : }
1013 :
1014 :
1015 27457 : static int iomap_write_delalloc_punch(struct inode *inode, struct folio *folio,
1016 : loff_t *punch_start_byte, loff_t start_byte, loff_t end_byte,
1017 : iomap_punch_t punch)
1018 : {
1019 27457 : int ret = 0;
1020 :
1021 27457 : if (!folio_test_dirty(folio))
1022 : return ret;
1023 :
1024 : /* if dirty, punch up to offset */
1025 154 : if (start_byte > *punch_start_byte) {
1026 0 : ret = punch(inode, *punch_start_byte,
1027 : start_byte - *punch_start_byte);
1028 0 : if (ret)
1029 : return ret;
1030 : }
1031 :
1032 : /* Punch non-dirty blocks within folio */
1033 154 : ret = iomap_write_delalloc_ifs_punch(inode, folio, start_byte,
1034 : end_byte, punch);
1035 154 : if (ret)
1036 : return ret;
1037 :
1038 : /*
1039 : * Make sure the next punch start is correctly bound to
1040 : * the end of this data range, not the end of the folio.
1041 : */
1042 154 : *punch_start_byte = min_t(loff_t, end_byte,
1043 : folio_pos(folio) + folio_size(folio));
1044 :
1045 154 : return ret;
1046 : }
1047 :
1048 : /*
1049 : * Scan the data range passed to us for dirty page cache folios. If we find a
1050 : * dirty folio, punch out the preceeding range and update the offset from which
1051 : * the next punch will start from.
1052 : *
1053 : * We can punch out storage reservations under clean pages because they either
1054 : * contain data that has been written back - in which case the delalloc punch
1055 : * over that range is a no-op - or they have been read faults in which case they
1056 : * contain zeroes and we can remove the delalloc backing range and any new
1057 : * writes to those pages will do the normal hole filling operation...
1058 : *
1059 : * This makes the logic simple: we only need to keep the delalloc extents only
1060 : * over the dirty ranges of the page cache.
1061 : *
1062 : * This function uses [start_byte, end_byte) intervals (i.e. open ended) to
1063 : * simplify range iterations.
1064 : */
1065 3611 : static int iomap_write_delalloc_scan(struct inode *inode,
1066 : loff_t *punch_start_byte, loff_t start_byte, loff_t end_byte,
1067 : iomap_punch_t punch)
1068 : {
1069 31229 : while (start_byte < end_byte) {
1070 27618 : struct folio *folio;
1071 27618 : int ret;
1072 :
1073 : /* grab locked page */
1074 27618 : folio = filemap_lock_folio(inode->i_mapping,
1075 27618 : start_byte >> PAGE_SHIFT);
1076 27618 : if (IS_ERR(folio)) {
1077 161 : start_byte = ALIGN_DOWN(start_byte, PAGE_SIZE) +
1078 : PAGE_SIZE;
1079 161 : continue;
1080 : }
1081 :
1082 27457 : ret = iomap_write_delalloc_punch(inode, folio, punch_start_byte,
1083 : start_byte, end_byte, punch);
1084 27457 : if (ret) {
1085 0 : folio_unlock(folio);
1086 0 : folio_put(folio);
1087 0 : return ret;
1088 : }
1089 :
1090 : /* move offset to start of next folio in range */
1091 27457 : start_byte = folio_next_index(folio) << PAGE_SHIFT;
1092 27457 : folio_unlock(folio);
1093 27457 : folio_put(folio);
1094 : }
1095 : return 0;
1096 : }
1097 :
1098 : /*
1099 : * Punch out all the delalloc blocks in the range given except for those that
1100 : * have dirty data still pending in the page cache - those are going to be
1101 : * written and so must still retain the delalloc backing for writeback.
1102 : *
1103 : * As we are scanning the page cache for data, we don't need to reimplement the
1104 : * wheel - mapping_seek_hole_data() does exactly what we need to identify the
1105 : * start and end of data ranges correctly even for sub-folio block sizes. This
1106 : * byte range based iteration is especially convenient because it means we
1107 : * don't have to care about variable size folios, nor where the start or end of
1108 : * the data range lies within a folio, if they lie within the same folio or even
1109 : * if there are multiple discontiguous data ranges within the folio.
1110 : *
1111 : * It should be noted that mapping_seek_hole_data() is not aware of EOF, and so
1112 : * can return data ranges that exist in the cache beyond EOF. e.g. a page fault
1113 : * spanning EOF will initialise the post-EOF data to zeroes and mark it up to
1114 : * date. A write page fault can then mark it dirty. If we then fail a write()
1115 : * beyond EOF into that up to date cached range, we allocate a delalloc block
1116 : * beyond EOF and then have to punch it out. Because the range is up to date,
1117 : * mapping_seek_hole_data() will return it, and we will skip the punch because
1118 : * the folio is dirty. THis is incorrect - we always need to punch out delalloc
1119 : * beyond EOF in this case as writeback will never write back and covert that
1120 : * delalloc block beyond EOF. Hence we limit the cached data scan range to EOF,
1121 : * resulting in always punching out the range from the EOF to the end of the
1122 : * range the iomap spans.
1123 : *
1124 : * Intervals are of the form [start_byte, end_byte) (i.e. open ended) because it
1125 : * matches the intervals returned by mapping_seek_hole_data(). i.e. SEEK_DATA
1126 : * returns the start of a data range (start_byte), and SEEK_HOLE(start_byte)
1127 : * returns the end of the data range (data_end). Using closed intervals would
1128 : * require sprinkling this code with magic "+ 1" and "- 1" arithmetic and expose
1129 : * the code to subtle off-by-one bugs....
1130 : */
1131 38192 : static int iomap_write_delalloc_release(struct inode *inode,
1132 : loff_t start_byte, loff_t end_byte, iomap_punch_t punch)
1133 : {
1134 38192 : loff_t punch_start_byte = start_byte;
1135 38192 : loff_t scan_end_byte = min(i_size_read(inode), end_byte);
1136 38192 : int error = 0;
1137 :
1138 : /*
1139 : * Lock the mapping to avoid races with page faults re-instantiating
1140 : * folios and dirtying them via ->page_mkwrite whilst we walk the
1141 : * cache and perform delalloc extent removal. Failing to do this can
1142 : * leave dirty pages with no space reservation in the cache.
1143 : */
1144 38192 : filemap_invalidate_lock(inode->i_mapping);
1145 41803 : while (start_byte < scan_end_byte) {
1146 32460 : loff_t data_end;
1147 :
1148 32460 : start_byte = mapping_seek_hole_data(inode->i_mapping,
1149 : start_byte, scan_end_byte, SEEK_DATA);
1150 : /*
1151 : * If there is no more data to scan, all that is left is to
1152 : * punch out the remaining range.
1153 : */
1154 32461 : if (start_byte == -ENXIO || start_byte == scan_end_byte)
1155 : break;
1156 3611 : if (start_byte < 0) {
1157 0 : error = start_byte;
1158 0 : goto out_unlock;
1159 : }
1160 3611 : WARN_ON_ONCE(start_byte < punch_start_byte);
1161 3611 : WARN_ON_ONCE(start_byte > scan_end_byte);
1162 :
1163 : /*
1164 : * We find the end of this contiguous cached data range by
1165 : * seeking from start_byte to the beginning of the next hole.
1166 : */
1167 3611 : data_end = mapping_seek_hole_data(inode->i_mapping, start_byte,
1168 : scan_end_byte, SEEK_HOLE);
1169 3611 : if (data_end < 0) {
1170 0 : error = data_end;
1171 0 : goto out_unlock;
1172 : }
1173 3611 : WARN_ON_ONCE(data_end <= start_byte);
1174 3611 : WARN_ON_ONCE(data_end > scan_end_byte);
1175 :
1176 3611 : error = iomap_write_delalloc_scan(inode, &punch_start_byte,
1177 : start_byte, data_end, punch);
1178 3611 : if (error)
1179 0 : goto out_unlock;
1180 :
1181 : /* The next data search starts at the end of this one. */
1182 : start_byte = data_end;
1183 : }
1184 :
1185 38193 : if (punch_start_byte < end_byte)
1186 38041 : error = punch(inode, punch_start_byte,
1187 : end_byte - punch_start_byte);
1188 152 : out_unlock:
1189 38193 : filemap_invalidate_unlock(inode->i_mapping);
1190 38193 : return error;
1191 : }
1192 :
1193 : /*
1194 : * When a short write occurs, the filesystem may need to remove reserved space
1195 : * that was allocated in ->iomap_begin from it's ->iomap_end method. For
1196 : * filesystems that use delayed allocation, we need to punch out delalloc
1197 : * extents from the range that are not dirty in the page cache. As the write can
1198 : * race with page faults, there can be dirty pages over the delalloc extent
1199 : * outside the range of a short write but still within the delalloc extent
1200 : * allocated for this iomap.
1201 : *
1202 : * This function uses [start_byte, end_byte) intervals (i.e. open ended) to
1203 : * simplify range iterations.
1204 : *
1205 : * The punch() callback *must* only punch delalloc extents in the range passed
1206 : * to it. It must skip over all other types of extents in the range and leave
1207 : * them completely unchanged. It must do this punch atomically with respect to
1208 : * other extent modifications.
1209 : *
1210 : * The punch() callback may be called with a folio locked to prevent writeback
1211 : * extent allocation racing at the edge of the range we are currently punching.
1212 : * The locked folio may or may not cover the range being punched, so it is not
1213 : * safe for the punch() callback to lock folios itself.
1214 : *
1215 : * Lock order is:
1216 : *
1217 : * inode->i_rwsem (shared or exclusive)
1218 : * inode->i_mapping->invalidate_lock (exclusive)
1219 : * folio_lock()
1220 : * ->punch
1221 : * internal filesystem allocation lock
1222 : */
1223 385850474 : int iomap_file_buffered_write_punch_delalloc(struct inode *inode,
1224 : struct iomap *iomap, loff_t pos, loff_t length,
1225 : ssize_t written, iomap_punch_t punch)
1226 : {
1227 385850474 : loff_t start_byte;
1228 385850474 : loff_t end_byte;
1229 385850474 : unsigned int blocksize = i_blocksize(inode);
1230 :
1231 385252954 : if (iomap->type != IOMAP_DELALLOC)
1232 : return 0;
1233 :
1234 : /* If we didn't reserve the blocks, we're not allowed to punch them. */
1235 162181697 : if (!(iomap->flags & IOMAP_F_NEW))
1236 : return 0;
1237 :
1238 : /*
1239 : * start_byte refers to the first unused block after a short write. If
1240 : * nothing was written, round offset down to point at the first block in
1241 : * the range.
1242 : */
1243 26599937 : if (unlikely(!written))
1244 23208 : start_byte = round_down(pos, blocksize);
1245 : else
1246 26576729 : start_byte = round_up(pos + written, blocksize);
1247 26599937 : end_byte = round_up(pos + length, blocksize);
1248 :
1249 : /* Nothing to do if we've written the entire delalloc extent */
1250 26599937 : if (start_byte >= end_byte)
1251 : return 0;
1252 :
1253 38191 : return iomap_write_delalloc_release(inode, start_byte, end_byte,
1254 : punch);
1255 : }
1256 : EXPORT_SYMBOL_GPL(iomap_file_buffered_write_punch_delalloc);
1257 :
1258 65760 : static loff_t iomap_unshare_iter(struct iomap_iter *iter)
1259 : {
1260 65760 : struct iomap *iomap = &iter->iomap;
1261 65760 : const struct iomap *srcmap = iomap_iter_srcmap(iter);
1262 65760 : loff_t pos = iter->pos;
1263 65760 : loff_t length = iomap_length(iter);
1264 65760 : long status = 0;
1265 65760 : loff_t written = 0;
1266 :
1267 : /* don't bother with blocks that are not shared to start with */
1268 65760 : if (!(iomap->flags & IOMAP_F_SHARED))
1269 : return length;
1270 : /* don't bother with holes or unwritten extents */
1271 31790 : if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
1272 : return length;
1273 :
1274 85236 : do {
1275 85236 : unsigned long offset = offset_in_page(pos);
1276 85236 : unsigned long bytes = min_t(loff_t, PAGE_SIZE - offset, length);
1277 85236 : struct folio *folio;
1278 :
1279 85236 : status = iomap_write_begin(iter, pos, bytes, &folio);
1280 85236 : if (unlikely(status))
1281 0 : return status;
1282 85236 : if (iter->iomap.flags & IOMAP_F_STALE)
1283 : break;
1284 :
1285 85236 : status = iomap_write_end(iter, pos, bytes, bytes, folio);
1286 85236 : if (WARN_ON_ONCE(status == 0))
1287 : return -EIO;
1288 :
1289 85236 : cond_resched();
1290 :
1291 85236 : pos += status;
1292 85236 : written += status;
1293 85236 : length -= status;
1294 :
1295 85236 : balance_dirty_pages_ratelimited(iter->inode->i_mapping);
1296 85236 : } while (length);
1297 :
1298 : return written;
1299 : }
1300 :
1301 : int
1302 116 : iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len,
1303 : const struct iomap_ops *ops)
1304 : {
1305 116 : struct iomap_iter iter = {
1306 : .inode = inode,
1307 : .pos = pos,
1308 : .len = len,
1309 : .flags = IOMAP_WRITE | IOMAP_UNSHARE,
1310 : };
1311 116 : int ret;
1312 :
1313 65876 : while ((ret = iomap_iter(&iter, ops)) > 0)
1314 65760 : iter.processed = iomap_unshare_iter(&iter);
1315 116 : return ret;
1316 : }
1317 : EXPORT_SYMBOL_GPL(iomap_file_unshare);
1318 :
1319 140455706 : static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero)
1320 : {
1321 140455706 : const struct iomap *srcmap = iomap_iter_srcmap(iter);
1322 140455706 : loff_t pos = iter->pos;
1323 140455706 : loff_t length = iomap_length(iter);
1324 140455706 : loff_t written = 0;
1325 :
1326 : /* already zeroed? we're done. */
1327 140455706 : if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
1328 : return length;
1329 :
1330 39195372 : do {
1331 39195372 : struct folio *folio;
1332 39195372 : int status;
1333 39195372 : size_t offset;
1334 39195372 : size_t bytes = min_t(u64, SIZE_MAX, length);
1335 :
1336 39195372 : status = iomap_write_begin(iter, pos, bytes, &folio);
1337 39239766 : if (status)
1338 1325 : return status;
1339 39238441 : if (iter->iomap.flags & IOMAP_F_STALE)
1340 : break;
1341 :
1342 39237206 : offset = offset_in_folio(folio, pos);
1343 39263079 : if (bytes > folio_size(folio) - offset)
1344 2413678 : bytes = folio_size(folio) - offset;
1345 :
1346 39262472 : folio_zero_range(folio, offset, bytes);
1347 39051236 : folio_mark_accessed(folio);
1348 :
1349 39536300 : bytes = iomap_write_end(iter, pos, bytes, bytes, folio);
1350 39386068 : if (WARN_ON_ONCE(bytes == 0))
1351 : return -EIO;
1352 :
1353 39386068 : pos += bytes;
1354 39386068 : length -= bytes;
1355 39386068 : written += bytes;
1356 39386068 : } while (length > 0);
1357 :
1358 36967543 : if (did_zero)
1359 5498119 : *did_zero = true;
1360 : return written;
1361 : }
1362 :
1363 : int
1364 122779081 : iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
1365 : const struct iomap_ops *ops)
1366 : {
1367 122779081 : struct iomap_iter iter = {
1368 : .inode = inode,
1369 : .pos = pos,
1370 : .len = len,
1371 : .flags = IOMAP_ZERO,
1372 : };
1373 122779081 : int ret;
1374 :
1375 263422216 : while ((ret = iomap_iter(&iter, ops)) > 0)
1376 140486056 : iter.processed = iomap_zero_iter(&iter, did_zero);
1377 122572616 : return ret;
1378 : }
1379 : EXPORT_SYMBOL_GPL(iomap_zero_range);
1380 :
1381 : int
1382 6266344 : iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
1383 : const struct iomap_ops *ops)
1384 : {
1385 6266344 : unsigned int blocksize = i_blocksize(inode);
1386 6266299 : unsigned int off = pos & (blocksize - 1);
1387 :
1388 : /* Block boundary? Nothing to do */
1389 6266299 : if (!off)
1390 : return 0;
1391 4142535 : return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops);
1392 : }
1393 : EXPORT_SYMBOL_GPL(iomap_truncate_page);
1394 :
1395 80835098 : static loff_t iomap_folio_mkwrite_iter(struct iomap_iter *iter,
1396 : struct folio *folio)
1397 : {
1398 80835098 : loff_t length = iomap_length(iter);
1399 80835098 : int ret;
1400 :
1401 80835098 : if (iter->iomap.flags & IOMAP_F_BUFFER_HEAD) {
1402 0 : ret = __block_write_begin_int(folio, iter->pos, length, NULL,
1403 0 : &iter->iomap);
1404 0 : if (ret)
1405 0 : return ret;
1406 0 : block_commit_write(&folio->page, 0, length);
1407 : } else {
1408 80835098 : WARN_ON_ONCE(!folio_test_uptodate(folio));
1409 80820806 : folio_mark_dirty(folio);
1410 : }
1411 :
1412 : return length;
1413 : }
1414 :
1415 78503389 : vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops)
1416 : {
1417 78503389 : struct iomap_iter iter = {
1418 78503389 : .inode = file_inode(vmf->vma->vm_file),
1419 : .flags = IOMAP_WRITE | IOMAP_FAULT,
1420 : };
1421 78503389 : struct folio *folio = page_folio(vmf->page);
1422 78106953 : ssize_t ret;
1423 :
1424 78106953 : folio_lock(folio);
1425 78478325 : ret = folio_mkwrite_check_truncate(folio, iter.inode);
1426 78381697 : if (ret < 0)
1427 13291 : goto out_unlock;
1428 78368406 : iter.pos = folio_pos(folio);
1429 78368406 : iter.len = ret;
1430 159404270 : while ((ret = iomap_iter(&iter, ops)) > 0)
1431 80827401 : iter.processed = iomap_folio_mkwrite_iter(&iter, folio);
1432 :
1433 78325397 : if (ret < 0)
1434 45062 : goto out_unlock;
1435 78280335 : folio_wait_stable(folio);
1436 78280335 : return VM_FAULT_LOCKED;
1437 58353 : out_unlock:
1438 58353 : folio_unlock(folio);
1439 58354 : return block_page_mkwrite_return(ret);
1440 : }
1441 : EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
1442 :
1443 263377180 : static void iomap_finish_folio_write(struct inode *inode, struct folio *folio,
1444 : size_t len, int error)
1445 : {
1446 263377180 : struct iomap_folio_state *ifs = folio->private;
1447 :
1448 263377180 : if (error) {
1449 1294858 : folio_set_error(folio);
1450 1294859 : mapping_set_error(inode->i_mapping, error);
1451 : }
1452 :
1453 526754364 : WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !ifs);
1454 311964632 : WARN_ON_ONCE(ifs && atomic_read(&ifs->write_bytes_pending) <= 0);
1455 :
1456 263377181 : if (!ifs || atomic_sub_and_test(len, &ifs->write_bytes_pending))
1457 261833302 : folio_end_writeback(folio);
1458 263377183 : }
1459 :
1460 : /*
1461 : * We're now finished for good with this ioend structure. Update the page
1462 : * state, release holds on bios, and finally free up memory. Do not use the
1463 : * ioend after this.
1464 : */
1465 : static u32
1466 58929649 : iomap_finish_ioend(struct iomap_ioend *ioend, int error)
1467 : {
1468 58929649 : struct inode *inode = ioend->io_inode;
1469 58929649 : struct bio *bio = &ioend->io_inline_bio;
1470 58929649 : struct bio *last = ioend->io_bio, *next;
1471 58929649 : u64 start = bio->bi_iter.bi_sector;
1472 58929649 : loff_t offset = ioend->io_offset;
1473 58929649 : bool quiet = bio_flagged(bio, BIO_QUIET);
1474 58929649 : u32 folio_count = 0;
1475 :
1476 117895498 : for (bio = &ioend->io_inline_bio; bio; bio = next) {
1477 58965848 : struct folio_iter fi;
1478 :
1479 : /*
1480 : * For the last bio, bi_private points to the ioend, so we
1481 : * need to explicitly end the iteration here.
1482 : */
1483 58965848 : if (bio == last)
1484 : next = NULL;
1485 : else
1486 36199 : next = bio->bi_private;
1487 :
1488 : /* walk all folios in bio, ending page IO on them */
1489 322343028 : bio_for_each_folio_all(fi, bio) {
1490 263377180 : iomap_finish_folio_write(inode, fi.folio, fi.length,
1491 : error);
1492 263377183 : folio_count++;
1493 : }
1494 58965849 : bio_put(bio);
1495 : }
1496 : /* The ioend has been freed by bio_put() */
1497 :
1498 58929650 : if (unlikely(error && !quiet)) {
1499 53198 : printk_ratelimited(KERN_ERR
1500 : "%s: writeback error on inode %lu, offset %lld, sector %llu",
1501 : inode->i_sb->s_id, inode->i_ino, offset, start);
1502 : }
1503 58929650 : return folio_count;
1504 : }
1505 :
1506 : /*
1507 : * Ioend completion routine for merged bios. This can only be called from task
1508 : * contexts as merged ioends can be of unbound length. Hence we have to break up
1509 : * the writeback completions into manageable chunks to avoid long scheduler
1510 : * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
1511 : * good batch processing throughput without creating adverse scheduler latency
1512 : * conditions.
1513 : */
1514 : void
1515 51170711 : iomap_finish_ioends(struct iomap_ioend *ioend, int error)
1516 : {
1517 51170711 : struct list_head tmp;
1518 51170711 : u32 completions;
1519 :
1520 51170711 : might_sleep();
1521 :
1522 51170713 : list_replace_init(&ioend->io_list, &tmp);
1523 51170713 : completions = iomap_finish_ioend(ioend, error);
1524 :
1525 51263232 : while (!list_empty(&tmp)) {
1526 92519 : if (completions > IOEND_BATCH_SIZE * 8) {
1527 0 : cond_resched();
1528 0 : completions = 0;
1529 : }
1530 92519 : ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
1531 92519 : list_del_init(&ioend->io_list);
1532 92519 : completions += iomap_finish_ioend(ioend, error);
1533 : }
1534 51170713 : }
1535 : EXPORT_SYMBOL_GPL(iomap_finish_ioends);
1536 :
1537 : /*
1538 : * We can merge two adjacent ioends if they have the same set of work to do.
1539 : */
1540 : static bool
1541 8831727 : iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next)
1542 : {
1543 8831727 : if (ioend->io_bio->bi_status != next->io_bio->bi_status)
1544 : return false;
1545 8831707 : if ((ioend->io_flags & IOMAP_F_SHARED) ^
1546 8831707 : (next->io_flags & IOMAP_F_SHARED))
1547 : return false;
1548 8744206 : if ((ioend->io_type == IOMAP_UNWRITTEN) ^
1549 8744206 : (next->io_type == IOMAP_UNWRITTEN))
1550 : return false;
1551 7698694 : if (ioend->io_offset + ioend->io_size != next->io_offset)
1552 : return false;
1553 : /*
1554 : * Do not merge physically discontiguous ioends. The filesystem
1555 : * completion functions will have to iterate the physical
1556 : * discontiguities even if we merge the ioends at a logical level, so
1557 : * we don't gain anything by merging physical discontiguities here.
1558 : *
1559 : * We cannot use bio->bi_iter.bi_sector here as it is modified during
1560 : * submission so does not point to the start sector of the bio at
1561 : * completion.
1562 : */
1563 2539181 : if (ioend->io_sector + (ioend->io_size >> 9) != next->io_sector)
1564 2446662 : return false;
1565 : return true;
1566 : }
1567 :
1568 : void
1569 51170713 : iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends)
1570 : {
1571 51170713 : struct iomap_ioend *next;
1572 :
1573 51170713 : INIT_LIST_HEAD(&ioend->io_list);
1574 :
1575 51263231 : while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
1576 : io_list))) {
1577 8831727 : if (!iomap_ioend_can_merge(ioend, next))
1578 : break;
1579 92519 : list_move_tail(&next->io_list, &ioend->io_list);
1580 92518 : ioend->io_size += next->io_size;
1581 : }
1582 51170712 : }
1583 : EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
1584 :
1585 : static int
1586 21762569 : iomap_ioend_compare(void *priv, const struct list_head *a,
1587 : const struct list_head *b)
1588 : {
1589 21762569 : struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
1590 21762569 : struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
1591 :
1592 21762569 : if (ia->io_offset < ib->io_offset)
1593 : return -1;
1594 9198898 : if (ia->io_offset > ib->io_offset)
1595 9197825 : return 1;
1596 : return 0;
1597 : }
1598 :
1599 : void
1600 42431505 : iomap_sort_ioends(struct list_head *ioend_list)
1601 : {
1602 42431505 : list_sort(NULL, ioend_list, iomap_ioend_compare);
1603 42431504 : }
1604 : EXPORT_SYMBOL_GPL(iomap_sort_ioends);
1605 :
1606 7666418 : static void iomap_writepage_end_bio(struct bio *bio)
1607 : {
1608 7666418 : struct iomap_ioend *ioend = bio->bi_private;
1609 :
1610 7666418 : iomap_finish_ioend(ioend, blk_status_to_errno(bio->bi_status));
1611 7666418 : }
1612 :
1613 : /*
1614 : * Submit the final bio for an ioend.
1615 : *
1616 : * If @error is non-zero, it means that we have a situation where some part of
1617 : * the submission process has failed after we've marked pages for writeback
1618 : * and unlocked them. In this situation, we need to fail the bio instead of
1619 : * submitting it. This typically only happens on a filesystem shutdown.
1620 : */
1621 : static int
1622 58929159 : iomap_submit_ioend(struct iomap_writepage_ctx *wpc, struct iomap_ioend *ioend,
1623 : int error)
1624 : {
1625 58929159 : ioend->io_bio->bi_private = ioend;
1626 58929159 : ioend->io_bio->bi_end_io = iomap_writepage_end_bio;
1627 :
1628 58929159 : if (wpc->ops->prepare_ioend)
1629 58929159 : error = wpc->ops->prepare_ioend(ioend, error);
1630 58929069 : if (error) {
1631 : /*
1632 : * If we're failing the IO now, just mark the ioend with an
1633 : * error and finish it. This will run IO completion immediately
1634 : * as there is only one reference to the ioend at this point in
1635 : * time.
1636 : */
1637 438 : ioend->io_bio->bi_status = errno_to_blk_status(error);
1638 438 : bio_endio(ioend->io_bio);
1639 438 : return error;
1640 : }
1641 :
1642 58928631 : submit_bio(ioend->io_bio);
1643 58928631 : return 0;
1644 : }
1645 :
1646 : static struct iomap_ioend *
1647 58929127 : iomap_alloc_ioend(struct inode *inode, struct iomap_writepage_ctx *wpc,
1648 : loff_t offset, sector_t sector, struct writeback_control *wbc)
1649 : {
1650 58929127 : struct iomap_ioend *ioend;
1651 58929127 : struct bio *bio;
1652 :
1653 92856585 : bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
1654 : REQ_OP_WRITE | wbc_to_write_flags(wbc),
1655 : GFP_NOFS, &iomap_ioend_bioset);
1656 58929346 : bio->bi_iter.bi_sector = sector;
1657 58929346 : wbc_init_bio(wbc, bio);
1658 :
1659 58929259 : ioend = container_of(bio, struct iomap_ioend, io_inline_bio);
1660 58929259 : INIT_LIST_HEAD(&ioend->io_list);
1661 58929259 : ioend->io_type = wpc->iomap.type;
1662 58929259 : ioend->io_flags = wpc->iomap.flags;
1663 58929259 : ioend->io_inode = inode;
1664 58929259 : ioend->io_size = 0;
1665 58929259 : ioend->io_folios = 0;
1666 58929259 : ioend->io_offset = offset;
1667 58929259 : ioend->io_bio = bio;
1668 58929259 : ioend->io_sector = sector;
1669 58929259 : return ioend;
1670 : }
1671 :
1672 : /*
1673 : * Allocate a new bio, and chain the old bio to the new one.
1674 : *
1675 : * Note that we have to perform the chaining in this unintuitive order
1676 : * so that the bi_private linkage is set up in the right direction for the
1677 : * traversal in iomap_finish_ioend().
1678 : */
1679 : static struct bio *
1680 36199 : iomap_chain_bio(struct bio *prev)
1681 : {
1682 36199 : struct bio *new;
1683 :
1684 36199 : new = bio_alloc(prev->bi_bdev, BIO_MAX_VECS, prev->bi_opf, GFP_NOFS);
1685 36199 : bio_clone_blkg_association(new, prev);
1686 36199 : new->bi_iter.bi_sector = bio_end_sector(prev);
1687 :
1688 36199 : bio_chain(prev, new);
1689 36199 : bio_get(prev); /* for iomap_finish_ioend */
1690 36199 : submit_bio(prev);
1691 36199 : return new;
1692 : }
1693 :
1694 : static bool
1695 608862840 : iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t offset,
1696 : sector_t sector)
1697 : {
1698 608862840 : if ((wpc->iomap.flags & IOMAP_F_SHARED) !=
1699 608862840 : (wpc->ioend->io_flags & IOMAP_F_SHARED))
1700 : return false;
1701 608423417 : if (wpc->iomap.type != wpc->ioend->io_type)
1702 : return false;
1703 600938493 : if (offset != wpc->ioend->io_offset + wpc->ioend->io_size)
1704 : return false;
1705 591956291 : if (sector != bio_end_sector(wpc->ioend->io_bio))
1706 : return false;
1707 : /*
1708 : * Limit ioend bio chain lengths to minimise IO completion latency. This
1709 : * also prevents long tight loops ending page writeback on all the
1710 : * folios in the ioend.
1711 : */
1712 585559025 : if (wpc->ioend->io_folios >= IOEND_BATCH_SIZE)
1713 6021 : return false;
1714 : return true;
1715 : }
1716 :
1717 : /*
1718 : * Test to see if we have an existing ioend structure that we could append to
1719 : * first; otherwise finish off the current ioend and start another.
1720 : */
1721 : static void
1722 644490607 : iomap_add_to_ioend(struct inode *inode, loff_t pos, struct folio *folio,
1723 : struct iomap_folio_state *ifs, struct iomap_writepage_ctx *wpc,
1724 : struct writeback_control *wbc, struct list_head *iolist)
1725 : {
1726 644490607 : sector_t sector = iomap_sector(&wpc->iomap, pos);
1727 644490607 : unsigned len = i_blocksize(inode);
1728 644489629 : size_t poff = offset_in_folio(folio, pos);
1729 :
1730 644483856 : if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, pos, sector)) {
1731 58928987 : if (wpc->ioend)
1732 23308658 : list_add(&wpc->ioend->io_list, iolist);
1733 58928946 : wpc->ioend = iomap_alloc_ioend(inode, wpc, pos, sector, wbc);
1734 : }
1735 :
1736 644483929 : if (!bio_add_folio(wpc->ioend->io_bio, folio, len, poff)) {
1737 36199 : wpc->ioend->io_bio = iomap_chain_bio(wpc->ioend->io_bio);
1738 36199 : bio_add_folio_nofail(wpc->ioend->io_bio, folio, len, poff);
1739 : }
1740 :
1741 644505389 : if (ifs)
1742 429718534 : atomic_add(len, &ifs->write_bytes_pending);
1743 644516088 : wpc->ioend->io_size += len;
1744 644516088 : wbc_account_cgroup_owner(wbc, &folio->page, len);
1745 644500745 : }
1746 :
1747 : /*
1748 : * We implement an immediate ioend submission policy here to avoid needing to
1749 : * chain multiple ioends and hence nest mempool allocations which can violate
1750 : * the forward progress guarantees we need to provide. The current ioend we're
1751 : * adding blocks to is cached in the writepage context, and if the new block
1752 : * doesn't append to the cached ioend, it will create a new ioend and cache that
1753 : * instead.
1754 : *
1755 : * If a new ioend is created and cached, the old ioend is returned and queued
1756 : * locally for submission once the entire page is processed or an error has been
1757 : * detected. While ioends are submitted immediately after they are completed,
1758 : * batching optimisations are provided by higher level block plugging.
1759 : *
1760 : * At the end of a writeback pass, there will be a cached ioend remaining on the
1761 : * writepage context that the caller will need to submit.
1762 : */
1763 : static int
1764 262480813 : iomap_writepage_map(struct iomap_writepage_ctx *wpc,
1765 : struct writeback_control *wbc, struct inode *inode,
1766 : struct folio *folio, u64 end_pos)
1767 : {
1768 262480813 : struct iomap_folio_state *ifs = folio->private;
1769 262480813 : struct iomap_ioend *ioend, *next;
1770 262480813 : unsigned len = i_blocksize(inode);
1771 262480657 : unsigned nblocks = i_blocks_per_folio(inode, folio);
1772 262479986 : u64 pos = folio_pos(folio);
1773 262479986 : int error = 0, count = 0, i;
1774 262479986 : LIST_HEAD(submit_list);
1775 :
1776 262479986 : WARN_ON_ONCE(end_pos <= pos);
1777 :
1778 262479986 : if (!ifs && nblocks > 1) {
1779 30837630 : ifs = ifs_alloc(inode, folio, 0);
1780 30837862 : iomap_set_range_dirty(folio, 0, end_pos - pos);
1781 : }
1782 :
1783 309674717 : WARN_ON_ONCE(ifs && atomic_read(&ifs->write_bytes_pending) != 0);
1784 :
1785 : /*
1786 : * Walk through the folio to find areas to write back. If we
1787 : * run off the end of the current map or find the current map
1788 : * invalid, grab a new one.
1789 : */
1790 913758176 : for (i = 0; i < nblocks && pos < end_pos; i++, pos += len) {
1791 651929159 : if (ifs && !ifs_block_is_dirty(folio, ifs, i))
1792 6778192 : continue;
1793 :
1794 645140778 : error = wpc->ops->map_blocks(wpc, inode, pos);
1795 645146100 : if (error)
1796 : break;
1797 644492727 : trace_iomap_writepage_map(inode, &wpc->iomap);
1798 644491703 : if (WARN_ON_ONCE(wpc->iomap.type == IOMAP_INLINE))
1799 0 : continue;
1800 644491703 : if (wpc->iomap.type == IOMAP_HOLE)
1801 291 : continue;
1802 644491412 : iomap_add_to_ioend(inode, pos, folio, ifs, wpc, wbc,
1803 : &submit_list);
1804 644499314 : count++;
1805 : }
1806 262482390 : if (count)
1807 261829030 : wpc->ioend->io_folios++;
1808 :
1809 263133568 : WARN_ON_ONCE(!wpc->ioend && !list_empty(&submit_list));
1810 262482390 : WARN_ON_ONCE(!folio_test_locked(folio));
1811 262482489 : WARN_ON_ONCE(folio_test_writeback(folio));
1812 262482487 : WARN_ON_ONCE(folio_test_dirty(folio));
1813 :
1814 : /*
1815 : * We cannot cancel the ioend directly here on error. We may have
1816 : * already set other pages under writeback and hence we have to run I/O
1817 : * completion to mark the error state of the pages under writeback
1818 : * appropriately.
1819 : */
1820 262482728 : if (unlikely(error)) {
1821 : /*
1822 : * Let the filesystem know what portion of the current page
1823 : * failed to map. If the page hasn't been added to ioend, it
1824 : * won't be affected by I/O completion and we must unlock it
1825 : * now.
1826 : */
1827 653373 : if (wpc->ops->discard_folio)
1828 653373 : wpc->ops->discard_folio(folio, pos);
1829 653373 : if (!count) {
1830 653360 : folio_unlock(folio);
1831 653360 : goto done;
1832 : }
1833 : }
1834 :
1835 : /*
1836 : * We can have dirty bits set past end of file in page_mkwrite path
1837 : * while mapping the last partial folio. Hence it's better to clear
1838 : * all the dirty bits in the folio here.
1839 : */
1840 261829368 : iomap_clear_range_dirty(folio, 0, folio_size(folio));
1841 261828509 : folio_start_writeback(folio);
1842 261829517 : folio_unlock(folio);
1843 :
1844 : /*
1845 : * Preserve the original error if there was one; catch
1846 : * submission errors here and propagate into subsequent ioend
1847 : * submissions.
1848 : */
1849 285139136 : list_for_each_entry_safe(ioend, next, &submit_list, io_list) {
1850 23308591 : int error2;
1851 :
1852 23308591 : list_del_init(&ioend->io_list);
1853 23308745 : error2 = iomap_submit_ioend(wpc, ioend, error);
1854 23308829 : if (error2 && !error)
1855 0 : error = error2;
1856 : }
1857 :
1858 : /*
1859 : * We can end up here with no error and nothing to write only if we race
1860 : * with a partial page truncate on a sub-page block sized filesystem.
1861 : */
1862 261830545 : if (!count)
1863 0 : folio_end_writeback(folio);
1864 261830545 : done:
1865 262483905 : mapping_set_error(inode->i_mapping, error);
1866 262483332 : return error;
1867 : }
1868 :
1869 : /*
1870 : * Write out a dirty page.
1871 : *
1872 : * For delalloc space on the page, we need to allocate space and flush it.
1873 : * For unwritten space on the page, we need to start the conversion to
1874 : * regular allocated space.
1875 : */
1876 262488109 : static int iomap_do_writepage(struct folio *folio,
1877 : struct writeback_control *wbc, void *data)
1878 : {
1879 262488109 : struct iomap_writepage_ctx *wpc = data;
1880 262488109 : struct inode *inode = folio->mapping->host;
1881 262488109 : u64 end_pos, isize;
1882 :
1883 262488109 : trace_iomap_writepage(inode, folio_pos(folio), folio_size(folio));
1884 :
1885 : /*
1886 : * Refuse to write the folio out if we're called from reclaim context.
1887 : *
1888 : * This avoids stack overflows when called from deeply used stacks in
1889 : * random callers for direct reclaim or memcg reclaim. We explicitly
1890 : * allow reclaim from kswapd as the stack usage there is relatively low.
1891 : *
1892 : * This should never happen except in the case of a VM regression so
1893 : * warn about it.
1894 : */
1895 262487257 : if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
1896 : PF_MEMALLOC))
1897 0 : goto redirty;
1898 :
1899 : /*
1900 : * Is this folio beyond the end of the file?
1901 : *
1902 : * The folio index is less than the end_index, adjust the end_pos
1903 : * to the highest offset that this folio should represent.
1904 : * -----------------------------------------------------
1905 : * | file mapping | <EOF> |
1906 : * -----------------------------------------------------
1907 : * | Page ... | Page N-2 | Page N-1 | Page N | |
1908 : * ^--------------------------------^----------|--------
1909 : * | desired writeback range | see else |
1910 : * ---------------------------------^------------------|
1911 : */
1912 262487257 : isize = i_size_read(inode);
1913 262487257 : end_pos = folio_pos(folio) + folio_size(folio);
1914 262486656 : if (end_pos > isize) {
1915 : /*
1916 : * Check whether the page to write out is beyond or straddles
1917 : * i_size or not.
1918 : * -------------------------------------------------------
1919 : * | file mapping | <EOF> |
1920 : * -------------------------------------------------------
1921 : * | Page ... | Page N-2 | Page N-1 | Page N | Beyond |
1922 : * ^--------------------------------^-----------|---------
1923 : * | | Straddles |
1924 : * ---------------------------------^-----------|--------|
1925 : */
1926 15149032 : size_t poff = offset_in_folio(folio, isize);
1927 15149032 : pgoff_t end_index = isize >> PAGE_SHIFT;
1928 :
1929 : /*
1930 : * Skip the page if it's fully outside i_size, e.g.
1931 : * due to a truncate operation that's in progress. We've
1932 : * cleaned this page and truncate will finish things off for
1933 : * us.
1934 : *
1935 : * Note that the end_index is unsigned long. If the given
1936 : * offset is greater than 16TB on a 32-bit system then if we
1937 : * checked if the page is fully outside i_size with
1938 : * "if (page->index >= end_index + 1)", "end_index + 1" would
1939 : * overflow and evaluate to 0. Hence this page would be
1940 : * redirtied and written out repeatedly, which would result in
1941 : * an infinite loop; the user program performing this operation
1942 : * would hang. Instead, we can detect this situation by
1943 : * checking if the page is totally beyond i_size or if its
1944 : * offset is just equal to the EOF.
1945 : */
1946 15149032 : if (folio->index > end_index ||
1947 15042276 : (folio->index == end_index && poff == 0))
1948 6103 : goto unlock;
1949 :
1950 : /*
1951 : * The page straddles i_size. It must be zeroed out on each
1952 : * and every writepage invocation because it may be mmapped.
1953 : * "A file is mapped in multiples of the page size. For a file
1954 : * that is not a multiple of the page size, the remaining
1955 : * memory is zeroed when mapped, and writes to that region are
1956 : * not written out to the file."
1957 : */
1958 15142929 : folio_zero_segment(folio, poff, folio_size(folio));
1959 15142927 : end_pos = isize;
1960 : }
1961 :
1962 262480562 : return iomap_writepage_map(wpc, wbc, inode, folio, end_pos);
1963 :
1964 : redirty:
1965 0 : folio_redirty_for_writepage(wbc, folio);
1966 6103 : unlock:
1967 6103 : folio_unlock(folio);
1968 6103 : return 0;
1969 : }
1970 :
1971 : int
1972 64960048 : iomap_writepages(struct address_space *mapping, struct writeback_control *wbc,
1973 : struct iomap_writepage_ctx *wpc,
1974 : const struct iomap_writeback_ops *ops)
1975 : {
1976 64960048 : int ret;
1977 :
1978 64960048 : wpc->ops = ops;
1979 64960048 : ret = write_cache_pages(mapping, wbc, iomap_do_writepage, wpc);
1980 64958503 : if (!wpc->ioend)
1981 : return ret;
1982 35620574 : return iomap_submit_ioend(wpc, wpc->ioend, ret);
1983 : }
1984 : EXPORT_SYMBOL_GPL(iomap_writepages);
1985 :
1986 0 : static int __init iomap_init(void)
1987 : {
1988 0 : return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
1989 : offsetof(struct iomap_ioend, io_inline_bio),
1990 : BIOSET_NEED_BVECS);
1991 : }
1992 : fs_initcall(iomap_init);
|
