Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * linux/fs/buffer.c
4 : *
5 : * Copyright (C) 1991, 1992, 2002 Linus Torvalds
6 : */
7 :
8 : /*
9 : * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10 : *
11 : * Removed a lot of unnecessary code and simplified things now that
12 : * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13 : *
14 : * Speed up hash, lru, and free list operations. Use gfp() for allocating
15 : * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16 : *
17 : * Added 32k buffer block sizes - these are required older ARM systems. - RMK
18 : *
19 : * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
20 : */
21 :
22 : #include <linux/kernel.h>
23 : #include <linux/sched/signal.h>
24 : #include <linux/syscalls.h>
25 : #include <linux/fs.h>
26 : #include <linux/iomap.h>
27 : #include <linux/mm.h>
28 : #include <linux/percpu.h>
29 : #include <linux/slab.h>
30 : #include <linux/capability.h>
31 : #include <linux/blkdev.h>
32 : #include <linux/file.h>
33 : #include <linux/quotaops.h>
34 : #include <linux/highmem.h>
35 : #include <linux/export.h>
36 : #include <linux/backing-dev.h>
37 : #include <linux/writeback.h>
38 : #include <linux/hash.h>
39 : #include <linux/suspend.h>
40 : #include <linux/buffer_head.h>
41 : #include <linux/task_io_accounting_ops.h>
42 : #include <linux/bio.h>
43 : #include <linux/cpu.h>
44 : #include <linux/bitops.h>
45 : #include <linux/mpage.h>
46 : #include <linux/bit_spinlock.h>
47 : #include <linux/pagevec.h>
48 : #include <linux/sched/mm.h>
49 : #include <trace/events/block.h>
50 : #include <linux/fscrypt.h>
51 : #include <linux/fsverity.h>
52 :
53 : #include "internal.h"
54 :
55 : static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
56 : static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
57 : struct writeback_control *wbc);
58 :
59 : #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
60 :
61 161545 : inline void touch_buffer(struct buffer_head *bh)
62 : {
63 161545 : trace_block_touch_buffer(bh);
64 161545 : folio_mark_accessed(bh->b_folio);
65 161545 : }
66 : EXPORT_SYMBOL(touch_buffer);
67 :
68 0 : void __lock_buffer(struct buffer_head *bh)
69 : {
70 7 : wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
71 7 : }
72 : EXPORT_SYMBOL(__lock_buffer);
73 :
74 601284 : void unlock_buffer(struct buffer_head *bh)
75 : {
76 601284 : clear_bit_unlock(BH_Lock, &bh->b_state);
77 601283 : smp_mb__after_atomic();
78 601285 : wake_up_bit(&bh->b_state, BH_Lock);
79 601286 : }
80 : EXPORT_SYMBOL(unlock_buffer);
81 :
82 : /*
83 : * Returns if the folio has dirty or writeback buffers. If all the buffers
84 : * are unlocked and clean then the folio_test_dirty information is stale. If
85 : * any of the buffers are locked, it is assumed they are locked for IO.
86 : */
87 39 : void buffer_check_dirty_writeback(struct folio *folio,
88 : bool *dirty, bool *writeback)
89 : {
90 39 : struct buffer_head *head, *bh;
91 39 : *dirty = false;
92 39 : *writeback = false;
93 :
94 39 : BUG_ON(!folio_test_locked(folio));
95 :
96 39 : head = folio_buffers(folio);
97 39 : if (!head)
98 : return;
99 :
100 39 : if (folio_test_writeback(folio))
101 0 : *writeback = true;
102 :
103 : bh = head;
104 39 : do {
105 78 : if (buffer_locked(bh))
106 0 : *writeback = true;
107 :
108 78 : if (buffer_dirty(bh))
109 0 : *dirty = true;
110 :
111 39 : bh = bh->b_this_page;
112 39 : } while (bh != head);
113 : }
114 :
115 : /*
116 : * Block until a buffer comes unlocked. This doesn't stop it
117 : * from becoming locked again - you have to lock it yourself
118 : * if you want to preserve its state.
119 : */
120 2 : void __wait_on_buffer(struct buffer_head * bh)
121 : {
122 110279 : wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
123 110277 : }
124 : EXPORT_SYMBOL(__wait_on_buffer);
125 :
126 89863 : static void buffer_io_error(struct buffer_head *bh, char *msg)
127 : {
128 179726 : if (!test_bit(BH_Quiet, &bh->b_state))
129 89863 : printk_ratelimited(KERN_ERR
130 : "Buffer I/O error on dev %pg, logical block %llu%s\n",
131 : bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
132 89863 : }
133 :
134 : /*
135 : * End-of-IO handler helper function which does not touch the bh after
136 : * unlocking it.
137 : * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
138 : * a race there is benign: unlock_buffer() only use the bh's address for
139 : * hashing after unlocking the buffer, so it doesn't actually touch the bh
140 : * itself.
141 : */
142 109515 : static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
143 : {
144 109515 : if (uptodate) {
145 109515 : set_buffer_uptodate(bh);
146 : } else {
147 : /* This happens, due to failed read-ahead attempts. */
148 0 : clear_buffer_uptodate(bh);
149 : }
150 109515 : unlock_buffer(bh);
151 109515 : }
152 :
153 : /*
154 : * Default synchronous end-of-IO handler.. Just mark it up-to-date and
155 : * unlock the buffer.
156 : */
157 109515 : void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
158 : {
159 109515 : __end_buffer_read_notouch(bh, uptodate);
160 109515 : put_bh(bh);
161 109515 : }
162 : EXPORT_SYMBOL(end_buffer_read_sync);
163 :
164 884 : void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
165 : {
166 884 : if (uptodate) {
167 884 : set_buffer_uptodate(bh);
168 : } else {
169 0 : buffer_io_error(bh, ", lost sync page write");
170 0 : mark_buffer_write_io_error(bh);
171 0 : clear_buffer_uptodate(bh);
172 : }
173 884 : unlock_buffer(bh);
174 884 : put_bh(bh);
175 884 : }
176 : EXPORT_SYMBOL(end_buffer_write_sync);
177 :
178 : /*
179 : * Various filesystems appear to want __find_get_block to be non-blocking.
180 : * But it's the page lock which protects the buffers. To get around this,
181 : * we get exclusion from try_to_free_buffers with the blockdev mapping's
182 : * private_lock.
183 : *
184 : * Hack idea: for the blockdev mapping, private_lock contention
185 : * may be quite high. This code could TryLock the page, and if that
186 : * succeeds, there is no need to take private_lock.
187 : */
188 : static struct buffer_head *
189 2739 : __find_get_block_slow(struct block_device *bdev, sector_t block)
190 : {
191 2739 : struct inode *bd_inode = bdev->bd_inode;
192 2739 : struct address_space *bd_mapping = bd_inode->i_mapping;
193 2739 : struct buffer_head *ret = NULL;
194 2739 : pgoff_t index;
195 2739 : struct buffer_head *bh;
196 2739 : struct buffer_head *head;
197 2739 : struct folio *folio;
198 2739 : int all_mapped = 1;
199 2739 : static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
200 :
201 2739 : index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
202 2739 : folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0);
203 2739 : if (IS_ERR(folio))
204 445 : goto out;
205 :
206 2294 : spin_lock(&bd_mapping->private_lock);
207 2294 : head = folio_buffers(folio);
208 2294 : if (!head)
209 41 : goto out_unlock;
210 : bh = head;
211 70268 : do {
212 140536 : if (!buffer_mapped(bh))
213 : all_mapped = 0;
214 70268 : else if (bh->b_blocknr == block) {
215 2253 : ret = bh;
216 2253 : get_bh(bh);
217 2253 : goto out_unlock;
218 : }
219 68015 : bh = bh->b_this_page;
220 68015 : } while (bh != head);
221 :
222 : /* we might be here because some of the buffers on this page are
223 : * not mapped. This is due to various races between
224 : * file io on the block device and getblk. It gets dealt with
225 : * elsewhere, don't buffer_error if we had some unmapped buffers
226 : */
227 0 : ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
228 0 : if (all_mapped && __ratelimit(&last_warned)) {
229 0 : printk("__find_get_block_slow() failed. block=%llu, "
230 : "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
231 : "device %pg blocksize: %d\n",
232 : (unsigned long long)block,
233 : (unsigned long long)bh->b_blocknr,
234 : bh->b_state, bh->b_size, bdev,
235 : 1 << bd_inode->i_blkbits);
236 : }
237 0 : out_unlock:
238 2294 : spin_unlock(&bd_mapping->private_lock);
239 2294 : folio_put(folio);
240 2739 : out:
241 2739 : return ret;
242 : }
243 :
244 92377 : static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
245 : {
246 92377 : unsigned long flags;
247 92377 : struct buffer_head *first;
248 92377 : struct buffer_head *tmp;
249 92377 : struct folio *folio;
250 92377 : int folio_uptodate = 1;
251 :
252 184754 : BUG_ON(!buffer_async_read(bh));
253 :
254 92377 : folio = bh->b_folio;
255 92377 : if (uptodate) {
256 2536 : set_buffer_uptodate(bh);
257 : } else {
258 89841 : clear_buffer_uptodate(bh);
259 89841 : buffer_io_error(bh, ", async page read");
260 89841 : folio_set_error(folio);
261 : }
262 :
263 : /*
264 : * Be _very_ careful from here on. Bad things can happen if
265 : * two buffer heads end IO at almost the same time and both
266 : * decide that the page is now completely done.
267 : */
268 92377 : first = folio_buffers(folio);
269 92377 : spin_lock_irqsave(&first->b_uptodate_lock, flags);
270 92377 : clear_buffer_async_read(bh);
271 92377 : unlock_buffer(bh);
272 92377 : tmp = bh;
273 271240 : do {
274 542480 : if (!buffer_uptodate(tmp))
275 259856 : folio_uptodate = 0;
276 542480 : if (buffer_async_read(tmp)) {
277 171174 : BUG_ON(!buffer_locked(tmp));
278 85587 : goto still_busy;
279 : }
280 185653 : tmp = tmp->b_this_page;
281 185653 : } while (tmp != bh);
282 6790 : spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
283 :
284 : /*
285 : * If all of the buffers are uptodate then we can set the page
286 : * uptodate.
287 : */
288 6790 : if (folio_uptodate)
289 1170 : folio_mark_uptodate(folio);
290 6790 : folio_unlock(folio);
291 6790 : return;
292 :
293 : still_busy:
294 85587 : spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
295 : return;
296 : }
297 :
298 : struct postprocess_bh_ctx {
299 : struct work_struct work;
300 : struct buffer_head *bh;
301 : };
302 :
303 : static void verify_bh(struct work_struct *work)
304 : {
305 : struct postprocess_bh_ctx *ctx =
306 : container_of(work, struct postprocess_bh_ctx, work);
307 : struct buffer_head *bh = ctx->bh;
308 : bool valid;
309 :
310 : valid = fsverity_verify_blocks(bh->b_folio, bh->b_size, bh_offset(bh));
311 : end_buffer_async_read(bh, valid);
312 : kfree(ctx);
313 : }
314 :
315 : static bool need_fsverity(struct buffer_head *bh)
316 : {
317 : struct folio *folio = bh->b_folio;
318 : struct inode *inode = folio->mapping->host;
319 :
320 : return fsverity_active(inode) &&
321 : /* needed by ext4 */
322 : folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
323 : }
324 :
325 : static void decrypt_bh(struct work_struct *work)
326 : {
327 : struct postprocess_bh_ctx *ctx =
328 : container_of(work, struct postprocess_bh_ctx, work);
329 : struct buffer_head *bh = ctx->bh;
330 : int err;
331 :
332 : err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size,
333 : bh_offset(bh));
334 : if (err == 0 && need_fsverity(bh)) {
335 : /*
336 : * We use different work queues for decryption and for verity
337 : * because verity may require reading metadata pages that need
338 : * decryption, and we shouldn't recurse to the same workqueue.
339 : */
340 : INIT_WORK(&ctx->work, verify_bh);
341 : fsverity_enqueue_verify_work(&ctx->work);
342 : return;
343 : }
344 : end_buffer_async_read(bh, err == 0);
345 : kfree(ctx);
346 : }
347 :
348 : /*
349 : * I/O completion handler for block_read_full_folio() - pages
350 : * which come unlocked at the end of I/O.
351 : */
352 92376 : static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
353 : {
354 92376 : struct inode *inode = bh->b_folio->mapping->host;
355 92376 : bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode);
356 92376 : bool verify = need_fsverity(bh);
357 :
358 : /* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */
359 92376 : if (uptodate && (decrypt || verify)) {
360 : struct postprocess_bh_ctx *ctx =
361 : kmalloc(sizeof(*ctx), GFP_ATOMIC);
362 :
363 : if (ctx) {
364 : ctx->bh = bh;
365 : if (decrypt) {
366 : INIT_WORK(&ctx->work, decrypt_bh);
367 : fscrypt_enqueue_decrypt_work(&ctx->work);
368 : } else {
369 : INIT_WORK(&ctx->work, verify_bh);
370 : fsverity_enqueue_verify_work(&ctx->work);
371 : }
372 : return;
373 : }
374 : uptodate = 0;
375 : }
376 92376 : end_buffer_async_read(bh, uptodate);
377 : }
378 :
379 : /*
380 : * Completion handler for block_write_full_page() - pages which are unlocked
381 : * during I/O, and which have PageWriteback cleared upon I/O completion.
382 : */
383 173995 : void end_buffer_async_write(struct buffer_head *bh, int uptodate)
384 : {
385 173995 : unsigned long flags;
386 173995 : struct buffer_head *first;
387 173995 : struct buffer_head *tmp;
388 173995 : struct folio *folio;
389 :
390 347990 : BUG_ON(!buffer_async_write(bh));
391 :
392 173995 : folio = bh->b_folio;
393 173995 : if (uptodate) {
394 173973 : set_buffer_uptodate(bh);
395 : } else {
396 22 : buffer_io_error(bh, ", lost async page write");
397 22 : mark_buffer_write_io_error(bh);
398 22 : clear_buffer_uptodate(bh);
399 22 : folio_set_error(folio);
400 : }
401 :
402 173995 : first = folio_buffers(folio);
403 173995 : spin_lock_irqsave(&first->b_uptodate_lock, flags);
404 :
405 173995 : clear_buffer_async_write(bh);
406 173995 : unlock_buffer(bh);
407 173995 : tmp = bh->b_this_page;
408 287008 : while (tmp != bh) {
409 255666 : if (buffer_async_write(tmp)) {
410 29640 : BUG_ON(!buffer_locked(tmp));
411 14820 : goto still_busy;
412 : }
413 113013 : tmp = tmp->b_this_page;
414 : }
415 159175 : spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
416 159175 : folio_end_writeback(folio);
417 159175 : return;
418 :
419 : still_busy:
420 14820 : spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
421 : return;
422 : }
423 : EXPORT_SYMBOL(end_buffer_async_write);
424 :
425 : /*
426 : * If a page's buffers are under async readin (end_buffer_async_read
427 : * completion) then there is a possibility that another thread of
428 : * control could lock one of the buffers after it has completed
429 : * but while some of the other buffers have not completed. This
430 : * locked buffer would confuse end_buffer_async_read() into not unlocking
431 : * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
432 : * that this buffer is not under async I/O.
433 : *
434 : * The page comes unlocked when it has no locked buffer_async buffers
435 : * left.
436 : *
437 : * PageLocked prevents anyone starting new async I/O reads any of
438 : * the buffers.
439 : *
440 : * PageWriteback is used to prevent simultaneous writeout of the same
441 : * page.
442 : *
443 : * PageLocked prevents anyone from starting writeback of a page which is
444 : * under read I/O (PageWriteback is only ever set against a locked page).
445 : */
446 92377 : static void mark_buffer_async_read(struct buffer_head *bh)
447 : {
448 92377 : bh->b_end_io = end_buffer_async_read_io;
449 92377 : set_buffer_async_read(bh);
450 92377 : }
451 :
452 173995 : static void mark_buffer_async_write_endio(struct buffer_head *bh,
453 : bh_end_io_t *handler)
454 : {
455 173995 : bh->b_end_io = handler;
456 173995 : set_buffer_async_write(bh);
457 173995 : }
458 :
459 0 : void mark_buffer_async_write(struct buffer_head *bh)
460 : {
461 0 : mark_buffer_async_write_endio(bh, end_buffer_async_write);
462 0 : }
463 : EXPORT_SYMBOL(mark_buffer_async_write);
464 :
465 :
466 : /*
467 : * fs/buffer.c contains helper functions for buffer-backed address space's
468 : * fsync functions. A common requirement for buffer-based filesystems is
469 : * that certain data from the backing blockdev needs to be written out for
470 : * a successful fsync(). For example, ext2 indirect blocks need to be
471 : * written back and waited upon before fsync() returns.
472 : *
473 : * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
474 : * inode_has_buffers() and invalidate_inode_buffers() are provided for the
475 : * management of a list of dependent buffers at ->i_mapping->private_list.
476 : *
477 : * Locking is a little subtle: try_to_free_buffers() will remove buffers
478 : * from their controlling inode's queue when they are being freed. But
479 : * try_to_free_buffers() will be operating against the *blockdev* mapping
480 : * at the time, not against the S_ISREG file which depends on those buffers.
481 : * So the locking for private_list is via the private_lock in the address_space
482 : * which backs the buffers. Which is different from the address_space
483 : * against which the buffers are listed. So for a particular address_space,
484 : * mapping->private_lock does *not* protect mapping->private_list! In fact,
485 : * mapping->private_list will always be protected by the backing blockdev's
486 : * ->private_lock.
487 : *
488 : * Which introduces a requirement: all buffers on an address_space's
489 : * ->private_list must be from the same address_space: the blockdev's.
490 : *
491 : * address_spaces which do not place buffers at ->private_list via these
492 : * utility functions are free to use private_lock and private_list for
493 : * whatever they want. The only requirement is that list_empty(private_list)
494 : * be true at clear_inode() time.
495 : *
496 : * FIXME: clear_inode should not call invalidate_inode_buffers(). The
497 : * filesystems should do that. invalidate_inode_buffers() should just go
498 : * BUG_ON(!list_empty).
499 : *
500 : * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
501 : * take an address_space, not an inode. And it should be called
502 : * mark_buffer_dirty_fsync() to clearly define why those buffers are being
503 : * queued up.
504 : *
505 : * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
506 : * list if it is already on a list. Because if the buffer is on a list,
507 : * it *must* already be on the right one. If not, the filesystem is being
508 : * silly. This will save a ton of locking. But first we have to ensure
509 : * that buffers are taken *off* the old inode's list when they are freed
510 : * (presumably in truncate). That requires careful auditing of all
511 : * filesystems (do it inside bforget()). It could also be done by bringing
512 : * b_inode back.
513 : */
514 :
515 : /*
516 : * The buffer's backing address_space's private_lock must be held
517 : */
518 761 : static void __remove_assoc_queue(struct buffer_head *bh)
519 : {
520 761 : list_del_init(&bh->b_assoc_buffers);
521 761 : WARN_ON(!bh->b_assoc_map);
522 761 : bh->b_assoc_map = NULL;
523 761 : }
524 :
525 1534139319 : int inode_has_buffers(struct inode *inode)
526 : {
527 1534294899 : return !list_empty(&inode->i_data.private_list);
528 : }
529 :
530 : /*
531 : * osync is designed to support O_SYNC io. It waits synchronously for
532 : * all already-submitted IO to complete, but does not queue any new
533 : * writes to the disk.
534 : *
535 : * To do O_SYNC writes, just queue the buffer writes with write_dirty_buffer
536 : * as you dirty the buffers, and then use osync_inode_buffers to wait for
537 : * completion. Any other dirty buffers which are not yet queued for
538 : * write will not be flushed to disk by the osync.
539 : */
540 119 : static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
541 : {
542 119 : struct buffer_head *bh;
543 119 : struct list_head *p;
544 119 : int err = 0;
545 :
546 119 : spin_lock(lock);
547 119 : repeat:
548 125 : list_for_each_prev(p, list) {
549 6 : bh = BH_ENTRY(p);
550 12 : if (buffer_locked(bh)) {
551 0 : get_bh(bh);
552 0 : spin_unlock(lock);
553 0 : wait_on_buffer(bh);
554 0 : if (!buffer_uptodate(bh))
555 0 : err = -EIO;
556 0 : brelse(bh);
557 0 : spin_lock(lock);
558 0 : goto repeat;
559 : }
560 : }
561 119 : spin_unlock(lock);
562 119 : return err;
563 : }
564 :
565 0 : void emergency_thaw_bdev(struct super_block *sb)
566 : {
567 0 : while (sb->s_bdev && !thaw_bdev(sb->s_bdev))
568 0 : printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
569 0 : }
570 :
571 : /**
572 : * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
573 : * @mapping: the mapping which wants those buffers written
574 : *
575 : * Starts I/O against the buffers at mapping->private_list, and waits upon
576 : * that I/O.
577 : *
578 : * Basically, this is a convenience function for fsync().
579 : * @mapping is a file or directory which needs those buffers to be written for
580 : * a successful fsync().
581 : */
582 354 : int sync_mapping_buffers(struct address_space *mapping)
583 : {
584 354 : struct address_space *buffer_mapping = mapping->private_data;
585 :
586 354 : if (buffer_mapping == NULL || list_empty(&mapping->private_list))
587 : return 0;
588 :
589 119 : return fsync_buffers_list(&buffer_mapping->private_lock,
590 : &mapping->private_list);
591 : }
592 : EXPORT_SYMBOL(sync_mapping_buffers);
593 :
594 : /**
595 : * generic_buffers_fsync_noflush - generic buffer fsync implementation
596 : * for simple filesystems with no inode lock
597 : *
598 : * @file: file to synchronize
599 : * @start: start offset in bytes
600 : * @end: end offset in bytes (inclusive)
601 : * @datasync: only synchronize essential metadata if true
602 : *
603 : * This is a generic implementation of the fsync method for simple
604 : * filesystems which track all non-inode metadata in the buffers list
605 : * hanging off the address_space structure.
606 : */
607 354 : int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end,
608 : bool datasync)
609 : {
610 354 : struct inode *inode = file->f_mapping->host;
611 354 : int err;
612 354 : int ret;
613 :
614 354 : err = file_write_and_wait_range(file, start, end);
615 354 : if (err)
616 : return err;
617 :
618 354 : ret = sync_mapping_buffers(inode->i_mapping);
619 354 : if (!(inode->i_state & I_DIRTY_ALL))
620 230 : goto out;
621 124 : if (datasync && !(inode->i_state & I_DIRTY_DATASYNC))
622 1 : goto out;
623 :
624 123 : err = sync_inode_metadata(inode, 1);
625 123 : if (ret == 0)
626 123 : ret = err;
627 :
628 0 : out:
629 : /* check and advance again to catch errors after syncing out buffers */
630 354 : err = file_check_and_advance_wb_err(file);
631 354 : if (ret == 0)
632 354 : ret = err;
633 : return ret;
634 : }
635 : EXPORT_SYMBOL(generic_buffers_fsync_noflush);
636 :
637 : /**
638 : * generic_buffers_fsync - generic buffer fsync implementation
639 : * for simple filesystems with no inode lock
640 : *
641 : * @file: file to synchronize
642 : * @start: start offset in bytes
643 : * @end: end offset in bytes (inclusive)
644 : * @datasync: only synchronize essential metadata if true
645 : *
646 : * This is a generic implementation of the fsync method for simple
647 : * filesystems which track all non-inode metadata in the buffers list
648 : * hanging off the address_space structure. This also makes sure that
649 : * a device cache flush operation is called at the end.
650 : */
651 354 : int generic_buffers_fsync(struct file *file, loff_t start, loff_t end,
652 : bool datasync)
653 : {
654 354 : struct inode *inode = file->f_mapping->host;
655 354 : int ret;
656 :
657 354 : ret = generic_buffers_fsync_noflush(file, start, end, datasync);
658 354 : if (!ret)
659 354 : ret = blkdev_issue_flush(inode->i_sb->s_bdev);
660 354 : return ret;
661 : }
662 : EXPORT_SYMBOL(generic_buffers_fsync);
663 :
664 : /*
665 : * Called when we've recently written block `bblock', and it is known that
666 : * `bblock' was for a buffer_boundary() buffer. This means that the block at
667 : * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
668 : * dirty, schedule it for IO. So that indirects merge nicely with their data.
669 : */
670 8 : void write_boundary_block(struct block_device *bdev,
671 : sector_t bblock, unsigned blocksize)
672 : {
673 8 : struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
674 8 : if (bh) {
675 8 : if (buffer_dirty(bh))
676 0 : write_dirty_buffer(bh, 0);
677 4 : put_bh(bh);
678 : }
679 8 : }
680 :
681 32958 : void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
682 : {
683 32958 : struct address_space *mapping = inode->i_mapping;
684 32958 : struct address_space *buffer_mapping = bh->b_folio->mapping;
685 :
686 32958 : mark_buffer_dirty(bh);
687 32958 : if (!mapping->private_data) {
688 82 : mapping->private_data = buffer_mapping;
689 : } else {
690 32876 : BUG_ON(mapping->private_data != buffer_mapping);
691 : }
692 32958 : if (!bh->b_assoc_map) {
693 612 : spin_lock(&buffer_mapping->private_lock);
694 612 : list_move_tail(&bh->b_assoc_buffers,
695 : &mapping->private_list);
696 612 : bh->b_assoc_map = mapping;
697 612 : spin_unlock(&buffer_mapping->private_lock);
698 : }
699 32958 : }
700 : EXPORT_SYMBOL(mark_buffer_dirty_inode);
701 :
702 : /*
703 : * Add a page to the dirty page list.
704 : *
705 : * It is a sad fact of life that this function is called from several places
706 : * deeply under spinlocking. It may not sleep.
707 : *
708 : * If the page has buffers, the uptodate buffers are set dirty, to preserve
709 : * dirty-state coherency between the page and the buffers. It the page does
710 : * not have buffers then when they are later attached they will all be set
711 : * dirty.
712 : *
713 : * The buffers are dirtied before the page is dirtied. There's a small race
714 : * window in which a writepage caller may see the page cleanness but not the
715 : * buffer dirtiness. That's fine. If this code were to set the page dirty
716 : * before the buffers, a concurrent writepage caller could clear the page dirty
717 : * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
718 : * page on the dirty page list.
719 : *
720 : * We use private_lock to lock against try_to_free_buffers while using the
721 : * page's buffer list. Also use this to protect against clean buffers being
722 : * added to the page after it was set dirty.
723 : *
724 : * FIXME: may need to call ->reservepage here as well. That's rather up to the
725 : * address_space though.
726 : */
727 105 : bool block_dirty_folio(struct address_space *mapping, struct folio *folio)
728 : {
729 105 : struct buffer_head *head;
730 105 : bool newly_dirty;
731 :
732 105 : spin_lock(&mapping->private_lock);
733 105 : head = folio_buffers(folio);
734 105 : if (head) {
735 : struct buffer_head *bh = head;
736 :
737 1152 : do {
738 1152 : set_buffer_dirty(bh);
739 1152 : bh = bh->b_this_page;
740 1152 : } while (bh != head);
741 : }
742 : /*
743 : * Lock out page's memcg migration to keep PageDirty
744 : * synchronized with per-memcg dirty page counters.
745 : */
746 105 : folio_memcg_lock(folio);
747 105 : newly_dirty = !folio_test_set_dirty(folio);
748 105 : spin_unlock(&mapping->private_lock);
749 :
750 105 : if (newly_dirty)
751 34 : __folio_mark_dirty(folio, mapping, 1);
752 :
753 105 : folio_memcg_unlock(folio);
754 :
755 105 : if (newly_dirty)
756 34 : __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
757 :
758 105 : return newly_dirty;
759 : }
760 : EXPORT_SYMBOL(block_dirty_folio);
761 :
762 : /*
763 : * Write out and wait upon a list of buffers.
764 : *
765 : * We have conflicting pressures: we want to make sure that all
766 : * initially dirty buffers get waited on, but that any subsequently
767 : * dirtied buffers don't. After all, we don't want fsync to last
768 : * forever if somebody is actively writing to the file.
769 : *
770 : * Do this in two main stages: first we copy dirty buffers to a
771 : * temporary inode list, queueing the writes as we go. Then we clean
772 : * up, waiting for those writes to complete.
773 : *
774 : * During this second stage, any subsequent updates to the file may end
775 : * up refiling the buffer on the original inode's dirty list again, so
776 : * there is a chance we will end up with a buffer queued for write but
777 : * not yet completed on that list. So, as a final cleanup we go through
778 : * the osync code to catch these locked, dirty buffers without requeuing
779 : * any newly dirty buffers for write.
780 : */
781 119 : static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
782 : {
783 119 : struct buffer_head *bh;
784 119 : struct list_head tmp;
785 119 : struct address_space *mapping;
786 119 : int err = 0, err2;
787 119 : struct blk_plug plug;
788 :
789 119 : INIT_LIST_HEAD(&tmp);
790 119 : blk_start_plug(&plug);
791 :
792 119 : spin_lock(lock);
793 417 : while (!list_empty(list)) {
794 298 : bh = BH_ENTRY(list->next);
795 298 : mapping = bh->b_assoc_map;
796 298 : __remove_assoc_queue(bh);
797 : /* Avoid race with mark_buffer_dirty_inode() which does
798 : * a lockless check and we rely on seeing the dirty bit */
799 298 : smp_mb();
800 678 : if (buffer_dirty(bh) || buffer_locked(bh)) {
801 217 : list_add(&bh->b_assoc_buffers, &tmp);
802 217 : bh->b_assoc_map = mapping;
803 434 : if (buffer_dirty(bh)) {
804 216 : get_bh(bh);
805 216 : spin_unlock(lock);
806 : /*
807 : * Ensure any pending I/O completes so that
808 : * write_dirty_buffer() actually writes the
809 : * current contents - it is a noop if I/O is
810 : * still in flight on potentially older
811 : * contents.
812 : */
813 216 : write_dirty_buffer(bh, REQ_SYNC);
814 :
815 : /*
816 : * Kick off IO for the previous mapping. Note
817 : * that we will not run the very last mapping,
818 : * wait_on_buffer() will do that for us
819 : * through sync_buffer().
820 : */
821 216 : brelse(bh);
822 216 : spin_lock(lock);
823 : }
824 : }
825 : }
826 :
827 119 : spin_unlock(lock);
828 119 : blk_finish_plug(&plug);
829 119 : spin_lock(lock);
830 :
831 336 : while (!list_empty(&tmp)) {
832 217 : bh = BH_ENTRY(tmp.prev);
833 217 : get_bh(bh);
834 217 : mapping = bh->b_assoc_map;
835 217 : __remove_assoc_queue(bh);
836 : /* Avoid race with mark_buffer_dirty_inode() which does
837 : * a lockless check and we rely on seeing the dirty bit */
838 217 : smp_mb();
839 434 : if (buffer_dirty(bh)) {
840 0 : list_add(&bh->b_assoc_buffers,
841 : &mapping->private_list);
842 0 : bh->b_assoc_map = mapping;
843 : }
844 217 : spin_unlock(lock);
845 217 : wait_on_buffer(bh);
846 434 : if (!buffer_uptodate(bh))
847 0 : err = -EIO;
848 217 : brelse(bh);
849 217 : spin_lock(lock);
850 : }
851 :
852 119 : spin_unlock(lock);
853 119 : err2 = osync_buffers_list(lock, list);
854 119 : if (err)
855 : return err;
856 : else
857 119 : return err2;
858 : }
859 :
860 : /*
861 : * Invalidate any and all dirty buffers on a given inode. We are
862 : * probably unmounting the fs, but that doesn't mean we have already
863 : * done a sync(). Just drop the buffers from the inode list.
864 : *
865 : * NOTE: we take the inode's blockdev's mapping's private_lock. Which
866 : * assumes that all the buffers are against the blockdev. Not true
867 : * for reiserfs.
868 : */
869 496 : void invalidate_inode_buffers(struct inode *inode)
870 : {
871 496 : if (inode_has_buffers(inode)) {
872 60 : struct address_space *mapping = &inode->i_data;
873 60 : struct list_head *list = &mapping->private_list;
874 60 : struct address_space *buffer_mapping = mapping->private_data;
875 :
876 60 : spin_lock(&buffer_mapping->private_lock);
877 306 : while (!list_empty(list))
878 246 : __remove_assoc_queue(BH_ENTRY(list->next));
879 60 : spin_unlock(&buffer_mapping->private_lock);
880 : }
881 496 : }
882 : EXPORT_SYMBOL(invalidate_inode_buffers);
883 :
884 : /*
885 : * Remove any clean buffers from the inode's buffer list. This is called
886 : * when we're trying to free the inode itself. Those buffers can pin it.
887 : *
888 : * Returns true if all buffers were removed.
889 : */
890 155084 : int remove_inode_buffers(struct inode *inode)
891 : {
892 155084 : int ret = 1;
893 :
894 155084 : if (inode_has_buffers(inode)) {
895 0 : struct address_space *mapping = &inode->i_data;
896 0 : struct list_head *list = &mapping->private_list;
897 0 : struct address_space *buffer_mapping = mapping->private_data;
898 :
899 0 : spin_lock(&buffer_mapping->private_lock);
900 0 : while (!list_empty(list)) {
901 0 : struct buffer_head *bh = BH_ENTRY(list->next);
902 0 : if (buffer_dirty(bh)) {
903 : ret = 0;
904 : break;
905 : }
906 0 : __remove_assoc_queue(bh);
907 : }
908 0 : spin_unlock(&buffer_mapping->private_lock);
909 : }
910 155084 : return ret;
911 : }
912 :
913 : /*
914 : * Create the appropriate buffers when given a folio for data area and
915 : * the size of each buffer.. Use the bh->b_this_page linked list to
916 : * follow the buffers created. Return NULL if unable to create more
917 : * buffers.
918 : *
919 : * The retry flag is used to differentiate async IO (paging, swapping)
920 : * which may not fail from ordinary buffer allocations.
921 : */
922 174909 : struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
923 : bool retry)
924 : {
925 174909 : struct buffer_head *bh, *head;
926 174909 : gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;
927 174909 : long offset;
928 174909 : struct mem_cgroup *memcg, *old_memcg;
929 :
930 174909 : if (retry)
931 174909 : gfp |= __GFP_NOFAIL;
932 :
933 : /* The folio lock pins the memcg */
934 174909 : memcg = folio_memcg(folio);
935 174909 : old_memcg = set_active_memcg(memcg);
936 :
937 174909 : head = NULL;
938 174909 : offset = folio_size(folio);
939 524583 : while ((offset -= size) >= 0) {
940 349674 : bh = alloc_buffer_head(gfp);
941 349674 : if (!bh)
942 0 : goto no_grow;
943 :
944 349674 : bh->b_this_page = head;
945 349674 : bh->b_blocknr = -1;
946 349674 : head = bh;
947 :
948 349674 : bh->b_size = size;
949 :
950 : /* Link the buffer to its folio */
951 349674 : folio_set_bh(bh, folio, offset);
952 : }
953 174909 : out:
954 174909 : set_active_memcg(old_memcg);
955 174909 : return head;
956 : /*
957 : * In case anything failed, we just free everything we got.
958 : */
959 : no_grow:
960 0 : if (head) {
961 0 : do {
962 0 : bh = head;
963 0 : head = head->b_this_page;
964 0 : free_buffer_head(bh);
965 0 : } while (head);
966 : }
967 :
968 0 : goto out;
969 : }
970 : EXPORT_SYMBOL_GPL(folio_alloc_buffers);
971 :
972 0 : struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
973 : bool retry)
974 : {
975 0 : return folio_alloc_buffers(page_folio(page), size, retry);
976 : }
977 : EXPORT_SYMBOL_GPL(alloc_page_buffers);
978 :
979 241 : static inline void link_dev_buffers(struct folio *folio,
980 : struct buffer_head *head)
981 : {
982 241 : struct buffer_head *bh, *tail;
983 :
984 241 : bh = head;
985 14224 : do {
986 14224 : tail = bh;
987 14224 : bh = bh->b_this_page;
988 14224 : } while (bh);
989 241 : tail->b_this_page = head;
990 241 : folio_attach_private(folio, head);
991 241 : }
992 :
993 241 : static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
994 : {
995 241 : sector_t retval = ~((sector_t)0);
996 241 : loff_t sz = bdev_nr_bytes(bdev);
997 :
998 241 : if (sz) {
999 241 : unsigned int sizebits = blksize_bits(size);
1000 241 : retval = (sz >> sizebits);
1001 : }
1002 241 : return retval;
1003 : }
1004 :
1005 : /*
1006 : * Initialise the state of a blockdev folio's buffers.
1007 : */
1008 241 : static sector_t folio_init_buffers(struct folio *folio,
1009 : struct block_device *bdev, sector_t block, int size)
1010 : {
1011 241 : struct buffer_head *head = folio_buffers(folio);
1012 241 : struct buffer_head *bh = head;
1013 241 : bool uptodate = folio_test_uptodate(folio);
1014 241 : sector_t end_block = blkdev_max_block(bdev, size);
1015 :
1016 14224 : do {
1017 28448 : if (!buffer_mapped(bh)) {
1018 14224 : bh->b_end_io = NULL;
1019 14224 : bh->b_private = NULL;
1020 14224 : bh->b_bdev = bdev;
1021 14224 : bh->b_blocknr = block;
1022 14224 : if (uptodate)
1023 1280 : set_buffer_uptodate(bh);
1024 14224 : if (block < end_block)
1025 14224 : set_buffer_mapped(bh);
1026 : }
1027 14224 : block++;
1028 14224 : bh = bh->b_this_page;
1029 14224 : } while (bh != head);
1030 :
1031 : /*
1032 : * Caller needs to validate requested block against end of device.
1033 : */
1034 241 : return end_block;
1035 : }
1036 :
1037 : /*
1038 : * Create the page-cache page that contains the requested block.
1039 : *
1040 : * This is used purely for blockdev mappings.
1041 : */
1042 : static int
1043 241 : grow_dev_page(struct block_device *bdev, sector_t block,
1044 : pgoff_t index, int size, int sizebits, gfp_t gfp)
1045 : {
1046 241 : struct inode *inode = bdev->bd_inode;
1047 241 : struct folio *folio;
1048 241 : struct buffer_head *bh;
1049 241 : sector_t end_block;
1050 241 : int ret = 0;
1051 241 : gfp_t gfp_mask;
1052 :
1053 241 : gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
1054 :
1055 : /*
1056 : * XXX: __getblk_slow() can not really deal with failure and
1057 : * will endlessly loop on improvised global reclaim. Prefer
1058 : * looping in the allocator rather than here, at least that
1059 : * code knows what it's doing.
1060 : */
1061 241 : gfp_mask |= __GFP_NOFAIL;
1062 :
1063 241 : folio = __filemap_get_folio(inode->i_mapping, index,
1064 : FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp_mask);
1065 :
1066 241 : bh = folio_buffers(folio);
1067 241 : if (bh) {
1068 0 : if (bh->b_size == size) {
1069 0 : end_block = folio_init_buffers(folio, bdev,
1070 : (sector_t)index << sizebits, size);
1071 0 : goto done;
1072 : }
1073 0 : if (!try_to_free_buffers(folio))
1074 0 : goto failed;
1075 : }
1076 :
1077 241 : bh = folio_alloc_buffers(folio, size, true);
1078 :
1079 : /*
1080 : * Link the folio to the buffers and initialise them. Take the
1081 : * lock to be atomic wrt __find_get_block(), which does not
1082 : * run under the folio lock.
1083 : */
1084 241 : spin_lock(&inode->i_mapping->private_lock);
1085 241 : link_dev_buffers(folio, bh);
1086 241 : end_block = folio_init_buffers(folio, bdev,
1087 : (sector_t)index << sizebits, size);
1088 241 : spin_unlock(&inode->i_mapping->private_lock);
1089 241 : done:
1090 241 : ret = (block < end_block) ? 1 : -ENXIO;
1091 241 : failed:
1092 241 : folio_unlock(folio);
1093 241 : folio_put(folio);
1094 241 : return ret;
1095 : }
1096 :
1097 : /*
1098 : * Create buffers for the specified block device block's page. If
1099 : * that page was dirty, the buffers are set dirty also.
1100 : */
1101 : static int
1102 241 : grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
1103 : {
1104 241 : pgoff_t index;
1105 241 : int sizebits;
1106 :
1107 241 : sizebits = PAGE_SHIFT - __ffs(size);
1108 241 : index = block >> sizebits;
1109 :
1110 : /*
1111 : * Check for a block which wants to lie outside our maximum possible
1112 : * pagecache index. (this comparison is done using sector_t types).
1113 : */
1114 241 : if (unlikely(index != block >> sizebits)) {
1115 : printk(KERN_ERR "%s: requested out-of-range block %llu for "
1116 : "device %pg\n",
1117 : __func__, (unsigned long long)block,
1118 : bdev);
1119 : return -EIO;
1120 : }
1121 :
1122 : /* Create a page with the proper size buffers.. */
1123 241 : return grow_dev_page(bdev, block, index, size, sizebits, gfp);
1124 : }
1125 :
1126 : static struct buffer_head *
1127 241 : __getblk_slow(struct block_device *bdev, sector_t block,
1128 : unsigned size, gfp_t gfp)
1129 : {
1130 : /* Size must be multiple of hard sectorsize */
1131 482 : if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1132 : (size < 512 || size > PAGE_SIZE))) {
1133 0 : printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1134 : size);
1135 0 : printk(KERN_ERR "logical block size: %d\n",
1136 : bdev_logical_block_size(bdev));
1137 :
1138 0 : dump_stack();
1139 0 : return NULL;
1140 : }
1141 :
1142 482 : for (;;) {
1143 482 : struct buffer_head *bh;
1144 482 : int ret;
1145 :
1146 482 : bh = __find_get_block(bdev, block, size);
1147 482 : if (bh)
1148 241 : return bh;
1149 :
1150 241 : ret = grow_buffers(bdev, block, size, gfp);
1151 241 : if (ret < 0)
1152 : return NULL;
1153 : }
1154 : }
1155 :
1156 : /*
1157 : * The relationship between dirty buffers and dirty pages:
1158 : *
1159 : * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1160 : * the page is tagged dirty in the page cache.
1161 : *
1162 : * At all times, the dirtiness of the buffers represents the dirtiness of
1163 : * subsections of the page. If the page has buffers, the page dirty bit is
1164 : * merely a hint about the true dirty state.
1165 : *
1166 : * When a page is set dirty in its entirety, all its buffers are marked dirty
1167 : * (if the page has buffers).
1168 : *
1169 : * When a buffer is marked dirty, its page is dirtied, but the page's other
1170 : * buffers are not.
1171 : *
1172 : * Also. When blockdev buffers are explicitly read with bread(), they
1173 : * individually become uptodate. But their backing page remains not
1174 : * uptodate - even if all of its buffers are uptodate. A subsequent
1175 : * block_read_full_folio() against that folio will discover all the uptodate
1176 : * buffers, will set the folio uptodate and will perform no I/O.
1177 : */
1178 :
1179 : /**
1180 : * mark_buffer_dirty - mark a buffer_head as needing writeout
1181 : * @bh: the buffer_head to mark dirty
1182 : *
1183 : * mark_buffer_dirty() will set the dirty bit against the buffer, then set
1184 : * its backing page dirty, then tag the page as dirty in the page cache
1185 : * and then attach the address_space's inode to its superblock's dirty
1186 : * inode list.
1187 : *
1188 : * mark_buffer_dirty() is atomic. It takes bh->b_folio->mapping->private_lock,
1189 : * i_pages lock and mapping->host->i_lock.
1190 : */
1191 5701517 : void mark_buffer_dirty(struct buffer_head *bh)
1192 : {
1193 11403034 : WARN_ON_ONCE(!buffer_uptodate(bh));
1194 :
1195 5701517 : trace_block_dirty_buffer(bh);
1196 :
1197 : /*
1198 : * Very *carefully* optimize the it-is-already-dirty case.
1199 : *
1200 : * Don't let the final "is it dirty" escape to before we
1201 : * perhaps modified the buffer.
1202 : */
1203 11403034 : if (buffer_dirty(bh)) {
1204 5498151 : smp_mb();
1205 10996302 : if (buffer_dirty(bh))
1206 : return;
1207 : }
1208 :
1209 406732 : if (!test_set_buffer_dirty(bh)) {
1210 203366 : struct folio *folio = bh->b_folio;
1211 203366 : struct address_space *mapping = NULL;
1212 :
1213 203366 : folio_memcg_lock(folio);
1214 406732 : if (!folio_test_set_dirty(folio)) {
1215 169168 : mapping = folio->mapping;
1216 169168 : if (mapping)
1217 169168 : __folio_mark_dirty(folio, mapping, 0);
1218 : }
1219 203366 : folio_memcg_unlock(folio);
1220 203366 : if (mapping)
1221 169168 : __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1222 : }
1223 : }
1224 : EXPORT_SYMBOL(mark_buffer_dirty);
1225 :
1226 22 : void mark_buffer_write_io_error(struct buffer_head *bh)
1227 : {
1228 22 : struct super_block *sb;
1229 :
1230 22 : set_buffer_write_io_error(bh);
1231 : /* FIXME: do we need to set this in both places? */
1232 22 : if (bh->b_folio && bh->b_folio->mapping)
1233 22 : mapping_set_error(bh->b_folio->mapping, -EIO);
1234 22 : if (bh->b_assoc_map)
1235 0 : mapping_set_error(bh->b_assoc_map, -EIO);
1236 22 : rcu_read_lock();
1237 22 : sb = READ_ONCE(bh->b_bdev->bd_super);
1238 22 : if (sb)
1239 0 : errseq_set(&sb->s_wb_err, -EIO);
1240 22 : rcu_read_unlock();
1241 22 : }
1242 : EXPORT_SYMBOL(mark_buffer_write_io_error);
1243 :
1244 : /*
1245 : * Decrement a buffer_head's reference count. If all buffers against a page
1246 : * have zero reference count, are clean and unlocked, and if the page is clean
1247 : * and unlocked then try_to_free_buffers() may strip the buffers from the page
1248 : * in preparation for freeing it (sometimes, rarely, buffers are removed from
1249 : * a page but it ends up not being freed, and buffers may later be reattached).
1250 : */
1251 166494 : void __brelse(struct buffer_head * buf)
1252 : {
1253 166494 : if (atomic_read(&buf->b_count)) {
1254 166494 : put_bh(buf);
1255 166494 : return;
1256 : }
1257 0 : WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1258 : }
1259 : EXPORT_SYMBOL(__brelse);
1260 :
1261 : /*
1262 : * bforget() is like brelse(), except it discards any
1263 : * potentially dirty data.
1264 : */
1265 95 : void __bforget(struct buffer_head *bh)
1266 : {
1267 95 : clear_buffer_dirty(bh);
1268 95 : if (bh->b_assoc_map) {
1269 68 : struct address_space *buffer_mapping = bh->b_folio->mapping;
1270 :
1271 68 : spin_lock(&buffer_mapping->private_lock);
1272 68 : list_del_init(&bh->b_assoc_buffers);
1273 68 : bh->b_assoc_map = NULL;
1274 68 : spin_unlock(&buffer_mapping->private_lock);
1275 : }
1276 95 : __brelse(bh);
1277 95 : }
1278 : EXPORT_SYMBOL(__bforget);
1279 :
1280 25 : static struct buffer_head *__bread_slow(struct buffer_head *bh)
1281 : {
1282 25 : lock_buffer(bh);
1283 50 : if (buffer_uptodate(bh)) {
1284 2 : unlock_buffer(bh);
1285 2 : return bh;
1286 : } else {
1287 23 : get_bh(bh);
1288 23 : bh->b_end_io = end_buffer_read_sync;
1289 23 : submit_bh(REQ_OP_READ, bh);
1290 23 : wait_on_buffer(bh);
1291 46 : if (buffer_uptodate(bh))
1292 : return bh;
1293 : }
1294 0 : brelse(bh);
1295 : return NULL;
1296 : }
1297 :
1298 : /*
1299 : * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1300 : * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1301 : * refcount elevated by one when they're in an LRU. A buffer can only appear
1302 : * once in a particular CPU's LRU. A single buffer can be present in multiple
1303 : * CPU's LRUs at the same time.
1304 : *
1305 : * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1306 : * sb_find_get_block().
1307 : *
1308 : * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1309 : * a local interrupt disable for that.
1310 : */
1311 :
1312 : #define BH_LRU_SIZE 16
1313 :
1314 : struct bh_lru {
1315 : struct buffer_head *bhs[BH_LRU_SIZE];
1316 : };
1317 :
1318 : static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1319 :
1320 : #ifdef CONFIG_SMP
1321 : #define bh_lru_lock() local_irq_disable()
1322 : #define bh_lru_unlock() local_irq_enable()
1323 : #else
1324 : #define bh_lru_lock() preempt_disable()
1325 : #define bh_lru_unlock() preempt_enable()
1326 : #endif
1327 :
1328 166536 : static inline void check_irqs_on(void)
1329 : {
1330 : #ifdef irqs_disabled
1331 166536 : BUG_ON(irqs_disabled());
1332 : #endif
1333 166535 : }
1334 :
1335 : /*
1336 : * Install a buffer_head into this cpu's LRU. If not already in the LRU, it is
1337 : * inserted at the front, and the buffer_head at the back if any is evicted.
1338 : * Or, if already in the LRU it is moved to the front.
1339 : */
1340 2253 : static void bh_lru_install(struct buffer_head *bh)
1341 : {
1342 2253 : struct buffer_head *evictee = bh;
1343 2253 : struct bh_lru *b;
1344 2253 : int i;
1345 :
1346 2253 : check_irqs_on();
1347 2253 : bh_lru_lock();
1348 :
1349 : /*
1350 : * the refcount of buffer_head in bh_lru prevents dropping the
1351 : * attached page(i.e., try_to_free_buffers) so it could cause
1352 : * failing page migration.
1353 : * Skip putting upcoming bh into bh_lru until migration is done.
1354 : */
1355 2253 : if (lru_cache_disabled()) {
1356 0 : bh_lru_unlock();
1357 0 : return;
1358 : }
1359 :
1360 2253 : b = this_cpu_ptr(&bh_lrus);
1361 38301 : for (i = 0; i < BH_LRU_SIZE; i++) {
1362 36048 : swap(evictee, b->bhs[i]);
1363 36048 : if (evictee == bh) {
1364 0 : bh_lru_unlock();
1365 0 : return;
1366 : }
1367 : }
1368 :
1369 2253 : get_bh(bh);
1370 2253 : bh_lru_unlock();
1371 2253 : brelse(evictee);
1372 : }
1373 :
1374 : /*
1375 : * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1376 : */
1377 : static struct buffer_head *
1378 164283 : lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1379 : {
1380 164283 : struct buffer_head *ret = NULL;
1381 164283 : unsigned int i;
1382 :
1383 164283 : check_irqs_on();
1384 164283 : bh_lru_lock();
1385 604795 : for (i = 0; i < BH_LRU_SIZE; i++) {
1386 437774 : struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1387 :
1388 437775 : if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1389 161544 : bh->b_size == size) {
1390 161544 : if (i) {
1391 367601 : while (i) {
1392 232417 : __this_cpu_write(bh_lrus.bhs[i],
1393 : __this_cpu_read(bh_lrus.bhs[i - 1]));
1394 232418 : i--;
1395 : }
1396 135184 : __this_cpu_write(bh_lrus.bhs[0], bh);
1397 : }
1398 161545 : get_bh(bh);
1399 161545 : ret = bh;
1400 161545 : break;
1401 : }
1402 : }
1403 164284 : bh_lru_unlock();
1404 164283 : return ret;
1405 : }
1406 :
1407 : /*
1408 : * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1409 : * it in the LRU and mark it as accessed. If it is not present then return
1410 : * NULL
1411 : */
1412 : struct buffer_head *
1413 164283 : __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1414 : {
1415 164283 : struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1416 :
1417 164284 : if (bh == NULL) {
1418 : /* __find_get_block_slow will mark the page accessed */
1419 2739 : bh = __find_get_block_slow(bdev, block);
1420 2739 : if (bh)
1421 2253 : bh_lru_install(bh);
1422 : } else
1423 161545 : touch_buffer(bh);
1424 :
1425 164283 : return bh;
1426 : }
1427 : EXPORT_SYMBOL(__find_get_block);
1428 :
1429 : /*
1430 : * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
1431 : * which corresponds to the passed block_device, block and size. The
1432 : * returned buffer has its reference count incremented.
1433 : *
1434 : * __getblk_gfp() will lock up the machine if grow_dev_page's
1435 : * try_to_free_buffers() attempt is failing. FIXME, perhaps?
1436 : */
1437 : struct buffer_head *
1438 163793 : __getblk_gfp(struct block_device *bdev, sector_t block,
1439 : unsigned size, gfp_t gfp)
1440 : {
1441 163793 : struct buffer_head *bh = __find_get_block(bdev, block, size);
1442 :
1443 163792 : might_sleep();
1444 163794 : if (bh == NULL)
1445 241 : bh = __getblk_slow(bdev, block, size, gfp);
1446 163794 : return bh;
1447 : }
1448 : EXPORT_SYMBOL(__getblk_gfp);
1449 :
1450 : /*
1451 : * Do async read-ahead on a buffer..
1452 : */
1453 209 : void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1454 : {
1455 209 : struct buffer_head *bh = __getblk(bdev, block, size);
1456 209 : if (likely(bh)) {
1457 209 : bh_readahead(bh, REQ_RAHEAD);
1458 209 : brelse(bh);
1459 : }
1460 209 : }
1461 : EXPORT_SYMBOL(__breadahead);
1462 :
1463 : /**
1464 : * __bread_gfp() - reads a specified block and returns the bh
1465 : * @bdev: the block_device to read from
1466 : * @block: number of block
1467 : * @size: size (in bytes) to read
1468 : * @gfp: page allocation flag
1469 : *
1470 : * Reads a specified block, and returns buffer head that contains it.
1471 : * The page cache can be allocated from non-movable area
1472 : * not to prevent page migration if you set gfp to zero.
1473 : * It returns NULL if the block was unreadable.
1474 : */
1475 : struct buffer_head *
1476 130046 : __bread_gfp(struct block_device *bdev, sector_t block,
1477 : unsigned size, gfp_t gfp)
1478 : {
1479 130046 : struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
1480 :
1481 260093 : if (likely(bh) && !buffer_uptodate(bh))
1482 25 : bh = __bread_slow(bh);
1483 130046 : return bh;
1484 : }
1485 : EXPORT_SYMBOL(__bread_gfp);
1486 :
1487 52271 : static void __invalidate_bh_lrus(struct bh_lru *b)
1488 : {
1489 52271 : int i;
1490 :
1491 887761 : for (i = 0; i < BH_LRU_SIZE; i++) {
1492 835465 : brelse(b->bhs[i]);
1493 835490 : b->bhs[i] = NULL;
1494 : }
1495 52296 : }
1496 : /*
1497 : * invalidate_bh_lrus() is called rarely - but not only at unmount.
1498 : * This doesn't race because it runs in each cpu either in irq
1499 : * or with preempt disabled.
1500 : */
1501 12 : static void invalidate_bh_lru(void *arg)
1502 : {
1503 12 : struct bh_lru *b = &get_cpu_var(bh_lrus);
1504 :
1505 12 : __invalidate_bh_lrus(b);
1506 12 : put_cpu_var(bh_lrus);
1507 12 : }
1508 :
1509 199924 : bool has_bh_in_lru(int cpu, void *dummy)
1510 : {
1511 199924 : struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1512 199924 : int i;
1513 :
1514 3398484 : for (i = 0; i < BH_LRU_SIZE; i++) {
1515 3198574 : if (b->bhs[i])
1516 : return true;
1517 : }
1518 :
1519 : return false;
1520 : }
1521 :
1522 85535 : void invalidate_bh_lrus(void)
1523 : {
1524 85535 : on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
1525 85535 : }
1526 : EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1527 :
1528 : /*
1529 : * It's called from workqueue context so we need a bh_lru_lock to close
1530 : * the race with preemption/irq.
1531 : */
1532 52259 : void invalidate_bh_lrus_cpu(void)
1533 : {
1534 52259 : struct bh_lru *b;
1535 :
1536 52259 : bh_lru_lock();
1537 52334 : b = this_cpu_ptr(&bh_lrus);
1538 52335 : __invalidate_bh_lrus(b);
1539 52238 : bh_lru_unlock();
1540 52238 : }
1541 :
1542 205 : void set_bh_page(struct buffer_head *bh,
1543 : struct page *page, unsigned long offset)
1544 : {
1545 205 : bh->b_page = page;
1546 205 : BUG_ON(offset >= PAGE_SIZE);
1547 205 : if (PageHighMem(page))
1548 : /*
1549 : * This catches illegal uses and preserves the offset:
1550 : */
1551 : bh->b_data = (char *)(0 + offset);
1552 : else
1553 205 : bh->b_data = page_address(page) + offset;
1554 205 : }
1555 : EXPORT_SYMBOL(set_bh_page);
1556 :
1557 349674 : void folio_set_bh(struct buffer_head *bh, struct folio *folio,
1558 : unsigned long offset)
1559 : {
1560 349674 : bh->b_folio = folio;
1561 349674 : BUG_ON(offset >= folio_size(folio));
1562 349674 : if (folio_test_highmem(folio))
1563 : /*
1564 : * This catches illegal uses and preserves the offset:
1565 : */
1566 : bh->b_data = (char *)(0 + offset);
1567 : else
1568 349674 : bh->b_data = folio_address(folio) + offset;
1569 349674 : }
1570 : EXPORT_SYMBOL(folio_set_bh);
1571 :
1572 : /*
1573 : * Called when truncating a buffer on a page completely.
1574 : */
1575 :
1576 : /* Bits that are cleared during an invalidate */
1577 : #define BUFFER_FLAGS_DISCARD \
1578 : (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1579 : 1 << BH_Delay | 1 << BH_Unwritten)
1580 :
1581 189780 : static void discard_buffer(struct buffer_head * bh)
1582 : {
1583 189780 : unsigned long b_state;
1584 :
1585 189780 : lock_buffer(bh);
1586 189780 : clear_buffer_dirty(bh);
1587 189780 : bh->b_bdev = NULL;
1588 189780 : b_state = READ_ONCE(bh->b_state);
1589 189780 : do {
1590 189780 : } while (!try_cmpxchg(&bh->b_state, &b_state,
1591 : b_state & ~BUFFER_FLAGS_DISCARD));
1592 189780 : unlock_buffer(bh);
1593 189780 : }
1594 :
1595 : /**
1596 : * block_invalidate_folio - Invalidate part or all of a buffer-backed folio.
1597 : * @folio: The folio which is affected.
1598 : * @offset: start of the range to invalidate
1599 : * @length: length of the range to invalidate
1600 : *
1601 : * block_invalidate_folio() is called when all or part of the folio has been
1602 : * invalidated by a truncate operation.
1603 : *
1604 : * block_invalidate_folio() does not have to release all buffers, but it must
1605 : * ensure that no dirty buffer is left outside @offset and that no I/O
1606 : * is underway against any of the blocks which are outside the truncation
1607 : * point. Because the caller is about to free (and possibly reuse) those
1608 : * blocks on-disk.
1609 : */
1610 133019 : void block_invalidate_folio(struct folio *folio, size_t offset, size_t length)
1611 : {
1612 133019 : struct buffer_head *head, *bh, *next;
1613 133019 : size_t curr_off = 0;
1614 133019 : size_t stop = length + offset;
1615 :
1616 133019 : BUG_ON(!folio_test_locked(folio));
1617 :
1618 : /*
1619 : * Check for overflow
1620 : */
1621 133019 : BUG_ON(stop > folio_size(folio) || stop < length);
1622 :
1623 133019 : head = folio_buffers(folio);
1624 133019 : if (!head)
1625 : return;
1626 :
1627 : bh = head;
1628 197010 : do {
1629 197010 : size_t next_off = curr_off + bh->b_size;
1630 197010 : next = bh->b_this_page;
1631 :
1632 : /*
1633 : * Are we still fully in range ?
1634 : */
1635 197010 : if (next_off > stop)
1636 4337 : goto out;
1637 :
1638 : /*
1639 : * is this block fully invalidated?
1640 : */
1641 192673 : if (offset <= curr_off)
1642 189780 : discard_buffer(bh);
1643 192673 : curr_off = next_off;
1644 192673 : bh = next;
1645 192673 : } while (bh != head);
1646 :
1647 : /*
1648 : * We release buffers only if the entire folio is being invalidated.
1649 : * The get_block cached value has been unconditionally invalidated,
1650 : * so real IO is not possible anymore.
1651 : */
1652 128682 : if (length == folio_size(folio))
1653 127172 : filemap_release_folio(folio, 0);
1654 1510 : out:
1655 : return;
1656 : }
1657 : EXPORT_SYMBOL(block_invalidate_folio);
1658 :
1659 : /*
1660 : * We attach and possibly dirty the buffers atomically wrt
1661 : * block_dirty_folio() via private_lock. try_to_free_buffers
1662 : * is already excluded via the folio lock.
1663 : */
1664 174668 : void folio_create_empty_buffers(struct folio *folio, unsigned long blocksize,
1665 : unsigned long b_state)
1666 : {
1667 174668 : struct buffer_head *bh, *head, *tail;
1668 :
1669 174668 : head = folio_alloc_buffers(folio, blocksize, true);
1670 174668 : bh = head;
1671 335450 : do {
1672 335450 : bh->b_state |= b_state;
1673 335450 : tail = bh;
1674 335450 : bh = bh->b_this_page;
1675 335450 : } while (bh);
1676 174668 : tail->b_this_page = head;
1677 :
1678 174668 : spin_lock(&folio->mapping->private_lock);
1679 177666 : if (folio_test_uptodate(folio) || folio_test_dirty(folio)) {
1680 : bh = head;
1681 8695 : do {
1682 8695 : if (folio_test_dirty(folio))
1683 0 : set_buffer_dirty(bh);
1684 17390 : if (folio_test_uptodate(folio))
1685 8695 : set_buffer_uptodate(bh);
1686 8695 : bh = bh->b_this_page;
1687 8695 : } while (bh != head);
1688 : }
1689 174668 : folio_attach_private(folio, head);
1690 174668 : spin_unlock(&folio->mapping->private_lock);
1691 174668 : }
1692 : EXPORT_SYMBOL(folio_create_empty_buffers);
1693 :
1694 0 : void create_empty_buffers(struct page *page,
1695 : unsigned long blocksize, unsigned long b_state)
1696 : {
1697 0 : folio_create_empty_buffers(page_folio(page), blocksize, b_state);
1698 0 : }
1699 : EXPORT_SYMBOL(create_empty_buffers);
1700 :
1701 : /**
1702 : * clean_bdev_aliases: clean a range of buffers in block device
1703 : * @bdev: Block device to clean buffers in
1704 : * @block: Start of a range of blocks to clean
1705 : * @len: Number of blocks to clean
1706 : *
1707 : * We are taking a range of blocks for data and we don't want writeback of any
1708 : * buffer-cache aliases starting from return from this function and until the
1709 : * moment when something will explicitly mark the buffer dirty (hopefully that
1710 : * will not happen until we will free that block ;-) We don't even need to mark
1711 : * it not-uptodate - nobody can expect anything from a newly allocated buffer
1712 : * anyway. We used to use unmap_buffer() for such invalidation, but that was
1713 : * wrong. We definitely don't want to mark the alias unmapped, for example - it
1714 : * would confuse anyone who might pick it with bread() afterwards...
1715 : *
1716 : * Also.. Note that bforget() doesn't lock the buffer. So there can be
1717 : * writeout I/O going on against recently-freed buffers. We don't wait on that
1718 : * I/O in bforget() - it's more efficient to wait on the I/O only if we really
1719 : * need to. That happens here.
1720 : */
1721 32560 : void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1722 : {
1723 32560 : struct inode *bd_inode = bdev->bd_inode;
1724 32560 : struct address_space *bd_mapping = bd_inode->i_mapping;
1725 32560 : struct folio_batch fbatch;
1726 32560 : pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
1727 32560 : pgoff_t end;
1728 32560 : int i, count;
1729 32560 : struct buffer_head *bh;
1730 32560 : struct buffer_head *head;
1731 :
1732 32560 : end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
1733 32560 : folio_batch_init(&fbatch);
1734 32560 : while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) {
1735 8515 : count = folio_batch_count(&fbatch);
1736 17030 : for (i = 0; i < count; i++) {
1737 8515 : struct folio *folio = fbatch.folios[i];
1738 :
1739 8515 : if (!folio_buffers(folio))
1740 695 : continue;
1741 : /*
1742 : * We use folio lock instead of bd_mapping->private_lock
1743 : * to pin buffers here since we can afford to sleep and
1744 : * it scales better than a global spinlock lock.
1745 : */
1746 7820 : folio_lock(folio);
1747 : /* Recheck when the folio is locked which pins bhs */
1748 7820 : head = folio_buffers(folio);
1749 7820 : if (!head)
1750 0 : goto unlock_page;
1751 : bh = head;
1752 299559 : do {
1753 599118 : if (!buffer_mapped(bh) || (bh->b_blocknr < block))
1754 284107 : goto next;
1755 15452 : if (bh->b_blocknr >= block + len)
1756 : break;
1757 7820 : clear_buffer_dirty(bh);
1758 7820 : wait_on_buffer(bh);
1759 7820 : clear_buffer_req(bh);
1760 291927 : next:
1761 291927 : bh = bh->b_this_page;
1762 291927 : } while (bh != head);
1763 7820 : unlock_page:
1764 7820 : folio_unlock(folio);
1765 : }
1766 8515 : folio_batch_release(&fbatch);
1767 8515 : cond_resched();
1768 : /* End of range already reached? */
1769 8515 : if (index > end || !index)
1770 : break;
1771 : }
1772 32560 : }
1773 : EXPORT_SYMBOL(clean_bdev_aliases);
1774 :
1775 : /*
1776 : * Size is a power-of-two in the range 512..PAGE_SIZE,
1777 : * and the case we care about most is PAGE_SIZE.
1778 : *
1779 : * So this *could* possibly be written with those
1780 : * constraints in mind (relevant mostly if some
1781 : * architecture has a slow bit-scan instruction)
1782 : */
1783 5733924 : static inline int block_size_bits(unsigned int blocksize)
1784 : {
1785 5733924 : return ilog2(blocksize);
1786 : }
1787 :
1788 5733924 : static struct buffer_head *folio_create_buffers(struct folio *folio,
1789 : struct inode *inode,
1790 : unsigned int b_state)
1791 : {
1792 5733924 : BUG_ON(!folio_test_locked(folio));
1793 :
1794 5733924 : if (!folio_buffers(folio))
1795 174641 : folio_create_empty_buffers(folio,
1796 174641 : 1 << READ_ONCE(inode->i_blkbits),
1797 : b_state);
1798 5733924 : return folio_buffers(folio);
1799 : }
1800 :
1801 : /*
1802 : * NOTE! All mapped/uptodate combinations are valid:
1803 : *
1804 : * Mapped Uptodate Meaning
1805 : *
1806 : * No No "unknown" - must do get_block()
1807 : * No Yes "hole" - zero-filled
1808 : * Yes No "allocated" - allocated on disk, not read in
1809 : * Yes Yes "valid" - allocated and up-to-date in memory.
1810 : *
1811 : * "Dirty" is valid only with the last case (mapped+uptodate).
1812 : */
1813 :
1814 : /*
1815 : * While block_write_full_page is writing back the dirty buffers under
1816 : * the page lock, whoever dirtied the buffers may decide to clean them
1817 : * again at any time. We handle that by only looking at the buffer
1818 : * state inside lock_buffer().
1819 : *
1820 : * If block_write_full_page() is called for regular writeback
1821 : * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1822 : * locked buffer. This only can happen if someone has written the buffer
1823 : * directly, with submit_bh(). At the address_space level PageWriteback
1824 : * prevents this contention from occurring.
1825 : *
1826 : * If block_write_full_page() is called with wbc->sync_mode ==
1827 : * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1828 : * causes the writes to be flagged as synchronous writes.
1829 : */
1830 159330 : int __block_write_full_folio(struct inode *inode, struct folio *folio,
1831 : get_block_t *get_block, struct writeback_control *wbc,
1832 : bh_end_io_t *handler)
1833 : {
1834 159330 : int err;
1835 159330 : sector_t block;
1836 159330 : sector_t last_block;
1837 159330 : struct buffer_head *bh, *head;
1838 159330 : unsigned int blocksize, bbits;
1839 159330 : int nr_underway = 0;
1840 159330 : blk_opf_t write_flags = wbc_to_write_flags(wbc);
1841 :
1842 159330 : head = folio_create_buffers(folio, inode,
1843 : (1 << BH_Dirty) | (1 << BH_Uptodate));
1844 :
1845 : /*
1846 : * Be very careful. We have no exclusion from block_dirty_folio
1847 : * here, and the (potentially unmapped) buffers may become dirty at
1848 : * any time. If a buffer becomes dirty here after we've inspected it
1849 : * then we just miss that fact, and the folio stays dirty.
1850 : *
1851 : * Buffers outside i_size may be dirtied by block_dirty_folio;
1852 : * handle that here by just cleaning them.
1853 : */
1854 :
1855 159330 : bh = head;
1856 159330 : blocksize = bh->b_size;
1857 159330 : bbits = block_size_bits(blocksize);
1858 :
1859 159330 : block = (sector_t)folio->index << (PAGE_SHIFT - bbits);
1860 159330 : last_block = (i_size_read(inode) - 1) >> bbits;
1861 :
1862 : /*
1863 : * Get all the dirty buffers mapped to disk addresses and
1864 : * handle any aliases from the underlying blockdev's mapping.
1865 : */
1866 238539 : do {
1867 238539 : if (block > last_block) {
1868 : /*
1869 : * mapped buffers outside i_size will occur, because
1870 : * this folio can be outside i_size when there is a
1871 : * truncate in progress.
1872 : */
1873 : /*
1874 : * The buffer was zeroed by block_write_full_page()
1875 : */
1876 1296 : clear_buffer_dirty(bh);
1877 1294 : set_buffer_uptodate(bh);
1878 711729 : } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1879 : buffer_dirty(bh)) {
1880 315 : WARN_ON(bh->b_size != blocksize);
1881 315 : err = get_block(inode, block, bh, 1);
1882 315 : if (err)
1883 1 : goto recover;
1884 314 : clear_buffer_delay(bh);
1885 628 : if (buffer_new(bh)) {
1886 : /* blockdev mappings never come here */
1887 281 : clear_buffer_new(bh);
1888 281 : clean_bdev_bh_alias(bh);
1889 : }
1890 : }
1891 238536 : bh = bh->b_this_page;
1892 238536 : block++;
1893 238536 : } while (bh != head);
1894 :
1895 238589 : do {
1896 477178 : if (!buffer_mapped(bh))
1897 30585 : continue;
1898 : /*
1899 : * If it's a fully non-blocking write attempt and we cannot
1900 : * lock the buffer then redirty the folio. Note that this can
1901 : * potentially cause a busy-wait loop from writeback threads
1902 : * and kswapd activity, but those code paths have their own
1903 : * higher-level throttling.
1904 : */
1905 208004 : if (wbc->sync_mode != WB_SYNC_NONE) {
1906 169955 : lock_buffer(bh);
1907 38049 : } else if (!trylock_buffer(bh)) {
1908 0 : folio_redirty_for_writepage(wbc, folio);
1909 0 : continue;
1910 : }
1911 416039 : if (test_clear_buffer_dirty(bh)) {
1912 173980 : mark_buffer_async_write_endio(bh, handler);
1913 : } else {
1914 34042 : unlock_buffer(bh);
1915 : }
1916 238591 : } while ((bh = bh->b_this_page) != head);
1917 :
1918 : /*
1919 : * The folio and its buffers are protected by the writeback flag,
1920 : * so we can drop the bh refcounts early.
1921 : */
1922 159329 : BUG_ON(folio_test_writeback(folio));
1923 159329 : folio_start_writeback(folio);
1924 :
1925 238581 : do {
1926 238581 : struct buffer_head *next = bh->b_this_page;
1927 477162 : if (buffer_async_write(bh)) {
1928 173981 : submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc);
1929 173981 : nr_underway++;
1930 : }
1931 238581 : bh = next;
1932 238581 : } while (bh != head);
1933 159329 : folio_unlock(folio);
1934 :
1935 159329 : err = 0;
1936 159330 : done:
1937 159330 : if (nr_underway == 0) {
1938 : /*
1939 : * The folio was marked dirty, but the buffers were
1940 : * clean. Someone wrote them back by hand with
1941 : * write_dirty_buffer/submit_bh. A rare case.
1942 : */
1943 155 : folio_end_writeback(folio);
1944 :
1945 : /*
1946 : * The folio and buffer_heads can be released at any time from
1947 : * here on.
1948 : */
1949 : }
1950 159330 : return err;
1951 :
1952 : recover:
1953 : /*
1954 : * ENOSPC, or some other error. We may already have added some
1955 : * blocks to the file, so we need to write these out to avoid
1956 : * exposing stale data.
1957 : * The folio is currently locked and not marked for writeback
1958 : */
1959 1 : bh = head;
1960 : /* Recovery: lock and submit the mapped buffers */
1961 64 : do {
1962 156 : if (buffer_mapped(bh) && buffer_dirty(bh) &&
1963 : !buffer_delay(bh)) {
1964 14 : lock_buffer(bh);
1965 14 : mark_buffer_async_write_endio(bh, handler);
1966 : } else {
1967 : /*
1968 : * The buffer may have been set dirty during
1969 : * attachment to a dirty folio.
1970 : */
1971 50 : clear_buffer_dirty(bh);
1972 : }
1973 64 : } while ((bh = bh->b_this_page) != head);
1974 1 : folio_set_error(folio);
1975 1 : BUG_ON(folio_test_writeback(folio));
1976 1 : mapping_set_error(folio->mapping, err);
1977 1 : folio_start_writeback(folio);
1978 64 : do {
1979 64 : struct buffer_head *next = bh->b_this_page;
1980 128 : if (buffer_async_write(bh)) {
1981 14 : clear_buffer_dirty(bh);
1982 14 : submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc);
1983 14 : nr_underway++;
1984 : }
1985 64 : bh = next;
1986 64 : } while (bh != head);
1987 1 : folio_unlock(folio);
1988 1 : goto done;
1989 : }
1990 : EXPORT_SYMBOL(__block_write_full_folio);
1991 :
1992 : /*
1993 : * If a folio has any new buffers, zero them out here, and mark them uptodate
1994 : * and dirty so they'll be written out (in order to prevent uninitialised
1995 : * block data from leaking). And clear the new bit.
1996 : */
1997 13 : void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to)
1998 : {
1999 13 : size_t block_start, block_end;
2000 13 : struct buffer_head *head, *bh;
2001 :
2002 13 : BUG_ON(!folio_test_locked(folio));
2003 13 : head = folio_buffers(folio);
2004 13 : if (!head)
2005 : return;
2006 :
2007 : bh = head;
2008 : block_start = 0;
2009 832 : do {
2010 832 : block_end = block_start + bh->b_size;
2011 :
2012 1664 : if (buffer_new(bh)) {
2013 388 : if (block_end > from && block_start < to) {
2014 388 : if (!folio_test_uptodate(folio)) {
2015 388 : size_t start, xend;
2016 :
2017 388 : start = max(from, block_start);
2018 388 : xend = min(to, block_end);
2019 :
2020 388 : folio_zero_segment(folio, start, xend);
2021 388 : set_buffer_uptodate(bh);
2022 : }
2023 :
2024 388 : clear_buffer_new(bh);
2025 388 : mark_buffer_dirty(bh);
2026 : }
2027 : }
2028 :
2029 832 : block_start = block_end;
2030 832 : bh = bh->b_this_page;
2031 832 : } while (bh != head);
2032 : }
2033 : EXPORT_SYMBOL(folio_zero_new_buffers);
2034 :
2035 : static void
2036 0 : iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
2037 : const struct iomap *iomap)
2038 : {
2039 0 : loff_t offset = block << inode->i_blkbits;
2040 :
2041 0 : bh->b_bdev = iomap->bdev;
2042 :
2043 : /*
2044 : * Block points to offset in file we need to map, iomap contains
2045 : * the offset at which the map starts. If the map ends before the
2046 : * current block, then do not map the buffer and let the caller
2047 : * handle it.
2048 : */
2049 0 : BUG_ON(offset >= iomap->offset + iomap->length);
2050 :
2051 0 : switch (iomap->type) {
2052 : case IOMAP_HOLE:
2053 : /*
2054 : * If the buffer is not up to date or beyond the current EOF,
2055 : * we need to mark it as new to ensure sub-block zeroing is
2056 : * executed if necessary.
2057 : */
2058 0 : if (!buffer_uptodate(bh) ||
2059 : (offset >= i_size_read(inode)))
2060 0 : set_buffer_new(bh);
2061 : break;
2062 : case IOMAP_DELALLOC:
2063 0 : if (!buffer_uptodate(bh) ||
2064 : (offset >= i_size_read(inode)))
2065 0 : set_buffer_new(bh);
2066 0 : set_buffer_uptodate(bh);
2067 0 : set_buffer_mapped(bh);
2068 0 : set_buffer_delay(bh);
2069 : break;
2070 : case IOMAP_UNWRITTEN:
2071 : /*
2072 : * For unwritten regions, we always need to ensure that regions
2073 : * in the block we are not writing to are zeroed. Mark the
2074 : * buffer as new to ensure this.
2075 : */
2076 0 : set_buffer_new(bh);
2077 0 : set_buffer_unwritten(bh);
2078 0 : fallthrough;
2079 0 : case IOMAP_MAPPED:
2080 0 : if ((iomap->flags & IOMAP_F_NEW) ||
2081 : offset >= i_size_read(inode))
2082 0 : set_buffer_new(bh);
2083 0 : bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
2084 0 : inode->i_blkbits;
2085 0 : set_buffer_mapped(bh);
2086 : break;
2087 : }
2088 0 : }
2089 :
2090 5567724 : int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,
2091 : get_block_t *get_block, const struct iomap *iomap)
2092 : {
2093 5567724 : unsigned from = pos & (PAGE_SIZE - 1);
2094 5567724 : unsigned to = from + len;
2095 5567724 : struct inode *inode = folio->mapping->host;
2096 5567724 : unsigned block_start, block_end;
2097 5567724 : sector_t block;
2098 5567724 : int err = 0;
2099 5567724 : unsigned blocksize, bbits;
2100 5567724 : struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
2101 :
2102 5567724 : BUG_ON(!folio_test_locked(folio));
2103 5567724 : BUG_ON(from > PAGE_SIZE);
2104 5567724 : BUG_ON(to > PAGE_SIZE);
2105 5567724 : BUG_ON(from > to);
2106 :
2107 5567724 : head = folio_create_buffers(folio, inode, 0);
2108 5567724 : blocksize = head->b_size;
2109 5567724 : bbits = block_size_bits(blocksize);
2110 :
2111 5567724 : block = (sector_t)folio->index << (PAGE_SHIFT - bbits);
2112 :
2113 11254605 : for(bh = head, block_start = 0; bh != head || !block_start;
2114 5686881 : block++, block_start=block_end, bh = bh->b_this_page) {
2115 5686894 : block_end = block_start + blocksize;
2116 5686894 : if (block_end <= from || block_start >= to) {
2117 127646 : if (folio_test_uptodate(folio)) {
2118 82380 : if (!buffer_uptodate(bh))
2119 0 : set_buffer_uptodate(bh);
2120 : }
2121 86456 : continue;
2122 : }
2123 11200876 : if (buffer_new(bh))
2124 0 : clear_buffer_new(bh);
2125 11200876 : if (!buffer_mapped(bh)) {
2126 200127 : WARN_ON(bh->b_size != blocksize);
2127 200127 : if (get_block) {
2128 200127 : err = get_block(inode, block, bh, 1);
2129 200127 : if (err)
2130 : break;
2131 : } else {
2132 0 : iomap_to_bh(inode, block, bh, iomap);
2133 : }
2134 :
2135 400228 : if (buffer_new(bh)) {
2136 31622 : clean_bdev_bh_alias(bh);
2137 32163 : if (folio_test_uptodate(folio)) {
2138 541 : clear_buffer_new(bh);
2139 541 : set_buffer_uptodate(bh);
2140 541 : mark_buffer_dirty(bh);
2141 541 : continue;
2142 : }
2143 31081 : if (block_end > to || block_start < from)
2144 1052 : folio_zero_segments(folio,
2145 : to, block_end,
2146 : block_start, from);
2147 31081 : continue;
2148 : }
2149 : }
2150 10971823 : if (folio_test_uptodate(folio)) {
2151 10806040 : if (!buffer_uptodate(bh))
2152 0 : set_buffer_uptodate(bh);
2153 5403020 : continue;
2154 : }
2155 661650 : if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
2156 165042 : !buffer_unwritten(bh) &&
2157 165042 : (block_start < from || block_end > to)) {
2158 109472 : bh_read_nowait(bh, 0);
2159 109472 : *wait_bh++=bh;
2160 : }
2161 : }
2162 : /*
2163 : * If we issued read requests - let them complete.
2164 : */
2165 5677196 : while(wait_bh > wait) {
2166 109472 : wait_on_buffer(*--wait_bh);
2167 218944 : if (!buffer_uptodate(*wait_bh))
2168 0 : err = -EIO;
2169 : }
2170 5567724 : if (unlikely(err))
2171 13 : folio_zero_new_buffers(folio, from, to);
2172 5567724 : return err;
2173 : }
2174 :
2175 5567724 : int __block_write_begin(struct page *page, loff_t pos, unsigned len,
2176 : get_block_t *get_block)
2177 : {
2178 11135448 : return __block_write_begin_int(page_folio(page), pos, len, get_block,
2179 : NULL);
2180 : }
2181 : EXPORT_SYMBOL(__block_write_begin);
2182 :
2183 5567711 : static int __block_commit_write(struct inode *inode, struct folio *folio,
2184 : size_t from, size_t to)
2185 : {
2186 5567711 : size_t block_start, block_end;
2187 5567711 : bool partial = false;
2188 5567711 : unsigned blocksize;
2189 5567711 : struct buffer_head *bh, *head;
2190 :
2191 5567711 : bh = head = folio_buffers(folio);
2192 5567711 : blocksize = bh->b_size;
2193 :
2194 5567711 : block_start = 0;
2195 5686436 : do {
2196 5686436 : block_end = block_start + blocksize;
2197 5686436 : if (block_end <= from || block_start >= to) {
2198 172798 : if (!buffer_uptodate(bh))
2199 42795 : partial = true;
2200 : } else {
2201 5600037 : set_buffer_uptodate(bh);
2202 5600037 : mark_buffer_dirty(bh);
2203 : }
2204 11372872 : if (buffer_new(bh))
2205 30693 : clear_buffer_new(bh);
2206 :
2207 5686436 : block_start = block_end;
2208 5686436 : bh = bh->b_this_page;
2209 5686436 : } while (bh != head);
2210 :
2211 : /*
2212 : * If this is a partial write which happened to make all buffers
2213 : * uptodate then we can optimize away a bogus read_folio() for
2214 : * the next read(). Here we 'discover' whether the folio went
2215 : * uptodate as a result of this (potentially partial) write.
2216 : */
2217 5567711 : if (!partial)
2218 5565379 : folio_mark_uptodate(folio);
2219 5567711 : return 0;
2220 : }
2221 :
2222 : /*
2223 : * block_write_begin takes care of the basic task of block allocation and
2224 : * bringing partial write blocks uptodate first.
2225 : *
2226 : * The filesystem needs to handle block truncation upon failure.
2227 : */
2228 5567241 : int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2229 : struct page **pagep, get_block_t *get_block)
2230 : {
2231 5567241 : pgoff_t index = pos >> PAGE_SHIFT;
2232 5567241 : struct page *page;
2233 5567241 : int status;
2234 :
2235 5567241 : page = grab_cache_page_write_begin(mapping, index);
2236 5567241 : if (!page)
2237 : return -ENOMEM;
2238 :
2239 5567241 : status = __block_write_begin(page, pos, len, get_block);
2240 5567241 : if (unlikely(status)) {
2241 12 : unlock_page(page);
2242 12 : put_page(page);
2243 12 : page = NULL;
2244 : }
2245 :
2246 5567241 : *pagep = page;
2247 5567241 : return status;
2248 : }
2249 : EXPORT_SYMBOL(block_write_begin);
2250 :
2251 5567711 : int block_write_end(struct file *file, struct address_space *mapping,
2252 : loff_t pos, unsigned len, unsigned copied,
2253 : struct page *page, void *fsdata)
2254 : {
2255 5567711 : struct folio *folio = page_folio(page);
2256 5567711 : struct inode *inode = mapping->host;
2257 5567711 : size_t start = pos - folio_pos(folio);
2258 :
2259 5567711 : if (unlikely(copied < len)) {
2260 : /*
2261 : * The buffers that were written will now be uptodate, so
2262 : * we don't have to worry about a read_folio reading them
2263 : * and overwriting a partial write. However if we have
2264 : * encountered a short write and only partially written
2265 : * into a buffer, it will not be marked uptodate, so a
2266 : * read_folio might come in and destroy our partial write.
2267 : *
2268 : * Do the simplest thing, and just treat any short write to a
2269 : * non uptodate folio as a zero-length write, and force the
2270 : * caller to redo the whole thing.
2271 : */
2272 0 : if (!folio_test_uptodate(folio))
2273 0 : copied = 0;
2274 :
2275 0 : folio_zero_new_buffers(folio, start+copied, start+len);
2276 : }
2277 5567711 : flush_dcache_folio(folio);
2278 :
2279 : /* This could be a short (even 0-length) commit */
2280 5567711 : __block_commit_write(inode, folio, start, start + copied);
2281 :
2282 5567711 : return copied;
2283 : }
2284 : EXPORT_SYMBOL(block_write_end);
2285 :
2286 2564 : int generic_write_end(struct file *file, struct address_space *mapping,
2287 : loff_t pos, unsigned len, unsigned copied,
2288 : struct page *page, void *fsdata)
2289 : {
2290 2564 : struct inode *inode = mapping->host;
2291 2564 : loff_t old_size = inode->i_size;
2292 2564 : bool i_size_changed = false;
2293 :
2294 2564 : copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2295 :
2296 : /*
2297 : * No need to use i_size_read() here, the i_size cannot change under us
2298 : * because we hold i_rwsem.
2299 : *
2300 : * But it's important to update i_size while still holding page lock:
2301 : * page writeout could otherwise come in and zero beyond i_size.
2302 : */
2303 2564 : if (pos + copied > inode->i_size) {
2304 331 : i_size_write(inode, pos + copied);
2305 331 : i_size_changed = true;
2306 : }
2307 :
2308 2564 : unlock_page(page);
2309 2564 : put_page(page);
2310 :
2311 2564 : if (old_size < pos)
2312 158 : pagecache_isize_extended(inode, old_size, pos);
2313 : /*
2314 : * Don't mark the inode dirty under page lock. First, it unnecessarily
2315 : * makes the holding time of page lock longer. Second, it forces lock
2316 : * ordering of page lock and transaction start for journaling
2317 : * filesystems.
2318 : */
2319 2564 : if (i_size_changed)
2320 331 : mark_inode_dirty(inode);
2321 2564 : return copied;
2322 : }
2323 : EXPORT_SYMBOL(generic_write_end);
2324 :
2325 : /*
2326 : * block_is_partially_uptodate checks whether buffers within a folio are
2327 : * uptodate or not.
2328 : *
2329 : * Returns true if all buffers which correspond to the specified part
2330 : * of the folio are uptodate.
2331 : */
2332 8 : bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
2333 : {
2334 8 : unsigned block_start, block_end, blocksize;
2335 8 : unsigned to;
2336 8 : struct buffer_head *bh, *head;
2337 8 : bool ret = true;
2338 :
2339 8 : head = folio_buffers(folio);
2340 8 : if (!head)
2341 : return false;
2342 8 : blocksize = head->b_size;
2343 8 : to = min_t(unsigned, folio_size(folio) - from, count);
2344 8 : to = from + to;
2345 8 : if (from < blocksize && to > folio_size(folio) - blocksize)
2346 : return false;
2347 :
2348 : bh = head;
2349 : block_start = 0;
2350 241 : do {
2351 241 : block_end = block_start + blocksize;
2352 241 : if (block_end > from && block_start < to) {
2353 120 : if (!buffer_uptodate(bh)) {
2354 : ret = false;
2355 : break;
2356 : }
2357 54 : if (block_end >= to)
2358 : break;
2359 : }
2360 233 : block_start = block_end;
2361 233 : bh = bh->b_this_page;
2362 233 : } while (bh != head);
2363 :
2364 : return ret;
2365 : }
2366 : EXPORT_SYMBOL(block_is_partially_uptodate);
2367 :
2368 : /*
2369 : * Generic "read_folio" function for block devices that have the normal
2370 : * get_block functionality. This is most of the block device filesystems.
2371 : * Reads the folio asynchronously --- the unlock_buffer() and
2372 : * set/clear_buffer_uptodate() functions propagate buffer state into the
2373 : * folio once IO has completed.
2374 : */
2375 6870 : int block_read_full_folio(struct folio *folio, get_block_t *get_block)
2376 : {
2377 6870 : struct inode *inode = folio->mapping->host;
2378 6870 : sector_t iblock, lblock;
2379 6870 : struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2380 6870 : unsigned int blocksize, bbits;
2381 6870 : int nr, i;
2382 6870 : int fully_mapped = 1;
2383 6870 : bool page_error = false;
2384 6870 : loff_t limit = i_size_read(inode);
2385 :
2386 : /* This is needed for ext4. */
2387 6870 : if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
2388 : limit = inode->i_sb->s_maxbytes;
2389 :
2390 6870 : VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
2391 :
2392 6870 : head = folio_create_buffers(folio, inode, 0);
2393 6870 : blocksize = head->b_size;
2394 6870 : bbits = block_size_bits(blocksize);
2395 :
2396 6870 : iblock = (sector_t)folio->index << (PAGE_SHIFT - bbits);
2397 6870 : lblock = (limit+blocksize-1) >> bbits;
2398 6870 : bh = head;
2399 6870 : nr = 0;
2400 6870 : i = 0;
2401 :
2402 98160 : do {
2403 196320 : if (buffer_uptodate(bh))
2404 1071 : continue;
2405 :
2406 194178 : if (!buffer_mapped(bh)) {
2407 96175 : int err = 0;
2408 :
2409 96175 : fully_mapped = 0;
2410 96175 : if (iblock < lblock) {
2411 93097 : WARN_ON(bh->b_size != blocksize);
2412 93097 : err = get_block(inode, iblock, bh, 0);
2413 93097 : if (err) {
2414 0 : folio_set_error(folio);
2415 : page_error = true;
2416 : }
2417 : }
2418 192350 : if (!buffer_mapped(bh)) {
2419 4712 : folio_zero_range(folio, i * blocksize,
2420 : blocksize);
2421 4712 : if (!err)
2422 4712 : set_buffer_uptodate(bh);
2423 4712 : continue;
2424 : }
2425 : /*
2426 : * get_block() might have updated the buffer
2427 : * synchronously
2428 : */
2429 182926 : if (buffer_uptodate(bh))
2430 0 : continue;
2431 : }
2432 92377 : arr[nr++] = bh;
2433 98160 : } while (i++, iblock++, (bh = bh->b_this_page) != head);
2434 :
2435 6870 : if (fully_mapped)
2436 56 : folio_set_mappedtodisk(folio);
2437 :
2438 6870 : if (!nr) {
2439 : /*
2440 : * All buffers are uptodate - we can set the folio uptodate
2441 : * as well. But not if get_block() returned an error.
2442 : */
2443 80 : if (!page_error)
2444 80 : folio_mark_uptodate(folio);
2445 80 : folio_unlock(folio);
2446 80 : return 0;
2447 : }
2448 :
2449 : /* Stage two: lock the buffers */
2450 99167 : for (i = 0; i < nr; i++) {
2451 92377 : bh = arr[i];
2452 92377 : lock_buffer(bh);
2453 92377 : mark_buffer_async_read(bh);
2454 : }
2455 :
2456 : /*
2457 : * Stage 3: start the IO. Check for uptodateness
2458 : * inside the buffer lock in case another process reading
2459 : * the underlying blockdev brought it uptodate (the sct fix).
2460 : */
2461 99167 : for (i = 0; i < nr; i++) {
2462 92377 : bh = arr[i];
2463 184754 : if (buffer_uptodate(bh))
2464 1 : end_buffer_async_read(bh, 1);
2465 : else
2466 92376 : submit_bh(REQ_OP_READ, bh);
2467 : }
2468 : return 0;
2469 : }
2470 : EXPORT_SYMBOL(block_read_full_folio);
2471 :
2472 : /* utility function for filesystems that need to do work on expanding
2473 : * truncates. Uses filesystem pagecache writes to allow the filesystem to
2474 : * deal with the hole.
2475 : */
2476 0 : int generic_cont_expand_simple(struct inode *inode, loff_t size)
2477 : {
2478 0 : struct address_space *mapping = inode->i_mapping;
2479 0 : const struct address_space_operations *aops = mapping->a_ops;
2480 0 : struct page *page;
2481 0 : void *fsdata = NULL;
2482 0 : int err;
2483 :
2484 0 : err = inode_newsize_ok(inode, size);
2485 0 : if (err)
2486 0 : goto out;
2487 :
2488 0 : err = aops->write_begin(NULL, mapping, size, 0, &page, &fsdata);
2489 0 : if (err)
2490 0 : goto out;
2491 :
2492 0 : err = aops->write_end(NULL, mapping, size, 0, 0, page, fsdata);
2493 0 : BUG_ON(err > 0);
2494 :
2495 0 : out:
2496 0 : return err;
2497 : }
2498 : EXPORT_SYMBOL(generic_cont_expand_simple);
2499 :
2500 0 : static int cont_expand_zero(struct file *file, struct address_space *mapping,
2501 : loff_t pos, loff_t *bytes)
2502 : {
2503 0 : struct inode *inode = mapping->host;
2504 0 : const struct address_space_operations *aops = mapping->a_ops;
2505 0 : unsigned int blocksize = i_blocksize(inode);
2506 0 : struct page *page;
2507 0 : void *fsdata = NULL;
2508 0 : pgoff_t index, curidx;
2509 0 : loff_t curpos;
2510 0 : unsigned zerofrom, offset, len;
2511 0 : int err = 0;
2512 :
2513 0 : index = pos >> PAGE_SHIFT;
2514 0 : offset = pos & ~PAGE_MASK;
2515 :
2516 0 : while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2517 0 : zerofrom = curpos & ~PAGE_MASK;
2518 0 : if (zerofrom & (blocksize-1)) {
2519 0 : *bytes |= (blocksize-1);
2520 0 : (*bytes)++;
2521 : }
2522 0 : len = PAGE_SIZE - zerofrom;
2523 :
2524 0 : err = aops->write_begin(file, mapping, curpos, len,
2525 : &page, &fsdata);
2526 0 : if (err)
2527 0 : goto out;
2528 0 : zero_user(page, zerofrom, len);
2529 0 : err = aops->write_end(file, mapping, curpos, len, len,
2530 : page, fsdata);
2531 0 : if (err < 0)
2532 0 : goto out;
2533 0 : BUG_ON(err != len);
2534 0 : err = 0;
2535 :
2536 0 : balance_dirty_pages_ratelimited(mapping);
2537 :
2538 0 : if (fatal_signal_pending(current)) {
2539 0 : err = -EINTR;
2540 0 : goto out;
2541 : }
2542 : }
2543 :
2544 : /* page covers the boundary, find the boundary offset */
2545 0 : if (index == curidx) {
2546 0 : zerofrom = curpos & ~PAGE_MASK;
2547 : /* if we will expand the thing last block will be filled */
2548 0 : if (offset <= zerofrom) {
2549 0 : goto out;
2550 : }
2551 0 : if (zerofrom & (blocksize-1)) {
2552 0 : *bytes |= (blocksize-1);
2553 0 : (*bytes)++;
2554 : }
2555 0 : len = offset - zerofrom;
2556 :
2557 0 : err = aops->write_begin(file, mapping, curpos, len,
2558 : &page, &fsdata);
2559 0 : if (err)
2560 0 : goto out;
2561 0 : zero_user(page, zerofrom, len);
2562 0 : err = aops->write_end(file, mapping, curpos, len, len,
2563 : page, fsdata);
2564 0 : if (err < 0)
2565 0 : goto out;
2566 0 : BUG_ON(err != len);
2567 : err = 0;
2568 : }
2569 0 : out:
2570 0 : return err;
2571 : }
2572 :
2573 : /*
2574 : * For moronic filesystems that do not allow holes in file.
2575 : * We may have to extend the file.
2576 : */
2577 0 : int cont_write_begin(struct file *file, struct address_space *mapping,
2578 : loff_t pos, unsigned len,
2579 : struct page **pagep, void **fsdata,
2580 : get_block_t *get_block, loff_t *bytes)
2581 : {
2582 0 : struct inode *inode = mapping->host;
2583 0 : unsigned int blocksize = i_blocksize(inode);
2584 0 : unsigned int zerofrom;
2585 0 : int err;
2586 :
2587 0 : err = cont_expand_zero(file, mapping, pos, bytes);
2588 0 : if (err)
2589 : return err;
2590 :
2591 0 : zerofrom = *bytes & ~PAGE_MASK;
2592 0 : if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2593 0 : *bytes |= (blocksize-1);
2594 0 : (*bytes)++;
2595 : }
2596 :
2597 0 : return block_write_begin(mapping, pos, len, pagep, get_block);
2598 : }
2599 : EXPORT_SYMBOL(cont_write_begin);
2600 :
2601 0 : int block_commit_write(struct page *page, unsigned from, unsigned to)
2602 : {
2603 0 : struct folio *folio = page_folio(page);
2604 0 : struct inode *inode = folio->mapping->host;
2605 0 : __block_commit_write(inode, folio, from, to);
2606 0 : return 0;
2607 : }
2608 : EXPORT_SYMBOL(block_commit_write);
2609 :
2610 : /*
2611 : * block_page_mkwrite() is not allowed to change the file size as it gets
2612 : * called from a page fault handler when a page is first dirtied. Hence we must
2613 : * be careful to check for EOF conditions here. We set the page up correctly
2614 : * for a written page which means we get ENOSPC checking when writing into
2615 : * holes and correct delalloc and unwritten extent mapping on filesystems that
2616 : * support these features.
2617 : *
2618 : * We are not allowed to take the i_mutex here so we have to play games to
2619 : * protect against truncate races as the page could now be beyond EOF. Because
2620 : * truncate writes the inode size before removing pages, once we have the
2621 : * page lock we can determine safely if the page is beyond EOF. If it is not
2622 : * beyond EOF, then the page is guaranteed safe against truncation until we
2623 : * unlock the page.
2624 : *
2625 : * Direct callers of this function should protect against filesystem freezing
2626 : * using sb_start_pagefault() - sb_end_pagefault() functions.
2627 : */
2628 0 : int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2629 : get_block_t get_block)
2630 : {
2631 0 : struct folio *folio = page_folio(vmf->page);
2632 0 : struct inode *inode = file_inode(vma->vm_file);
2633 0 : unsigned long end;
2634 0 : loff_t size;
2635 0 : int ret;
2636 :
2637 0 : folio_lock(folio);
2638 0 : size = i_size_read(inode);
2639 0 : if ((folio->mapping != inode->i_mapping) ||
2640 : (folio_pos(folio) >= size)) {
2641 : /* We overload EFAULT to mean page got truncated */
2642 0 : ret = -EFAULT;
2643 0 : goto out_unlock;
2644 : }
2645 :
2646 0 : end = folio_size(folio);
2647 : /* folio is wholly or partially inside EOF */
2648 0 : if (folio_pos(folio) + end > size)
2649 0 : end = size - folio_pos(folio);
2650 :
2651 0 : ret = __block_write_begin_int(folio, 0, end, get_block, NULL);
2652 0 : if (!ret)
2653 0 : ret = __block_commit_write(inode, folio, 0, end);
2654 :
2655 0 : if (unlikely(ret < 0))
2656 0 : goto out_unlock;
2657 0 : folio_mark_dirty(folio);
2658 0 : folio_wait_stable(folio);
2659 0 : return 0;
2660 0 : out_unlock:
2661 0 : folio_unlock(folio);
2662 0 : return ret;
2663 : }
2664 : EXPORT_SYMBOL(block_page_mkwrite);
2665 :
2666 34 : int block_truncate_page(struct address_space *mapping,
2667 : loff_t from, get_block_t *get_block)
2668 : {
2669 34 : pgoff_t index = from >> PAGE_SHIFT;
2670 34 : unsigned blocksize;
2671 34 : sector_t iblock;
2672 34 : size_t offset, length, pos;
2673 34 : struct inode *inode = mapping->host;
2674 34 : struct folio *folio;
2675 34 : struct buffer_head *bh;
2676 34 : int err = 0;
2677 :
2678 34 : blocksize = i_blocksize(inode);
2679 34 : length = from & (blocksize - 1);
2680 :
2681 : /* Block boundary? Nothing to do */
2682 34 : if (!length)
2683 : return 0;
2684 :
2685 32 : length = blocksize - length;
2686 32 : iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2687 :
2688 32 : folio = filemap_grab_folio(mapping, index);
2689 32 : if (IS_ERR(folio))
2690 0 : return PTR_ERR(folio);
2691 :
2692 32 : bh = folio_buffers(folio);
2693 32 : if (!bh) {
2694 27 : folio_create_empty_buffers(folio, blocksize, 0);
2695 27 : bh = folio_buffers(folio);
2696 : }
2697 :
2698 : /* Find the buffer that contains "offset" */
2699 32 : offset = offset_in_folio(folio, from);
2700 32 : pos = blocksize;
2701 1284 : while (offset >= pos) {
2702 1252 : bh = bh->b_this_page;
2703 1252 : iblock++;
2704 1252 : pos += blocksize;
2705 : }
2706 :
2707 64 : if (!buffer_mapped(bh)) {
2708 28 : WARN_ON(bh->b_size != blocksize);
2709 28 : err = get_block(inode, iblock, bh, 0);
2710 28 : if (err)
2711 0 : goto unlock;
2712 : /* unmapped? It's a hole - nothing to do */
2713 56 : if (!buffer_mapped(bh))
2714 27 : goto unlock;
2715 : }
2716 :
2717 : /* Ok, it's mapped. Make sure it's up-to-date */
2718 5 : if (folio_test_uptodate(folio))
2719 0 : set_buffer_uptodate(bh);
2720 :
2721 12 : if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2722 1 : err = bh_read(bh, 0);
2723 : /* Uhhuh. Read error. Complain and punt. */
2724 1 : if (err < 0)
2725 0 : goto unlock;
2726 : }
2727 :
2728 5 : folio_zero_range(folio, offset, length);
2729 5 : mark_buffer_dirty(bh);
2730 :
2731 32 : unlock:
2732 32 : folio_unlock(folio);
2733 32 : folio_put(folio);
2734 :
2735 32 : return err;
2736 : }
2737 : EXPORT_SYMBOL(block_truncate_page);
2738 :
2739 : /*
2740 : * The generic ->writepage function for buffer-backed address_spaces
2741 : */
2742 159330 : int block_write_full_page(struct page *page, get_block_t *get_block,
2743 : struct writeback_control *wbc)
2744 : {
2745 159330 : struct folio *folio = page_folio(page);
2746 159330 : struct inode * const inode = folio->mapping->host;
2747 159330 : loff_t i_size = i_size_read(inode);
2748 :
2749 : /* Is the folio fully inside i_size? */
2750 159330 : if (folio_pos(folio) + folio_size(folio) <= i_size)
2751 159277 : return __block_write_full_folio(inode, folio, get_block, wbc,
2752 : end_buffer_async_write);
2753 :
2754 : /* Is the folio fully outside i_size? (truncate in progress) */
2755 53 : if (folio_pos(folio) >= i_size) {
2756 0 : folio_unlock(folio);
2757 0 : return 0; /* don't care */
2758 : }
2759 :
2760 : /*
2761 : * The folio straddles i_size. It must be zeroed out on each and every
2762 : * writepage invocation because it may be mmapped. "A file is mapped
2763 : * in multiples of the page size. For a file that is not a multiple of
2764 : * the page size, the remaining memory is zeroed when mapped, and
2765 : * writes to that region are not written out to the file."
2766 : */
2767 53 : folio_zero_segment(folio, offset_in_folio(folio, i_size),
2768 : folio_size(folio));
2769 53 : return __block_write_full_folio(inode, folio, get_block, wbc,
2770 : end_buffer_async_write);
2771 : }
2772 : EXPORT_SYMBOL(block_write_full_page);
2773 :
2774 0 : sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2775 : get_block_t *get_block)
2776 : {
2777 0 : struct inode *inode = mapping->host;
2778 0 : struct buffer_head tmp = {
2779 0 : .b_size = i_blocksize(inode),
2780 : };
2781 :
2782 0 : get_block(inode, block, &tmp, 0);
2783 0 : return tmp.b_blocknr;
2784 : }
2785 : EXPORT_SYMBOL(generic_block_bmap);
2786 :
2787 376770 : static void end_bio_bh_io_sync(struct bio *bio)
2788 : {
2789 376770 : struct buffer_head *bh = bio->bi_private;
2790 :
2791 376770 : if (unlikely(bio_flagged(bio, BIO_QUIET)))
2792 0 : set_bit(BH_Quiet, &bh->b_state);
2793 :
2794 376770 : bh->b_end_io(bh, !bio->bi_status);
2795 376770 : bio_put(bio);
2796 376770 : }
2797 :
2798 376770 : static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
2799 : struct writeback_control *wbc)
2800 : {
2801 376770 : const enum req_op op = opf & REQ_OP_MASK;
2802 376770 : struct bio *bio;
2803 :
2804 753540 : BUG_ON(!buffer_locked(bh));
2805 753540 : BUG_ON(!buffer_mapped(bh));
2806 376770 : BUG_ON(!bh->b_end_io);
2807 753540 : BUG_ON(buffer_delay(bh));
2808 753540 : BUG_ON(buffer_unwritten(bh));
2809 :
2810 : /*
2811 : * Only clear out a write error when rewriting
2812 : */
2813 753540 : if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
2814 103419 : clear_buffer_write_io_error(bh);
2815 :
2816 753540 : if (buffer_meta(bh))
2817 0 : opf |= REQ_META;
2818 753540 : if (buffer_prio(bh))
2819 0 : opf |= REQ_PRIO;
2820 :
2821 376770 : bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);
2822 :
2823 376770 : fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
2824 :
2825 376770 : bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2826 :
2827 376770 : __bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
2828 :
2829 376770 : bio->bi_end_io = end_bio_bh_io_sync;
2830 376770 : bio->bi_private = bh;
2831 :
2832 : /* Take care of bh's that straddle the end of the device */
2833 376770 : guard_bio_eod(bio);
2834 :
2835 376770 : if (wbc) {
2836 173995 : wbc_init_bio(wbc, bio);
2837 173995 : wbc_account_cgroup_owner(wbc, bh->b_page, bh->b_size);
2838 : }
2839 :
2840 376770 : submit_bio(bio);
2841 376770 : }
2842 :
2843 2 : void submit_bh(blk_opf_t opf, struct buffer_head *bh)
2844 : {
2845 92401 : submit_bh_wbc(opf, bh, NULL);
2846 92592 : }
2847 : EXPORT_SYMBOL(submit_bh);
2848 :
2849 216 : void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
2850 : {
2851 216 : lock_buffer(bh);
2852 432 : if (!test_clear_buffer_dirty(bh)) {
2853 0 : unlock_buffer(bh);
2854 0 : return;
2855 : }
2856 216 : bh->b_end_io = end_buffer_write_sync;
2857 216 : get_bh(bh);
2858 216 : submit_bh(REQ_OP_WRITE | op_flags, bh);
2859 : }
2860 : EXPORT_SYMBOL(write_dirty_buffer);
2861 :
2862 : /*
2863 : * For a data-integrity writeout, we need to wait upon any in-progress I/O
2864 : * and then start new I/O and then wait upon it. The caller must have a ref on
2865 : * the buffer_head.
2866 : */
2867 668 : int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
2868 : {
2869 668 : WARN_ON(atomic_read(&bh->b_count) < 1);
2870 668 : lock_buffer(bh);
2871 1336 : if (test_clear_buffer_dirty(bh)) {
2872 : /*
2873 : * The bh should be mapped, but it might not be if the
2874 : * device was hot-removed. Not much we can do but fail the I/O.
2875 : */
2876 1336 : if (!buffer_mapped(bh)) {
2877 0 : unlock_buffer(bh);
2878 0 : return -EIO;
2879 : }
2880 :
2881 668 : get_bh(bh);
2882 668 : bh->b_end_io = end_buffer_write_sync;
2883 668 : submit_bh(REQ_OP_WRITE | op_flags, bh);
2884 668 : wait_on_buffer(bh);
2885 1336 : if (!buffer_uptodate(bh))
2886 0 : return -EIO;
2887 : } else {
2888 0 : unlock_buffer(bh);
2889 : }
2890 : return 0;
2891 : }
2892 : EXPORT_SYMBOL(__sync_dirty_buffer);
2893 :
2894 668 : int sync_dirty_buffer(struct buffer_head *bh)
2895 : {
2896 668 : return __sync_dirty_buffer(bh, REQ_SYNC);
2897 : }
2898 : EXPORT_SYMBOL(sync_dirty_buffer);
2899 :
2900 : /*
2901 : * try_to_free_buffers() checks if all the buffers on this particular folio
2902 : * are unused, and releases them if so.
2903 : *
2904 : * Exclusion against try_to_free_buffers may be obtained by either
2905 : * locking the folio or by holding its mapping's private_lock.
2906 : *
2907 : * If the folio is dirty but all the buffers are clean then we need to
2908 : * be sure to mark the folio clean as well. This is because the folio
2909 : * may be against a block device, and a later reattachment of buffers
2910 : * to a dirty folio will set *all* buffers dirty. Which would corrupt
2911 : * filesystem data on the same device.
2912 : *
2913 : * The same applies to regular filesystem folios: if all the buffers are
2914 : * clean then we set the folio clean and proceed. To do that, we require
2915 : * total exclusion from block_dirty_folio(). That is obtained with
2916 : * private_lock.
2917 : *
2918 : * try_to_free_buffers() is non-blocking.
2919 : */
2920 : static inline int buffer_busy(struct buffer_head *bh)
2921 : {
2922 349674 : return atomic_read(&bh->b_count) |
2923 349674 : (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2924 : }
2925 :
2926 : static bool
2927 174909 : drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free)
2928 : {
2929 174909 : struct buffer_head *head = folio_buffers(folio);
2930 174909 : struct buffer_head *bh;
2931 :
2932 174909 : bh = head;
2933 349674 : do {
2934 349674 : if (buffer_busy(bh))
2935 0 : goto failed;
2936 349674 : bh = bh->b_this_page;
2937 349674 : } while (bh != head);
2938 :
2939 349674 : do {
2940 349674 : struct buffer_head *next = bh->b_this_page;
2941 :
2942 349674 : if (bh->b_assoc_map)
2943 0 : __remove_assoc_queue(bh);
2944 349674 : bh = next;
2945 349674 : } while (bh != head);
2946 174909 : *buffers_to_free = head;
2947 174909 : folio_detach_private(folio);
2948 174909 : return true;
2949 : failed:
2950 0 : return false;
2951 : }
2952 :
2953 174909 : bool try_to_free_buffers(struct folio *folio)
2954 : {
2955 174909 : struct address_space * const mapping = folio->mapping;
2956 174909 : struct buffer_head *buffers_to_free = NULL;
2957 174909 : bool ret = 0;
2958 :
2959 174909 : BUG_ON(!folio_test_locked(folio));
2960 174909 : if (folio_test_writeback(folio))
2961 : return false;
2962 :
2963 174909 : if (mapping == NULL) { /* can this still happen? */
2964 0 : ret = drop_buffers(folio, &buffers_to_free);
2965 0 : goto out;
2966 : }
2967 :
2968 174909 : spin_lock(&mapping->private_lock);
2969 174909 : ret = drop_buffers(folio, &buffers_to_free);
2970 :
2971 : /*
2972 : * If the filesystem writes its buffers by hand (eg ext3)
2973 : * then we can have clean buffers against a dirty folio. We
2974 : * clean the folio here; otherwise the VM will never notice
2975 : * that the filesystem did any IO at all.
2976 : *
2977 : * Also, during truncate, discard_buffer will have marked all
2978 : * the folio's buffers clean. We discover that here and clean
2979 : * the folio also.
2980 : *
2981 : * private_lock must be held over this entire operation in order
2982 : * to synchronise against block_dirty_folio and prevent the
2983 : * dirty bit from being lost.
2984 : */
2985 174909 : if (ret)
2986 174909 : folio_cancel_dirty(folio);
2987 174909 : spin_unlock(&mapping->private_lock);
2988 174909 : out:
2989 174909 : if (buffers_to_free) {
2990 : struct buffer_head *bh = buffers_to_free;
2991 :
2992 349674 : do {
2993 349674 : struct buffer_head *next = bh->b_this_page;
2994 349674 : free_buffer_head(bh);
2995 349674 : bh = next;
2996 349674 : } while (bh != buffers_to_free);
2997 : }
2998 : return ret;
2999 : }
3000 : EXPORT_SYMBOL(try_to_free_buffers);
3001 :
3002 : /*
3003 : * Buffer-head allocation
3004 : */
3005 : static struct kmem_cache *bh_cachep __read_mostly;
3006 :
3007 : /*
3008 : * Once the number of bh's in the machine exceeds this level, we start
3009 : * stripping them in writeback.
3010 : */
3011 : static unsigned long max_buffer_heads;
3012 :
3013 : int buffer_heads_over_limit;
3014 :
3015 : struct bh_accounting {
3016 : int nr; /* Number of live bh's */
3017 : int ratelimit; /* Limit cacheline bouncing */
3018 : };
3019 :
3020 : static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3021 :
3022 699348 : static void recalc_bh_state(void)
3023 : {
3024 699348 : int i;
3025 699348 : int tot = 0;
3026 :
3027 699348 : if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
3028 : return;
3029 167 : __this_cpu_write(bh_accounting.ratelimit, 0);
3030 501 : for_each_online_cpu(i)
3031 334 : tot += per_cpu(bh_accounting, i).nr;
3032 167 : buffer_heads_over_limit = (tot > max_buffer_heads);
3033 : }
3034 :
3035 349674 : struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3036 : {
3037 349674 : struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3038 349674 : if (ret) {
3039 349674 : INIT_LIST_HEAD(&ret->b_assoc_buffers);
3040 349674 : spin_lock_init(&ret->b_uptodate_lock);
3041 349674 : preempt_disable();
3042 349674 : __this_cpu_inc(bh_accounting.nr);
3043 349674 : recalc_bh_state();
3044 349674 : preempt_enable();
3045 : }
3046 349674 : return ret;
3047 : }
3048 : EXPORT_SYMBOL(alloc_buffer_head);
3049 :
3050 349674 : void free_buffer_head(struct buffer_head *bh)
3051 : {
3052 349674 : BUG_ON(!list_empty(&bh->b_assoc_buffers));
3053 349674 : kmem_cache_free(bh_cachep, bh);
3054 349674 : preempt_disable();
3055 349674 : __this_cpu_dec(bh_accounting.nr);
3056 349674 : recalc_bh_state();
3057 349674 : preempt_enable();
3058 349674 : }
3059 : EXPORT_SYMBOL(free_buffer_head);
3060 :
3061 9 : static int buffer_exit_cpu_dead(unsigned int cpu)
3062 : {
3063 9 : int i;
3064 9 : struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3065 :
3066 153 : for (i = 0; i < BH_LRU_SIZE; i++) {
3067 144 : brelse(b->bhs[i]);
3068 144 : b->bhs[i] = NULL;
3069 : }
3070 9 : this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
3071 9 : per_cpu(bh_accounting, cpu).nr = 0;
3072 9 : return 0;
3073 : }
3074 :
3075 : /**
3076 : * bh_uptodate_or_lock - Test whether the buffer is uptodate
3077 : * @bh: struct buffer_head
3078 : *
3079 : * Return true if the buffer is up-to-date and false,
3080 : * with the buffer locked, if not.
3081 : */
3082 142539 : int bh_uptodate_or_lock(struct buffer_head *bh)
3083 : {
3084 285078 : if (!buffer_uptodate(bh)) {
3085 109481 : lock_buffer(bh);
3086 218962 : if (!buffer_uptodate(bh))
3087 : return 0;
3088 0 : unlock_buffer(bh);
3089 : }
3090 : return 1;
3091 : }
3092 : EXPORT_SYMBOL(bh_uptodate_or_lock);
3093 :
3094 : /**
3095 : * __bh_read - Submit read for a locked buffer
3096 : * @bh: struct buffer_head
3097 : * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
3098 : * @wait: wait until reading finish
3099 : *
3100 : * Returns zero on success or don't wait, and -EIO on error.
3101 : */
3102 109490 : int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait)
3103 : {
3104 109490 : int ret = 0;
3105 :
3106 218980 : BUG_ON(!buffer_locked(bh));
3107 :
3108 109490 : get_bh(bh);
3109 109490 : bh->b_end_io = end_buffer_read_sync;
3110 109490 : submit_bh(REQ_OP_READ | op_flags, bh);
3111 109490 : if (wait) {
3112 9 : wait_on_buffer(bh);
3113 18 : if (!buffer_uptodate(bh))
3114 0 : ret = -EIO;
3115 : }
3116 109490 : return ret;
3117 : }
3118 : EXPORT_SYMBOL(__bh_read);
3119 :
3120 : /**
3121 : * __bh_read_batch - Submit read for a batch of unlocked buffers
3122 : * @nr: entry number of the buffer batch
3123 : * @bhs: a batch of struct buffer_head
3124 : * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
3125 : * @force_lock: force to get a lock on the buffer if set, otherwise drops any
3126 : * buffer that cannot lock.
3127 : *
3128 : * Returns zero on success or don't wait, and -EIO on error.
3129 : */
3130 0 : void __bh_read_batch(int nr, struct buffer_head *bhs[],
3131 : blk_opf_t op_flags, bool force_lock)
3132 : {
3133 0 : int i;
3134 :
3135 0 : for (i = 0; i < nr; i++) {
3136 0 : struct buffer_head *bh = bhs[i];
3137 :
3138 0 : if (buffer_uptodate(bh))
3139 0 : continue;
3140 :
3141 0 : if (force_lock)
3142 0 : lock_buffer(bh);
3143 : else
3144 0 : if (!trylock_buffer(bh))
3145 0 : continue;
3146 :
3147 0 : if (buffer_uptodate(bh)) {
3148 0 : unlock_buffer(bh);
3149 0 : continue;
3150 : }
3151 :
3152 0 : bh->b_end_io = end_buffer_read_sync;
3153 0 : get_bh(bh);
3154 0 : submit_bh(REQ_OP_READ | op_flags, bh);
3155 : }
3156 0 : }
3157 : EXPORT_SYMBOL(__bh_read_batch);
3158 :
3159 0 : void __init buffer_init(void)
3160 : {
3161 0 : unsigned long nrpages;
3162 0 : int ret;
3163 :
3164 0 : bh_cachep = kmem_cache_create("buffer_head",
3165 : sizeof(struct buffer_head), 0,
3166 : (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3167 : SLAB_MEM_SPREAD),
3168 : NULL);
3169 :
3170 : /*
3171 : * Limit the bh occupancy to 10% of ZONE_NORMAL
3172 : */
3173 0 : nrpages = (nr_free_buffer_pages() * 10) / 100;
3174 0 : max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3175 0 : ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
3176 : NULL, buffer_exit_cpu_dead);
3177 0 : WARN_ON(ret < 0);
3178 0 : }
|
