LCOV - code coverage report
Current view: top level - fs/iomap - direct-io.c (source / functions) Hit Total Coverage
Test: fstests of 6.5.0-rc3-achx @ Mon Jul 31 20:08:12 PDT 2023 Lines: 265 298 88.9 %
Date: 2023-07-31 20:08:12 Functions: 11 12 91.7 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * Copyright (C) 2010 Red Hat, Inc.
       4             :  * Copyright (c) 2016-2021 Christoph Hellwig.
       5             :  */
       6             : #include <linux/module.h>
       7             : #include <linux/compiler.h>
       8             : #include <linux/fs.h>
       9             : #include <linux/fscrypt.h>
      10             : #include <linux/pagemap.h>
      11             : #include <linux/iomap.h>
      12             : #include <linux/backing-dev.h>
      13             : #include <linux/uio.h>
      14             : #include <linux/task_io_accounting_ops.h>
      15             : #include "trace.h"
      16             : 
      17             : #include "../internal.h"
      18             : 
      19             : /*
      20             :  * Private flags for iomap_dio, must not overlap with the public ones in
      21             :  * iomap.h:
      22             :  */
      23             : #define IOMAP_DIO_WRITE_FUA     (1 << 28)
      24             : #define IOMAP_DIO_NEED_SYNC     (1 << 29)
      25             : #define IOMAP_DIO_WRITE         (1 << 30)
      26             : #define IOMAP_DIO_DIRTY         (1 << 31)
      27             : 
      28             : struct iomap_dio {
      29             :         struct kiocb            *iocb;
      30             :         const struct iomap_dio_ops *dops;
      31             :         loff_t                  i_size;
      32             :         loff_t                  size;
      33             :         atomic_t                ref;
      34             :         unsigned                flags;
      35             :         int                     error;
      36             :         size_t                  done_before;
      37             :         bool                    wait_for_completion;
      38             : 
      39             :         union {
      40             :                 /* used during submission and for synchronous completion: */
      41             :                 struct {
      42             :                         struct iov_iter         *iter;
      43             :                         struct task_struct      *waiter;
      44             :                         struct bio              *poll_bio;
      45             :                 } submit;
      46             : 
      47             :                 /* used for aio completion: */
      48             :                 struct {
      49             :                         struct work_struct      work;
      50             :                 } aio;
      51             :         };
      52             : };
      53             : 
      54    37561890 : static struct bio *iomap_dio_alloc_bio(const struct iomap_iter *iter,
      55             :                 struct iomap_dio *dio, unsigned short nr_vecs, blk_opf_t opf)
      56             : {
      57    37561890 :         if (dio->dops && dio->dops->bio_set)
      58     1198976 :                 return bio_alloc_bioset(iter->iomap.bdev, nr_vecs, opf,
      59             :                                         GFP_KERNEL, dio->dops->bio_set);
      60    36362914 :         return bio_alloc(iter->iomap.bdev, nr_vecs, opf, GFP_KERNEL);
      61             : }
      62             : 
      63    37561845 : static void iomap_dio_submit_bio(const struct iomap_iter *iter,
      64             :                 struct iomap_dio *dio, struct bio *bio, loff_t pos)
      65             : {
      66    37561845 :         atomic_inc(&dio->ref);
      67             : 
      68             :         /* Sync dio can't be polled reliably */
      69    37565391 :         if ((dio->iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(dio->iocb)) {
      70           0 :                 bio_set_polled(bio, dio->iocb);
      71           0 :                 dio->submit.poll_bio = bio;
      72             :         }
      73             : 
      74    37565391 :         if (dio->dops && dio->dops->submit_io)
      75     1198550 :                 dio->dops->submit_io(iter, bio, pos);
      76             :         else
      77    36366841 :                 submit_bio(bio);
      78    37561488 : }
      79             : 
      80    35306073 : ssize_t iomap_dio_complete(struct iomap_dio *dio)
      81             : {
      82    35306073 :         const struct iomap_dio_ops *dops = dio->dops;
      83    35306073 :         struct kiocb *iocb = dio->iocb;
      84    35306073 :         loff_t offset = iocb->ki_pos;
      85    35306073 :         ssize_t ret = dio->error;
      86             : 
      87    35306073 :         if (dops && dops->end_io)
      88    20702942 :                 ret = dops->end_io(iocb, dio->size, ret, dio->flags);
      89             : 
      90    35304824 :         if (likely(!ret)) {
      91    32361074 :                 ret = dio->size;
      92             :                 /* check for short read */
      93    32361074 :                 if (offset + ret > dio->i_size &&
      94     5798674 :                     !(dio->flags & IOMAP_DIO_WRITE))
      95      318476 :                         ret = dio->i_size - offset;
      96             :         }
      97             : 
      98             :         /*
      99             :          * Try again to invalidate clean pages which might have been cached by
     100             :          * non-direct readahead, or faulted in by get_user_pages() if the source
     101             :          * of the write was an mmap'ed region of the file we're writing.  Either
     102             :          * one is a pretty crazy thing to do, so we don't support it 100%.  If
     103             :          * this invalidation fails, tough, the write still worked...
     104             :          *
     105             :          * And this page cache invalidation has to be after ->end_io(), as some
     106             :          * filesystems convert unwritten extents to real allocations in
     107             :          * ->end_io() when necessary, otherwise a racing buffer read would cache
     108             :          * zeros from unwritten extents.
     109             :          */
     110    35304824 :         if (!dio->error && dio->size && (dio->flags & IOMAP_DIO_WRITE))
     111    18463181 :                 kiocb_invalidate_post_direct_write(iocb, dio->size);
     112             : 
     113    35305055 :         inode_dio_end(file_inode(iocb->ki_filp));
     114             : 
     115    35307717 :         if (ret > 0) {
     116    31671297 :                 iocb->ki_pos += ret;
     117             : 
     118             :                 /*
     119             :                  * If this is a DSYNC write, make sure we push it to stable
     120             :                  * storage now that we've written data.
     121             :                  */
     122    31671297 :                 if (dio->flags & IOMAP_DIO_NEED_SYNC)
     123      136096 :                         ret = generic_write_sync(iocb, ret);
     124    31671293 :                 if (ret > 0)
     125    31671293 :                         ret += dio->done_before;
     126             :         }
     127    35307713 :         trace_iomap_dio_complete(iocb, dio->error, ret);
     128    35305297 :         kfree(dio);
     129    35304282 :         return ret;
     130             : }
     131             : EXPORT_SYMBOL_GPL(iomap_dio_complete);
     132             : 
     133     8072563 : static void iomap_dio_complete_work(struct work_struct *work)
     134             : {
     135     8072563 :         struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
     136     8072563 :         struct kiocb *iocb = dio->iocb;
     137             : 
     138     8072563 :         iocb->ki_complete(iocb, iomap_dio_complete(dio));
     139     8072552 : }
     140             : 
     141             : /*
     142             :  * Set an error in the dio if none is set yet.  We have to use cmpxchg
     143             :  * as the submission context and the completion context(s) can race to
     144             :  * update the error.
     145             :  */
     146             : static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
     147             : {
     148       10677 :         cmpxchg(&dio->error, 0, ret);
     149     2947210 : }
     150             : 
     151    37567959 : void iomap_dio_bio_end_io(struct bio *bio)
     152             : {
     153    37567959 :         struct iomap_dio *dio = bio->bi_private;
     154    37567959 :         bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
     155             : 
     156    37567959 :         if (bio->bi_status)
     157       10677 :                 iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
     158             : 
     159    37567959 :         if (atomic_dec_and_test(&dio->ref)) {
     160    25363623 :                 if (dio->wait_for_completion) {
     161    17291062 :                         struct task_struct *waiter = dio->submit.waiter;
     162    17291062 :                         WRITE_ONCE(dio->submit.waiter, NULL);
     163    17291062 :                         blk_wake_io_task(waiter);
     164     8072561 :                 } else if (dio->flags & IOMAP_DIO_WRITE) {
     165     3858845 :                         struct inode *inode = file_inode(dio->iocb->ki_filp);
     166             : 
     167     3858845 :                         WRITE_ONCE(dio->iocb->private, NULL);
     168     3858845 :                         INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
     169     3858845 :                         queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
     170             :                 } else {
     171     4213716 :                         WRITE_ONCE(dio->iocb->private, NULL);
     172     4213716 :                         iomap_dio_complete_work(&dio->aio.work);
     173             :                 }
     174             :         }
     175             : 
     176    37567923 :         if (should_dirty) {
     177     5848649 :                 bio_check_pages_dirty(bio);
     178             :         } else {
     179    31719274 :                 bio_release_pages(bio, false);
     180    31719274 :                 bio_put(bio);
     181             :         }
     182    37567936 : }
     183             : EXPORT_SYMBOL_GPL(iomap_dio_bio_end_io);
     184             : 
     185     6729143 : static void iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio,
     186             :                 loff_t pos, unsigned len)
     187             : {
     188     6729143 :         struct inode *inode = file_inode(dio->iocb->ki_filp);
     189     6729143 :         struct page *page = ZERO_PAGE(0);
     190     6729143 :         struct bio *bio;
     191             : 
     192     6729143 :         bio = iomap_dio_alloc_bio(iter, dio, 1, REQ_OP_WRITE | REQ_SYNC | REQ_IDLE);
     193     6729282 :         fscrypt_set_bio_crypt_ctx(bio, inode, pos >> inode->i_blkbits,
     194             :                                   GFP_KERNEL);
     195     6729282 :         bio->bi_iter.bi_sector = iomap_sector(&iter->iomap, pos);
     196     6729282 :         bio->bi_private = dio;
     197     6729282 :         bio->bi_end_io = iomap_dio_bio_end_io;
     198             : 
     199     6729282 :         __bio_add_page(bio, page, len, 0);
     200     6729176 :         iomap_dio_submit_bio(iter, dio, bio, pos);
     201     6728999 : }
     202             : 
     203             : /*
     204             :  * Figure out the bio's operation flags from the dio request, the
     205             :  * mapping, and whether or not we want FUA.  Note that we can end up
     206             :  * clearing the WRITE_FUA flag in the dio request.
     207             :  */
     208             : static inline blk_opf_t iomap_dio_bio_opflags(struct iomap_dio *dio,
     209             :                 const struct iomap *iomap, bool use_fua)
     210             : {
     211    30802196 :         blk_opf_t opflags = REQ_SYNC | REQ_IDLE;
     212             : 
     213    30802196 :         if (!(dio->flags & IOMAP_DIO_WRITE))
     214             :                 return REQ_OP_READ;
     215             : 
     216    22548699 :         opflags |= REQ_OP_WRITE;
     217    22548699 :         if (use_fua)
     218             :                 opflags |= REQ_FUA;
     219             :         else
     220    22548524 :                 dio->flags &= ~IOMAP_DIO_WRITE_FUA;
     221             : 
     222             :         return opflags;
     223             : }
     224             : 
     225    30800939 : static loff_t iomap_dio_bio_iter(const struct iomap_iter *iter,
     226             :                 struct iomap_dio *dio)
     227             : {
     228    30800939 :         const struct iomap *iomap = &iter->iomap;
     229    30800939 :         struct inode *inode = iter->inode;
     230    30800939 :         unsigned int fs_block_size = i_blocksize(inode), pad;
     231    30802541 :         loff_t length = iomap_length(iter);
     232    30802541 :         loff_t pos = iter->pos;
     233    30802541 :         blk_opf_t bio_opf;
     234    30802541 :         struct bio *bio;
     235    30802541 :         bool need_zeroout = false;
     236    30802541 :         bool use_fua = false;
     237    30802541 :         int nr_pages, ret = 0;
     238    30802541 :         size_t copied = 0;
     239    30802541 :         size_t orig_count;
     240             : 
     241    92407334 :         if ((pos | length) & (bdev_logical_block_size(iomap->bdev) - 1) ||
     242    30802541 :             !bdev_iter_is_aligned(iomap->bdev, dio->submit.iter))
     243           0 :                 return -EINVAL;
     244             : 
     245    30802252 :         if (iomap->type == IOMAP_UNWRITTEN) {
     246    10199778 :                 dio->flags |= IOMAP_DIO_UNWRITTEN;
     247    10199778 :                 need_zeroout = true;
     248             :         }
     249             : 
     250    30802252 :         if (iomap->flags & IOMAP_F_SHARED)
     251     4260557 :                 dio->flags |= IOMAP_DIO_COW;
     252             : 
     253    30802252 :         if (iomap->flags & IOMAP_F_NEW) {
     254             :                 need_zeroout = true;
     255    22227971 :         } else if (iomap->type == IOMAP_MAPPED) {
     256             :                 /*
     257             :                  * Use a FUA write if we need datasync semantics, this is a pure
     258             :                  * data IO that doesn't require any metadata updates (including
     259             :                  * after IO completion such as unwritten extent conversion) and
     260             :                  * the underlying device supports FUA. This allows us to avoid
     261             :                  * cache flushes on IO completion.
     262             :                  */
     263    20350852 :                 if (!(iomap->flags & (IOMAP_F_SHARED|IOMAP_F_DIRTY)) &&
     264     5202564 :                     (dio->flags & IOMAP_DIO_WRITE_FUA) && bdev_fua(iomap->bdev))
     265           0 :                         use_fua = true;
     266             :         }
     267             : 
     268             :         /*
     269             :          * Save the original count and trim the iter to just the extent we
     270             :          * are operating on right now.  The iter will be re-expanded once
     271             :          * we are done.
     272             :          */
     273    30802252 :         orig_count = iov_iter_count(dio->submit.iter);
     274    30802252 :         iov_iter_truncate(dio->submit.iter, length);
     275             : 
     276    30802252 :         if (!iov_iter_count(dio->submit.iter))
     277           0 :                 goto out;
     278             : 
     279             :         /*
     280             :          * We can only poll for single bio I/Os.
     281             :          */
     282    30802252 :         if (need_zeroout ||
     283    20350779 :             ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode)))
     284    11086344 :                 dio->iocb->ki_flags &= ~IOCB_HIPRI;
     285             : 
     286    30802252 :         if (need_zeroout) {
     287             :                 /* zero out from the start of the block to the write offset */
     288    10451471 :                 pad = pos & (fs_block_size - 1);
     289    10451471 :                 if (pad)
     290     2883538 :                         iomap_dio_zero(iter, dio, pos - pad, pad);
     291             :         }
     292             : 
     293             :         /*
     294             :          * Set the operation flags early so that bio_iov_iter_get_pages
     295             :          * can set up the page vector appropriately for a ZONE_APPEND
     296             :          * operation.
     297             :          */
     298    30802196 :         bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua);
     299             : 
     300    30802196 :         nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS);
     301    30833263 :         do {
     302    30833263 :                 size_t n;
     303    30833263 :                 if (dio->error) {
     304         891 :                         iov_iter_revert(dio->submit.iter, copied);
     305         891 :                         copied = ret = 0;
     306         891 :                         goto out;
     307             :                 }
     308             : 
     309    30832372 :                 bio = iomap_dio_alloc_bio(iter, dio, nr_pages, bio_opf);
     310    30834693 :                 fscrypt_set_bio_crypt_ctx(bio, inode, pos >> inode->i_blkbits,
     311             :                                           GFP_KERNEL);
     312    30834693 :                 bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
     313    30834693 :                 bio->bi_ioprio = dio->iocb->ki_ioprio;
     314    30834693 :                 bio->bi_private = dio;
     315    30834693 :                 bio->bi_end_io = iomap_dio_bio_end_io;
     316             : 
     317    30834693 :                 ret = bio_iov_iter_get_pages(bio, dio->submit.iter);
     318    30833318 :                 if (unlikely(ret)) {
     319             :                         /*
     320             :                          * We have to stop part way through an IO. We must fall
     321             :                          * through to the sub-block tail zeroing here, otherwise
     322             :                          * this short IO may expose stale data in the tail of
     323             :                          * the block we haven't written data to.
     324             :                          */
     325         943 :                         bio_put(bio);
     326        1873 :                         goto zero_tail;
     327             :                 }
     328             : 
     329    30832375 :                 n = bio->bi_iter.bi_size;
     330    30832375 :                 if (dio->flags & IOMAP_DIO_WRITE) {
     331    22564656 :                         task_io_account_write(n);
     332             :                 } else {
     333     8267719 :                         if (dio->flags & IOMAP_DIO_DIRTY)
     334     5845428 :                                 bio_set_pages_dirty(bio);
     335             :                 }
     336             : 
     337    30834365 :                 dio->size += n;
     338    30834365 :                 copied += n;
     339             : 
     340    30834365 :                 nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter,
     341             :                                                  BIO_MAX_VECS);
     342             :                 /*
     343             :                  * We can only poll for single bio I/Os.
     344             :                  */
     345    25251471 :                 if (nr_pages)
     346       31780 :                         dio->iocb->ki_flags &= ~IOCB_HIPRI;
     347    30832709 :                 iomap_dio_submit_bio(iter, dio, bio, pos);
     348    30830760 :                 pos += n;
     349    30830760 :         } while (nr_pages);
     350             : 
     351             :         /*
     352             :          * We need to zeroout the tail of a sub-block write if the extent type
     353             :          * requires zeroing or the write extends beyond EOF. If we don't zero
     354             :          * the block tail in the latter case, we can expose stale data via mmap
     355             :          * reads of the EOF block.
     356             :          */
     357    30798788 : zero_tail:
     358    30800641 :         if (need_zeroout ||
     359    20350377 :             ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
     360             :                 /* zero out from the end of the write to the end of the block */
     361    11490208 :                 pad = pos & (fs_block_size - 1);
     362    11490208 :                 if (pad)
     363     3845540 :                         iomap_dio_zero(iter, dio, pos, fs_block_size - pad);
     364             :         }
     365    26955101 : out:
     366             :         /* Undo iter limitation to current extent */
     367    30801513 :         iov_iter_reexpand(dio->submit.iter, orig_count - copied);
     368    30801513 :         if (copied)
     369    30798811 :                 return copied;
     370        2702 :         return ret;
     371             : }
     372             : 
     373     9332710 : static loff_t iomap_dio_hole_iter(const struct iomap_iter *iter,
     374             :                 struct iomap_dio *dio)
     375             : {
     376     9332710 :         loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter);
     377             : 
     378     9333478 :         dio->size += length;
     379     9333478 :         if (!length)
     380        2221 :                 return -EFAULT;
     381             :         return length;
     382             : }
     383             : 
     384           0 : static loff_t iomap_dio_inline_iter(const struct iomap_iter *iomi,
     385             :                 struct iomap_dio *dio)
     386             : {
     387           0 :         const struct iomap *iomap = &iomi->iomap;
     388           0 :         struct iov_iter *iter = dio->submit.iter;
     389           0 :         void *inline_data = iomap_inline_data(iomap, iomi->pos);
     390           0 :         loff_t length = iomap_length(iomi);
     391           0 :         loff_t pos = iomi->pos;
     392           0 :         size_t copied;
     393             : 
     394           0 :         if (WARN_ON_ONCE(!iomap_inline_data_valid(iomap)))
     395             :                 return -EIO;
     396             : 
     397           0 :         if (dio->flags & IOMAP_DIO_WRITE) {
     398           0 :                 loff_t size = iomi->inode->i_size;
     399             : 
     400           0 :                 if (pos > size)
     401           0 :                         memset(iomap_inline_data(iomap, size), 0, pos - size);
     402           0 :                 copied = copy_from_iter(inline_data, length, iter);
     403           0 :                 if (copied) {
     404           0 :                         if (pos + copied > size)
     405           0 :                                 i_size_write(iomi->inode, pos + copied);
     406           0 :                         mark_inode_dirty(iomi->inode);
     407             :                 }
     408             :         } else {
     409           0 :                 copied = copy_to_iter(inline_data, length, iter);
     410             :         }
     411           0 :         dio->size += copied;
     412           0 :         if (!copied)
     413             :                 return -EFAULT;
     414           0 :         return copied;
     415             : }
     416             : 
     417    40134941 : static loff_t iomap_dio_iter(const struct iomap_iter *iter,
     418             :                 struct iomap_dio *dio)
     419             : {
     420    40134941 :         switch (iter->iomap.type) {
     421     7514754 :         case IOMAP_HOLE:
     422     7514754 :                 if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
     423             :                         return -EIO;
     424     7514754 :                 return iomap_dio_hole_iter(iter, dio);
     425    12017798 :         case IOMAP_UNWRITTEN:
     426    12017798 :                 if (!(dio->flags & IOMAP_DIO_WRITE))
     427     1818114 :                         return iomap_dio_hole_iter(iter, dio);
     428    10199684 :                 return iomap_dio_bio_iter(iter, dio);
     429    20602343 :         case IOMAP_MAPPED:
     430    20602343 :                 return iomap_dio_bio_iter(iter, dio);
     431           0 :         case IOMAP_INLINE:
     432           0 :                 return iomap_dio_inline_iter(iter, dio);
     433          46 :         case IOMAP_DELALLOC:
     434             :                 /*
     435             :                  * DIO is not serialised against mmap() access at all, and so
     436             :                  * if the page_mkwrite occurs between the writeback and the
     437             :                  * iomap_iter() call in the DIO path, then it will see the
     438             :                  * DELALLOC block that the page-mkwrite allocated.
     439             :                  */
     440          46 :                 pr_warn_ratelimited("Direct I/O collision with buffered writes! File: %pD4 Comm: %.20s\n",
     441             :                                     dio->iocb->ki_filp, current->comm);
     442             :                 return -EIO;
     443             :         default:
     444           0 :                 WARN_ON_ONCE(1);
     445           0 :                 return -EIO;
     446             :         }
     447             : }
     448             : 
     449             : /*
     450             :  * iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
     451             :  * is being issued as AIO or not.  This allows us to optimise pure data writes
     452             :  * to use REQ_FUA rather than requiring generic_write_sync() to issue a
     453             :  * REQ_FLUSH post write. This is slightly tricky because a single request here
     454             :  * can be mapped into multiple disjoint IOs and only a subset of the IOs issued
     455             :  * may be pure data writes. In that case, we still need to do a full data sync
     456             :  * completion.
     457             :  *
     458             :  * When page faults are disabled and @dio_flags includes IOMAP_DIO_PARTIAL,
     459             :  * __iomap_dio_rw can return a partial result if it encounters a non-resident
     460             :  * page in @iter after preparing a transfer.  In that case, the non-resident
     461             :  * pages can be faulted in and the request resumed with @done_before set to the
     462             :  * number of bytes previously transferred.  The request will then complete with
     463             :  * the correct total number of bytes transferred; this is essential for
     464             :  * completing partial requests asynchronously.
     465             :  *
     466             :  * Returns -ENOTBLK In case of a page invalidation invalidation failure for
     467             :  * writes.  The callers needs to fall back to buffered I/O in this case.
     468             :  */
     469             : struct iomap_dio *
     470   558364995 : __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
     471             :                 const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
     472             :                 unsigned int dio_flags, void *private, size_t done_before)
     473             : {
     474   558364995 :         struct inode *inode = file_inode(iocb->ki_filp);
     475   558364995 :         struct iomap_iter iomi = {
     476             :                 .inode          = inode,
     477   558364995 :                 .pos            = iocb->ki_pos,
     478             :                 .len            = iov_iter_count(iter),
     479             :                 .flags          = IOMAP_DIRECT,
     480             :                 .private        = private,
     481             :         };
     482   558364995 :         bool wait_for_completion =
     483   558364995 :                 is_sync_kiocb(iocb) || (dio_flags & IOMAP_DIO_FORCE_WAIT);
     484   558364995 :         struct blk_plug plug;
     485   558364995 :         struct iomap_dio *dio;
     486   558364995 :         loff_t ret = 0;
     487             : 
     488   558364995 :         trace_iomap_dio_rw_begin(iocb, iter, dio_flags, done_before);
     489             : 
     490   558353976 :         if (!iomi.len)
     491             :                 return NULL;
     492             : 
     493   558353976 :         dio = kmalloc(sizeof(*dio), GFP_KERNEL);
     494   558356935 :         if (!dio)
     495             :                 return ERR_PTR(-ENOMEM);
     496             : 
     497   558356935 :         dio->iocb = iocb;
     498   558356935 :         atomic_set(&dio->ref, 1);
     499   558356935 :         dio->size = 0;
     500   558356935 :         dio->i_size = i_size_read(inode);
     501   558356935 :         dio->dops = dops;
     502   558356935 :         dio->error = 0;
     503   558356935 :         dio->flags = 0;
     504   558356935 :         dio->done_before = done_before;
     505             : 
     506   558356935 :         dio->submit.iter = iter;
     507   558356935 :         dio->submit.waiter = current;
     508   558356935 :         dio->submit.poll_bio = NULL;
     509             : 
     510   558356935 :         if (iocb->ki_flags & IOCB_NOWAIT)
     511           5 :                 iomi.flags |= IOMAP_NOWAIT;
     512             : 
     513   558356935 :         if (iov_iter_rw(iter) == READ) {
     514   536967984 :                 if (iomi.pos >= dio->i_size)
     515   523065349 :                         goto out_free_dio;
     516             : 
     517    13902635 :                 if (user_backed_iter(iter))
     518    11308974 :                         dio->flags |= IOMAP_DIO_DIRTY;
     519             : 
     520    13902635 :                 ret = kiocb_write_and_wait(iocb, iomi.len);
     521    13902419 :                 if (ret)
     522         453 :                         goto out_free_dio;
     523             :         } else {
     524    21387505 :                 iomi.flags |= IOMAP_WRITE;
     525    21387505 :                 dio->flags |= IOMAP_DIO_WRITE;
     526             : 
     527    21387505 :                 if (dio_flags & IOMAP_DIO_OVERWRITE_ONLY) {
     528     1719460 :                         ret = -EAGAIN;
     529     1719460 :                         if (iomi.pos >= dio->i_size ||
     530     1719460 :                             iomi.pos + iomi.len > dio->i_size)
     531           0 :                                 goto out_free_dio;
     532     1719460 :                         iomi.flags |= IOMAP_OVERWRITE_ONLY;
     533             :                 }
     534             : 
     535             :                 /* for data sync or sync, we need sync completion processing */
     536    21387505 :                 if (iocb_is_dsync(iocb)) {
     537      136848 :                         dio->flags |= IOMAP_DIO_NEED_SYNC;
     538             : 
     539             :                        /*
     540             :                         * For datasync only writes, we optimistically try
     541             :                         * using FUA for this IO.  Any non-FUA write that
     542             :                         * occurs will clear this flag, hence we know before
     543             :                         * completion whether a cache flush is necessary.
     544             :                         */
     545      136848 :                         if (!(iocb->ki_flags & IOCB_SYNC))
     546           2 :                                 dio->flags |= IOMAP_DIO_WRITE_FUA;
     547             :                 }
     548             : 
     549             :                 /*
     550             :                  * Try to invalidate cache pages for the range we are writing.
     551             :                  * If this invalidation fails, let the caller fall back to
     552             :                  * buffered I/O.
     553             :                  */
     554    21387505 :                 ret = kiocb_invalidate_pages(iocb, iomi.len);
     555    21391192 :                 if (ret) {
     556         536 :                         if (ret != -EAGAIN) {
     557         534 :                                 trace_iomap_dio_invalidate_fail(inode, iomi.pos,
     558             :                                                                 iomi.len);
     559         534 :                                 ret = -ENOTBLK;
     560             :                         }
     561         536 :                         goto out_free_dio;
     562             :                 }
     563             : 
     564    21390656 :                 if (!wait_for_completion && !inode->i_sb->s_dio_done_wq) {
     565        8507 :                         ret = sb_init_dio_done_wq(inode->i_sb);
     566        8530 :                         if (ret < 0)
     567           0 :                                 goto out_free_dio;
     568             :                 }
     569             :         }
     570             : 
     571    35292645 :         inode_dio_begin(inode);
     572             : 
     573    35305473 :         blk_start_plug(&plug);
     574    75439244 :         while ((ret = iomap_iter(&iomi, ops)) > 0) {
     575    40135089 :                 iomi.processed = iomap_dio_iter(&iomi, dio);
     576             : 
     577             :                 /*
     578             :                  * We can only poll for single bio I/Os.
     579             :                  */
     580    40133771 :                 iocb->ki_flags &= ~IOCB_HIPRI;
     581             :         }
     582             : 
     583    35296056 :         blk_finish_plug(&plug);
     584             : 
     585             :         /*
     586             :          * We only report that we've read data up to i_size.
     587             :          * Revert iter to a state corresponding to that as some callers (such
     588             :          * as the splice code) rely on it.
     589             :          */
     590    35302908 :         if (iov_iter_rw(iter) == READ && iomi.pos >= dio->i_size)
     591      499950 :                 iov_iter_revert(iter, iomi.pos - dio->i_size);
     592             : 
     593    35300973 :         if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) {
     594         373 :                 if (!(iocb->ki_flags & IOCB_NOWAIT))
     595         373 :                         wait_for_completion = true;
     596             :                 ret = 0;
     597             :         }
     598             : 
     599             :         /* magic error code to fall back to buffered I/O */
     600    35300600 :         if (ret == -ENOTBLK) {
     601             :                 wait_for_completion = true;
     602             :                 ret = 0;
     603             :         }
     604    34610970 :         if (ret < 0)
     605     2932414 :                 iomap_dio_set_error(dio, ret);
     606             : 
     607             :         /*
     608             :          * If all the writes we issued were FUA, we don't need to flush the
     609             :          * cache on IO completion. Clear the sync flag for this case.
     610             :          */
     611    35305092 :         if (dio->flags & IOMAP_DIO_WRITE_FUA)
     612           0 :                 dio->flags &= ~IOMAP_DIO_NEED_SYNC;
     613             : 
     614    35305092 :         WRITE_ONCE(iocb->private, dio->submit.poll_bio);
     615             : 
     616             :         /*
     617             :          * We are about to drop our additional submission reference, which
     618             :          * might be the last reference to the dio.  There are three different
     619             :          * ways we can progress here:
     620             :          *
     621             :          *  (a) If this is the last reference we will always complete and free
     622             :          *      the dio ourselves.
     623             :          *  (b) If this is not the last reference, and we serve an asynchronous
     624             :          *      iocb, we must never touch the dio after the decrement, the
     625             :          *      I/O completion handler will complete and free it.
     626             :          *  (c) If this is not the last reference, but we serve a synchronous
     627             :          *      iocb, the I/O completion handler will wake us up on the drop
     628             :          *      of the final reference, and we will complete and free it here
     629             :          *      after we got woken by the I/O completion handler.
     630             :          */
     631    35305092 :         dio->wait_for_completion = wait_for_completion;
     632    35305092 :         if (!atomic_dec_and_test(&dio->ref)) {
     633    25362451 :                 if (!wait_for_completion) {
     634     8071713 :                         trace_iomap_dio_rw_queued(inode, iomi.pos, iomi.len);
     635     8071713 :                         return ERR_PTR(-EIOCBQUEUED);
     636             :                 }
     637             : 
     638    51871916 :                 for (;;) {
     639    34581230 :                         set_current_state(TASK_UNINTERRUPTIBLE);
     640    34581328 :                         if (!READ_ONCE(dio->submit.waiter))
     641             :                                 break;
     642             : 
     643    17290686 :                         blk_io_schedule();
     644             :                 }
     645    17290642 :                 __set_current_state(TASK_RUNNING);
     646             :         }
     647             : 
     648             :         return dio;
     649             : 
     650   523066338 : out_free_dio:
     651   523066338 :         kfree(dio);
     652   523066960 :         if (ret)
     653         989 :                 return ERR_PTR(ret);
     654             :         return NULL;
     655             : }
     656             : EXPORT_SYMBOL_GPL(__iomap_dio_rw);
     657             : 
     658             : ssize_t
     659   557662317 : iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
     660             :                 const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
     661             :                 unsigned int dio_flags, void *private, size_t done_before)
     662             : {
     663   557662317 :         struct iomap_dio *dio;
     664             : 
     665   557662317 :         dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private,
     666             :                              done_before);
     667   592282750 :         if (IS_ERR_OR_NULL(dio))
     668   538901590 :                 return PTR_ERR_OR_ZERO(dio);
     669    26690497 :         return iomap_dio_complete(dio);
     670             : }
     671             : EXPORT_SYMBOL_GPL(iomap_dio_rw);

Generated by: LCOV version 1.14