Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * linux/fs/read_write.c
4 : *
5 : * Copyright (C) 1991, 1992 Linus Torvalds
6 : */
7 :
8 : #include <linux/slab.h>
9 : #include <linux/stat.h>
10 : #include <linux/sched/xacct.h>
11 : #include <linux/fcntl.h>
12 : #include <linux/file.h>
13 : #include <linux/uio.h>
14 : #include <linux/fsnotify.h>
15 : #include <linux/security.h>
16 : #include <linux/export.h>
17 : #include <linux/syscalls.h>
18 : #include <linux/pagemap.h>
19 : #include <linux/splice.h>
20 : #include <linux/compat.h>
21 : #include <linux/mount.h>
22 : #include <linux/fs.h>
23 : #include "internal.h"
24 :
25 : #include <linux/uaccess.h>
26 : #include <asm/unistd.h>
27 :
28 : const struct file_operations generic_ro_fops = {
29 : .llseek = generic_file_llseek,
30 : .read_iter = generic_file_read_iter,
31 : .mmap = generic_file_readonly_mmap,
32 : .splice_read = filemap_splice_read,
33 : };
34 :
35 : EXPORT_SYMBOL(generic_ro_fops);
36 :
37 : static inline bool unsigned_offsets(struct file *file)
38 : {
39 60 : return file->f_mode & FMODE_UNSIGNED_OFFSET;
40 : }
41 :
42 : /**
43 : * vfs_setpos - update the file offset for lseek
44 : * @file: file structure in question
45 : * @offset: file offset to seek to
46 : * @maxsize: maximum file size
47 : *
48 : * This is a low-level filesystem helper for updating the file offset to
49 : * the value specified by @offset if the given offset is valid and it is
50 : * not equal to the current file offset.
51 : *
52 : * Return the specified offset on success and -EINVAL on invalid offset.
53 : */
54 127120 : loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
55 : {
56 127120 : if (offset < 0 && !unsigned_offsets(file))
57 : return -EINVAL;
58 96093560 : if (offset > maxsize)
59 : return -EINVAL;
60 :
61 96093558 : if (offset != file->f_pos) {
62 73034076 : file->f_pos = offset;
63 73034076 : file->f_version = 0;
64 : }
65 : return offset;
66 : }
67 : EXPORT_SYMBOL(vfs_setpos);
68 :
69 : /**
70 : * generic_file_llseek_size - generic llseek implementation for regular files
71 : * @file: file structure to seek on
72 : * @offset: file offset to seek to
73 : * @whence: type of seek
74 : * @size: max size of this file in file system
75 : * @eof: offset used for SEEK_END position
76 : *
77 : * This is a variant of generic_file_llseek that allows passing in a custom
78 : * maximum file size and a custom EOF position, for e.g. hashed directories
79 : *
80 : * Synchronization:
81 : * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
82 : * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
83 : * read/writes behave like SEEK_SET against seeks.
84 : */
85 : loff_t
86 105748270 : generic_file_llseek_size(struct file *file, loff_t offset, int whence,
87 : loff_t maxsize, loff_t eof)
88 : {
89 105748270 : switch (whence) {
90 10137275 : case SEEK_END:
91 10137275 : offset += eof;
92 10137275 : break;
93 13529783 : case SEEK_CUR:
94 : /*
95 : * Here we special-case the lseek(fd, 0, SEEK_CUR)
96 : * position-querying operation. Avoid rewriting the "same"
97 : * f_pos value back to the file because a concurrent read(),
98 : * write() or lseek() might have altered it
99 : */
100 13529783 : if (offset == 0)
101 9781844 : return file->f_pos;
102 : /*
103 : * f_lock protects against read/modify/write race with other
104 : * SEEK_CURs. Note that parallel writes and reads behave
105 : * like SEEK_SET.
106 : */
107 3747939 : spin_lock(&file->f_lock);
108 3747953 : offset = vfs_setpos(file, file->f_pos + offset, maxsize);
109 3747953 : spin_unlock(&file->f_lock);
110 3747953 : return offset;
111 0 : case SEEK_DATA:
112 : /*
113 : * In the generic case the entire file is data, so as long as
114 : * offset isn't at the end of the file then the offset is data.
115 : */
116 0 : if ((unsigned long long)offset >= eof)
117 : return -ENXIO;
118 : break;
119 354 : case SEEK_HOLE:
120 : /*
121 : * There is a virtual hole at the end of the file, so as long as
122 : * offset isn't i_size or larger, return i_size.
123 : */
124 354 : if ((unsigned long long)offset >= eof)
125 : return -ENXIO;
126 : offset = eof;
127 : break;
128 : }
129 :
130 92218487 : return vfs_setpos(file, offset, maxsize);
131 : }
132 : EXPORT_SYMBOL(generic_file_llseek_size);
133 :
134 : /**
135 : * generic_file_llseek - generic llseek implementation for regular files
136 : * @file: file structure to seek on
137 : * @offset: file offset to seek to
138 : * @whence: type of seek
139 : *
140 : * This is a generic implemenation of ->llseek useable for all normal local
141 : * filesystems. It just updates the file offset to the value specified by
142 : * @offset and @whence.
143 : */
144 100585369 : loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
145 : {
146 100585369 : struct inode *inode = file->f_mapping->host;
147 :
148 201170812 : return generic_file_llseek_size(file, offset, whence,
149 100585369 : inode->i_sb->s_maxbytes,
150 : i_size_read(inode));
151 : }
152 : EXPORT_SYMBOL(generic_file_llseek);
153 :
154 : /**
155 : * fixed_size_llseek - llseek implementation for fixed-sized devices
156 : * @file: file structure to seek on
157 : * @offset: file offset to seek to
158 : * @whence: type of seek
159 : * @size: size of the file
160 : *
161 : */
162 1653445 : loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
163 : {
164 1653445 : switch (whence) {
165 1653445 : case SEEK_SET: case SEEK_CUR: case SEEK_END:
166 1653445 : return generic_file_llseek_size(file, offset, whence,
167 : size, size);
168 : default:
169 : return -EINVAL;
170 : }
171 : }
172 : EXPORT_SYMBOL(fixed_size_llseek);
173 :
174 : /**
175 : * no_seek_end_llseek - llseek implementation for fixed-sized devices
176 : * @file: file structure to seek on
177 : * @offset: file offset to seek to
178 : * @whence: type of seek
179 : *
180 : */
181 0 : loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
182 : {
183 0 : switch (whence) {
184 0 : case SEEK_SET: case SEEK_CUR:
185 0 : return generic_file_llseek_size(file, offset, whence,
186 : OFFSET_MAX, 0);
187 : default:
188 : return -EINVAL;
189 : }
190 : }
191 : EXPORT_SYMBOL(no_seek_end_llseek);
192 :
193 : /**
194 : * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
195 : * @file: file structure to seek on
196 : * @offset: file offset to seek to
197 : * @whence: type of seek
198 : * @size: maximal offset allowed
199 : *
200 : */
201 0 : loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
202 : {
203 0 : switch (whence) {
204 0 : case SEEK_SET: case SEEK_CUR:
205 0 : return generic_file_llseek_size(file, offset, whence,
206 : size, 0);
207 : default:
208 : return -EINVAL;
209 : }
210 : }
211 : EXPORT_SYMBOL(no_seek_end_llseek_size);
212 :
213 : /**
214 : * noop_llseek - No Operation Performed llseek implementation
215 : * @file: file structure to seek on
216 : * @offset: file offset to seek to
217 : * @whence: type of seek
218 : *
219 : * This is an implementation of ->llseek useable for the rare special case when
220 : * userspace expects the seek to succeed but the (device) file is actually not
221 : * able to perform the seek. In this case you use noop_llseek() instead of
222 : * falling back to the default implementation of ->llseek.
223 : */
224 4576 : loff_t noop_llseek(struct file *file, loff_t offset, int whence)
225 : {
226 4576 : return file->f_pos;
227 : }
228 : EXPORT_SYMBOL(noop_llseek);
229 :
230 0 : loff_t default_llseek(struct file *file, loff_t offset, int whence)
231 : {
232 0 : struct inode *inode = file_inode(file);
233 0 : loff_t retval;
234 :
235 0 : inode_lock(inode);
236 0 : switch (whence) {
237 : case SEEK_END:
238 0 : offset += i_size_read(inode);
239 0 : break;
240 0 : case SEEK_CUR:
241 0 : if (offset == 0) {
242 0 : retval = file->f_pos;
243 0 : goto out;
244 : }
245 0 : offset += file->f_pos;
246 0 : break;
247 0 : case SEEK_DATA:
248 : /*
249 : * In the generic case the entire file is data, so as
250 : * long as offset isn't at the end of the file then the
251 : * offset is data.
252 : */
253 0 : if (offset >= inode->i_size) {
254 0 : retval = -ENXIO;
255 0 : goto out;
256 : }
257 : break;
258 0 : case SEEK_HOLE:
259 : /*
260 : * There is a virtual hole at the end of the file, so
261 : * as long as offset isn't i_size or larger, return
262 : * i_size.
263 : */
264 0 : if (offset >= inode->i_size) {
265 0 : retval = -ENXIO;
266 0 : goto out;
267 : }
268 : offset = inode->i_size;
269 : break;
270 : }
271 0 : retval = -EINVAL;
272 0 : if (offset >= 0 || unsigned_offsets(file)) {
273 0 : if (offset != file->f_pos) {
274 0 : file->f_pos = offset;
275 0 : file->f_version = 0;
276 : }
277 : retval = offset;
278 : }
279 0 : out:
280 0 : inode_unlock(inode);
281 0 : return retval;
282 : }
283 : EXPORT_SYMBOL(default_llseek);
284 :
285 107962512 : loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
286 : {
287 107962512 : if (!(file->f_mode & FMODE_LSEEK))
288 : return -ESPIPE;
289 106616857 : return file->f_op->llseek(file, offset, whence);
290 : }
291 : EXPORT_SYMBOL(vfs_llseek);
292 :
293 107964090 : static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
294 : {
295 107964090 : off_t retval;
296 107964090 : struct fd f = fdget_pos(fd);
297 107963999 : if (!f.file)
298 : return -EBADF;
299 :
300 107962227 : retval = -EINVAL;
301 107962227 : if (whence <= SEEK_MAX) {
302 107962162 : loff_t res = vfs_llseek(f.file, offset, whence);
303 107962162 : retval = res;
304 107962162 : if (res != (loff_t)retval)
305 : retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
306 : }
307 107962375 : fdput_pos(f);
308 107962375 : return retval;
309 : }
310 :
311 215928061 : SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
312 : {
313 107964136 : return ksys_lseek(fd, offset, whence);
314 : }
315 :
316 : #ifdef CONFIG_COMPAT
317 : COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
318 : {
319 : return ksys_lseek(fd, offset, whence);
320 : }
321 : #endif
322 :
323 : #if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT) || \
324 : defined(__ARCH_WANT_SYS_LLSEEK)
325 : SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
326 : unsigned long, offset_low, loff_t __user *, result,
327 : unsigned int, whence)
328 : {
329 : int retval;
330 : struct fd f = fdget_pos(fd);
331 : loff_t offset;
332 :
333 : if (!f.file)
334 : return -EBADF;
335 :
336 : retval = -EINVAL;
337 : if (whence > SEEK_MAX)
338 : goto out_putf;
339 :
340 : offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
341 : whence);
342 :
343 : retval = (int)offset;
344 : if (offset >= 0) {
345 : retval = -EFAULT;
346 : if (!copy_to_user(result, &offset, sizeof(offset)))
347 : retval = 0;
348 : }
349 : out_putf:
350 : fdput_pos(f);
351 : return retval;
352 : }
353 : #endif
354 :
355 1517711417 : int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
356 : {
357 1517711417 : if (unlikely((ssize_t) count < 0))
358 : return -EINVAL;
359 :
360 1517711417 : if (ppos) {
361 946858409 : loff_t pos = *ppos;
362 :
363 946858409 : if (unlikely(pos < 0)) {
364 0 : if (!unsigned_offsets(file))
365 : return -EINVAL;
366 0 : if (count >= -pos) /* both values are in 0..LLONG_MAX */
367 0 : return -EOVERFLOW;
368 946858409 : } else if (unlikely((loff_t) (pos + count) < 0)) {
369 60 : if (!unsigned_offsets(file))
370 60 : return -EINVAL;
371 : }
372 : }
373 :
374 : return security_file_permission(file,
375 : read_write == READ ? MAY_READ : MAY_WRITE);
376 : }
377 : EXPORT_SYMBOL(rw_verify_area);
378 :
379 986523019 : static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
380 : {
381 986523019 : struct kiocb kiocb;
382 986523019 : struct iov_iter iter;
383 986523019 : ssize_t ret;
384 :
385 986523019 : init_sync_kiocb(&kiocb, filp);
386 986533468 : kiocb.ki_pos = (ppos ? *ppos : 0);
387 986533468 : iov_iter_ubuf(&iter, ITER_DEST, buf, len);
388 :
389 986525354 : ret = call_read_iter(filp, &kiocb, &iter);
390 986485287 : BUG_ON(ret == -EIOCBQUEUED);
391 986485287 : if (ppos)
392 423494528 : *ppos = kiocb.ki_pos;
393 986485287 : return ret;
394 : }
395 :
396 0 : static int warn_unsupported(struct file *file, const char *op)
397 : {
398 0 : pr_warn_ratelimited(
399 : "kernel %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
400 : op, file, current->pid, current->comm);
401 0 : return -EINVAL;
402 : }
403 :
404 51114358 : ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
405 : {
406 51114358 : struct kvec iov = {
407 : .iov_base = buf,
408 51114358 : .iov_len = min_t(size_t, count, MAX_RW_COUNT),
409 : };
410 51114358 : struct kiocb kiocb;
411 51114358 : struct iov_iter iter;
412 51114358 : ssize_t ret;
413 :
414 51114358 : if (WARN_ON_ONCE(!(file->f_mode & FMODE_READ)))
415 : return -EINVAL;
416 51114358 : if (!(file->f_mode & FMODE_CAN_READ))
417 : return -EINVAL;
418 : /*
419 : * Also fail if ->read_iter and ->read are both wired up as that
420 : * implies very convoluted semantics.
421 : */
422 51114358 : if (unlikely(!file->f_op->read_iter || file->f_op->read))
423 0 : return warn_unsupported(file, "read");
424 :
425 51114358 : init_sync_kiocb(&kiocb, file);
426 51118570 : kiocb.ki_pos = pos ? *pos : 0;
427 51118570 : iov_iter_kvec(&iter, ITER_DEST, &iov, 1, iov.iov_len);
428 51116282 : ret = file->f_op->read_iter(&kiocb, &iter);
429 51120780 : if (ret > 0) {
430 51120625 : if (pos)
431 51120625 : *pos = kiocb.ki_pos;
432 51120625 : fsnotify_access(file);
433 51120375 : add_rchar(current, ret);
434 : }
435 51120530 : inc_syscr(current);
436 51120530 : return ret;
437 : }
438 :
439 51114178 : ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
440 : {
441 51114178 : ssize_t ret;
442 :
443 51114178 : ret = rw_verify_area(READ, file, pos, count);
444 51114178 : if (ret)
445 : return ret;
446 51114301 : return __kernel_read(file, buf, count, pos);
447 : }
448 : EXPORT_SYMBOL(kernel_read);
449 :
450 1090019085 : ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
451 : {
452 1090019085 : ssize_t ret;
453 :
454 1090019085 : if (!(file->f_mode & FMODE_READ))
455 : return -EBADF;
456 1090019085 : if (!(file->f_mode & FMODE_CAN_READ))
457 : return -EINVAL;
458 1090019085 : if (unlikely(!access_ok(buf, count)))
459 : return -EFAULT;
460 :
461 1090077628 : ret = rw_verify_area(READ, file, pos, count);
462 1090077628 : if (ret)
463 : return ret;
464 1090077588 : if (count > MAX_RW_COUNT)
465 : count = MAX_RW_COUNT;
466 :
467 1090077588 : if (file->f_op->read)
468 103559354 : ret = file->f_op->read(file, buf, count, pos);
469 986518234 : else if (file->f_op->read_iter)
470 986518234 : ret = new_sync_read(file, buf, count, pos);
471 : else
472 : ret = -EINVAL;
473 1089988693 : if (ret > 0) {
474 761842601 : fsnotify_access(file);
475 761848656 : add_rchar(current, ret);
476 : }
477 1089994748 : inc_syscr(current);
478 1089994748 : return ret;
479 : }
480 :
481 342894185 : static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
482 : {
483 342894185 : struct kiocb kiocb;
484 342894185 : struct iov_iter iter;
485 342894185 : ssize_t ret;
486 :
487 342894185 : init_sync_kiocb(&kiocb, filp);
488 342895511 : kiocb.ki_pos = (ppos ? *ppos : 0);
489 342895511 : iov_iter_ubuf(&iter, ITER_SOURCE, (void __user *)buf, len);
490 :
491 342887999 : ret = call_write_iter(filp, &kiocb, &iter);
492 342895206 : BUG_ON(ret == -EIOCBQUEUED);
493 342895206 : if (ret > 0 && ppos)
494 334860410 : *ppos = kiocb.ki_pos;
495 342895206 : return ret;
496 : }
497 :
498 : /* caller is responsible for file_start_write/file_end_write */
499 0 : ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos)
500 : {
501 0 : struct kiocb kiocb;
502 0 : ssize_t ret;
503 :
504 0 : if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE)))
505 : return -EBADF;
506 0 : if (!(file->f_mode & FMODE_CAN_WRITE))
507 : return -EINVAL;
508 : /*
509 : * Also fail if ->write_iter and ->write are both wired up as that
510 : * implies very convoluted semantics.
511 : */
512 0 : if (unlikely(!file->f_op->write_iter || file->f_op->write))
513 0 : return warn_unsupported(file, "write");
514 :
515 0 : init_sync_kiocb(&kiocb, file);
516 0 : kiocb.ki_pos = pos ? *pos : 0;
517 0 : ret = file->f_op->write_iter(&kiocb, from);
518 0 : if (ret > 0) {
519 0 : if (pos)
520 0 : *pos = kiocb.ki_pos;
521 0 : fsnotify_modify(file);
522 0 : add_wchar(current, ret);
523 : }
524 0 : inc_syscw(current);
525 0 : return ret;
526 : }
527 :
528 : /* caller is responsible for file_start_write/file_end_write */
529 0 : ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
530 : {
531 0 : struct kvec iov = {
532 : .iov_base = (void *)buf,
533 0 : .iov_len = min_t(size_t, count, MAX_RW_COUNT),
534 : };
535 0 : struct iov_iter iter;
536 0 : iov_iter_kvec(&iter, ITER_SOURCE, &iov, 1, iov.iov_len);
537 0 : return __kernel_write_iter(file, &iter, pos);
538 : }
539 : /*
540 : * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()",
541 : * but autofs is one of the few internal kernel users that actually
542 : * wants this _and_ can be built as a module. So we need to export
543 : * this symbol for autofs, even though it really isn't appropriate
544 : * for any other kernel modules.
545 : */
546 : EXPORT_SYMBOL_GPL(__kernel_write);
547 :
548 0 : ssize_t kernel_write(struct file *file, const void *buf, size_t count,
549 : loff_t *pos)
550 : {
551 0 : ssize_t ret;
552 :
553 0 : ret = rw_verify_area(WRITE, file, pos, count);
554 0 : if (ret)
555 : return ret;
556 :
557 0 : file_start_write(file);
558 0 : ret = __kernel_write(file, buf, count, pos);
559 0 : file_end_write(file);
560 0 : return ret;
561 : }
562 : EXPORT_SYMBOL(kernel_write);
563 :
564 345478606 : ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
565 : {
566 345478606 : ssize_t ret;
567 :
568 345478606 : if (!(file->f_mode & FMODE_WRITE))
569 : return -EBADF;
570 345478606 : if (!(file->f_mode & FMODE_CAN_WRITE))
571 : return -EINVAL;
572 345478606 : if (unlikely(!access_ok(buf, count)))
573 : return -EFAULT;
574 :
575 345510604 : ret = rw_verify_area(WRITE, file, pos, count);
576 345510604 : if (ret)
577 : return ret;
578 345509402 : if (count > MAX_RW_COUNT)
579 : count = MAX_RW_COUNT;
580 345509402 : file_start_write(file);
581 345509729 : if (file->f_op->write)
582 2615780 : ret = file->f_op->write(file, buf, count, pos);
583 342893949 : else if (file->f_op->write_iter)
584 342893949 : ret = new_sync_write(file, buf, count, pos);
585 : else
586 : ret = -EINVAL;
587 345511526 : if (ret > 0) {
588 344917654 : fsnotify_modify(file);
589 344917315 : add_wchar(current, ret);
590 : }
591 345511187 : inc_syscw(current);
592 345511187 : file_end_write(file);
593 345511187 : return ret;
594 : }
595 :
596 : /* file_ppos returns &file->f_pos or NULL if file is stream */
597 : static inline loff_t *file_ppos(struct file *file)
598 : {
599 1678462853 : return file->f_mode & FMODE_STREAM ? NULL : &file->f_pos;
600 : }
601 :
602 803756074 : ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
603 : {
604 803756074 : struct fd f = fdget_pos(fd);
605 803762796 : ssize_t ret = -EBADF;
606 :
607 803762796 : if (f.file) {
608 803762796 : loff_t pos, *ppos = file_ppos(f.file);
609 240772309 : if (ppos) {
610 240772309 : pos = *ppos;
611 240772309 : ppos = &pos;
612 : }
613 803762796 : ret = vfs_read(f.file, buf, count, ppos);
614 803738662 : if (ret >= 0 && ppos)
615 181991218 : f.file->f_pos = pos;
616 803738662 : fdput_pos(f);
617 : }
618 803739023 : return ret;
619 : }
620 :
621 1607482083 : SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
622 : {
623 803742712 : return ksys_read(fd, buf, count);
624 : }
625 :
626 316901337 : ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
627 : {
628 316901337 : struct fd f = fdget_pos(fd);
629 316902720 : ssize_t ret = -EBADF;
630 :
631 316902720 : if (f.file) {
632 316902720 : loff_t pos, *ppos = file_ppos(f.file);
633 309047181 : if (ppos) {
634 309047181 : pos = *ppos;
635 309047181 : ppos = &pos;
636 : }
637 316902720 : ret = vfs_write(f.file, buf, count, ppos);
638 316890801 : if (ret >= 0 && ppos)
639 308869595 : f.file->f_pos = pos;
640 316890801 : fdput_pos(f);
641 : }
642 :
643 316903194 : return ret;
644 : }
645 :
646 633792322 : SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
647 : size_t, count)
648 : {
649 316901952 : return ksys_write(fd, buf, count);
650 : }
651 :
652 286327171 : ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
653 : loff_t pos)
654 : {
655 286327171 : struct fd f;
656 286327171 : ssize_t ret = -EBADF;
657 :
658 286327171 : if (pos < 0)
659 : return -EINVAL;
660 :
661 286327171 : f = fdget(fd);
662 286318347 : if (f.file) {
663 286318347 : ret = -ESPIPE;
664 286318347 : if (f.file->f_mode & FMODE_PREAD)
665 286324859 : ret = vfs_read(f.file, buf, count, &pos);
666 286320710 : fdput(f);
667 : }
668 :
669 : return ret;
670 : }
671 :
672 572640009 : SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
673 : size_t, count, loff_t, pos)
674 : {
675 286312186 : return ksys_pread64(fd, buf, count, pos);
676 : }
677 :
678 : #if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PREAD64)
679 : COMPAT_SYSCALL_DEFINE5(pread64, unsigned int, fd, char __user *, buf,
680 : size_t, count, compat_arg_u64_dual(pos))
681 : {
682 : return ksys_pread64(fd, buf, count, compat_arg_u64_glue(pos));
683 : }
684 : #endif
685 :
686 28608997 : ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
687 : size_t count, loff_t pos)
688 : {
689 28608997 : struct fd f;
690 28608997 : ssize_t ret = -EBADF;
691 :
692 28608997 : if (pos < 0)
693 : return -EINVAL;
694 :
695 28608997 : f = fdget(fd);
696 28614415 : if (f.file) {
697 28614415 : ret = -ESPIPE;
698 28614415 : if (f.file->f_mode & FMODE_PWRITE)
699 28608891 : ret = vfs_write(f.file, buf, count, &pos);
700 28609415 : fdput(f);
701 : }
702 :
703 : return ret;
704 : }
705 :
706 57218790 : SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
707 : size_t, count, loff_t, pos)
708 : {
709 28609362 : return ksys_pwrite64(fd, buf, count, pos);
710 : }
711 :
712 : #if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PWRITE64)
713 : COMPAT_SYSCALL_DEFINE5(pwrite64, unsigned int, fd, const char __user *, buf,
714 : size_t, count, compat_arg_u64_dual(pos))
715 : {
716 : return ksys_pwrite64(fd, buf, count, compat_arg_u64_glue(pos));
717 : }
718 : #endif
719 :
720 12396898 : static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
721 : loff_t *ppos, int type, rwf_t flags)
722 : {
723 12396898 : struct kiocb kiocb;
724 12396898 : ssize_t ret;
725 :
726 12396898 : init_sync_kiocb(&kiocb, filp);
727 12396942 : ret = kiocb_set_rw_flags(&kiocb, flags);
728 12396962 : if (ret)
729 : return ret;
730 12396962 : kiocb.ki_pos = (ppos ? *ppos : 0);
731 :
732 12396962 : if (type == READ)
733 4940106 : ret = call_read_iter(filp, &kiocb, iter);
734 : else
735 7456856 : ret = call_write_iter(filp, &kiocb, iter);
736 12396976 : BUG_ON(ret == -EIOCBQUEUED);
737 12396976 : if (ppos)
738 12380565 : *ppos = kiocb.ki_pos;
739 : return ret;
740 : }
741 :
742 : /* Do it by hand, with file-ops */
743 0 : static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
744 : loff_t *ppos, int type, rwf_t flags)
745 : {
746 0 : ssize_t ret = 0;
747 :
748 0 : if (flags & ~RWF_HIPRI)
749 : return -EOPNOTSUPP;
750 :
751 0 : while (iov_iter_count(iter)) {
752 0 : ssize_t nr;
753 :
754 0 : if (type == READ) {
755 0 : nr = filp->f_op->read(filp, iter_iov_addr(iter),
756 0 : iter_iov_len(iter), ppos);
757 : } else {
758 0 : nr = filp->f_op->write(filp, iter_iov_addr(iter),
759 0 : iter_iov_len(iter), ppos);
760 : }
761 :
762 0 : if (nr < 0) {
763 0 : if (!ret)
764 0 : ret = nr;
765 : break;
766 : }
767 0 : ret += nr;
768 0 : if (nr != iter_iov_len(iter))
769 : break;
770 0 : iov_iter_advance(iter, nr);
771 : }
772 :
773 : return ret;
774 : }
775 :
776 4940098 : static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,
777 : loff_t *pos, rwf_t flags)
778 : {
779 4940098 : size_t tot_len;
780 4940098 : ssize_t ret = 0;
781 :
782 4940098 : if (!(file->f_mode & FMODE_READ))
783 : return -EBADF;
784 4940098 : if (!(file->f_mode & FMODE_CAN_READ))
785 : return -EINVAL;
786 :
787 4940098 : tot_len = iov_iter_count(iter);
788 4940098 : if (!tot_len)
789 0 : goto out;
790 4940098 : ret = rw_verify_area(READ, file, pos, tot_len);
791 4940098 : if (ret < 0)
792 : return ret;
793 :
794 4940098 : if (file->f_op->read_iter)
795 4940098 : ret = do_iter_readv_writev(file, iter, pos, READ, flags);
796 : else
797 0 : ret = do_loop_readv_writev(file, iter, pos, READ, flags);
798 4940114 : out:
799 4940114 : if (ret >= 0)
800 4940098 : fsnotify_access(file);
801 : return ret;
802 : }
803 :
804 0 : ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
805 : struct iov_iter *iter)
806 : {
807 0 : size_t tot_len;
808 0 : ssize_t ret = 0;
809 :
810 0 : if (!file->f_op->read_iter)
811 : return -EINVAL;
812 0 : if (!(file->f_mode & FMODE_READ))
813 : return -EBADF;
814 0 : if (!(file->f_mode & FMODE_CAN_READ))
815 : return -EINVAL;
816 :
817 0 : tot_len = iov_iter_count(iter);
818 0 : if (!tot_len)
819 0 : goto out;
820 0 : ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len);
821 0 : if (ret < 0)
822 : return ret;
823 :
824 0 : ret = call_read_iter(file, iocb, iter);
825 : out:
826 0 : if (ret >= 0)
827 0 : fsnotify_access(file);
828 : return ret;
829 : }
830 : EXPORT_SYMBOL(vfs_iocb_iter_read);
831 :
832 4256796 : ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
833 : rwf_t flags)
834 : {
835 4256796 : if (!file->f_op->read_iter)
836 : return -EINVAL;
837 4256796 : return do_iter_read(file, iter, ppos, flags);
838 : }
839 : EXPORT_SYMBOL(vfs_iter_read);
840 :
841 7456887 : static ssize_t do_iter_write(struct file *file, struct iov_iter *iter,
842 : loff_t *pos, rwf_t flags)
843 : {
844 7456887 : size_t tot_len;
845 7456887 : ssize_t ret = 0;
846 :
847 7456887 : if (!(file->f_mode & FMODE_WRITE))
848 : return -EBADF;
849 7456887 : if (!(file->f_mode & FMODE_CAN_WRITE))
850 : return -EINVAL;
851 :
852 7456887 : tot_len = iov_iter_count(iter);
853 7456887 : if (!tot_len)
854 : return 0;
855 7456887 : ret = rw_verify_area(WRITE, file, pos, tot_len);
856 7456887 : if (ret < 0)
857 : return ret;
858 :
859 7456887 : if (file->f_op->write_iter)
860 7456887 : ret = do_iter_readv_writev(file, iter, pos, WRITE, flags);
861 : else
862 0 : ret = do_loop_readv_writev(file, iter, pos, WRITE, flags);
863 7456931 : if (ret > 0)
864 7269457 : fsnotify_modify(file);
865 : return ret;
866 : }
867 :
868 0 : ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
869 : struct iov_iter *iter)
870 : {
871 0 : size_t tot_len;
872 0 : ssize_t ret = 0;
873 :
874 0 : if (!file->f_op->write_iter)
875 : return -EINVAL;
876 0 : if (!(file->f_mode & FMODE_WRITE))
877 : return -EBADF;
878 0 : if (!(file->f_mode & FMODE_CAN_WRITE))
879 : return -EINVAL;
880 :
881 0 : tot_len = iov_iter_count(iter);
882 0 : if (!tot_len)
883 : return 0;
884 0 : ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len);
885 0 : if (ret < 0)
886 : return ret;
887 :
888 0 : ret = call_write_iter(file, iocb, iter);
889 0 : if (ret > 0)
890 0 : fsnotify_modify(file);
891 :
892 : return ret;
893 : }
894 : EXPORT_SYMBOL(vfs_iocb_iter_write);
895 :
896 4143093 : ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
897 : rwf_t flags)
898 : {
899 4143093 : if (!file->f_op->write_iter)
900 : return -EINVAL;
901 4143093 : return do_iter_write(file, iter, ppos, flags);
902 : }
903 : EXPORT_SYMBOL(vfs_iter_write);
904 :
905 683307 : static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
906 : unsigned long vlen, loff_t *pos, rwf_t flags)
907 : {
908 683307 : struct iovec iovstack[UIO_FASTIOV];
909 683307 : struct iovec *iov = iovstack;
910 683307 : struct iov_iter iter;
911 683307 : ssize_t ret;
912 :
913 683307 : ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
914 683307 : if (ret >= 0) {
915 683307 : ret = do_iter_read(file, &iter, pos, flags);
916 683307 : kfree(iov);
917 : }
918 :
919 683307 : return ret;
920 : }
921 :
922 3313814 : static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
923 : unsigned long vlen, loff_t *pos, rwf_t flags)
924 : {
925 3313814 : struct iovec iovstack[UIO_FASTIOV];
926 3313814 : struct iovec *iov = iovstack;
927 3313814 : struct iov_iter iter;
928 3313814 : ssize_t ret;
929 :
930 3313814 : ret = import_iovec(ITER_SOURCE, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
931 3313820 : if (ret >= 0) {
932 3313818 : file_start_write(file);
933 3313820 : ret = do_iter_write(file, &iter, pos, flags);
934 3313826 : file_end_write(file);
935 3313823 : kfree(iov);
936 : }
937 3313821 : return ret;
938 : }
939 :
940 683307 : static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
941 : unsigned long vlen, rwf_t flags)
942 : {
943 683307 : struct fd f = fdget_pos(fd);
944 683307 : ssize_t ret = -EBADF;
945 :
946 683307 : if (f.file) {
947 683307 : loff_t pos, *ppos = file_ppos(f.file);
948 675343 : if (ppos) {
949 675343 : pos = *ppos;
950 675343 : ppos = &pos;
951 : }
952 683307 : ret = vfs_readv(f.file, vec, vlen, ppos, flags);
953 683307 : if (ret >= 0 && ppos)
954 675327 : f.file->f_pos = pos;
955 683307 : fdput_pos(f);
956 : }
957 :
958 683307 : if (ret > 0)
959 683291 : add_rchar(current, ret);
960 683307 : inc_syscr(current);
961 683307 : return ret;
962 : }
963 :
964 3313821 : static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
965 : unsigned long vlen, rwf_t flags)
966 : {
967 3313821 : struct fd f = fdget_pos(fd);
968 3313822 : ssize_t ret = -EBADF;
969 :
970 3313822 : if (f.file) {
971 3313822 : loff_t pos, *ppos = file_ppos(f.file);
972 3305375 : if (ppos) {
973 3305375 : pos = *ppos;
974 3305375 : ppos = &pos;
975 : }
976 3313822 : ret = vfs_writev(f.file, vec, vlen, ppos, flags);
977 3313815 : if (ret >= 0 && ppos)
978 3220229 : f.file->f_pos = pos;
979 3313815 : fdput_pos(f);
980 : }
981 :
982 3313813 : if (ret > 0)
983 3228676 : add_wchar(current, ret);
984 3313813 : inc_syscw(current);
985 3313813 : return ret;
986 : }
987 :
988 : static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
989 : {
990 : #define HALF_LONG_BITS (BITS_PER_LONG / 2)
991 0 : return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
992 : }
993 :
994 0 : static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
995 : unsigned long vlen, loff_t pos, rwf_t flags)
996 : {
997 0 : struct fd f;
998 0 : ssize_t ret = -EBADF;
999 :
1000 0 : if (pos < 0)
1001 : return -EINVAL;
1002 :
1003 0 : f = fdget(fd);
1004 0 : if (f.file) {
1005 0 : ret = -ESPIPE;
1006 0 : if (f.file->f_mode & FMODE_PREAD)
1007 0 : ret = vfs_readv(f.file, vec, vlen, &pos, flags);
1008 0 : fdput(f);
1009 : }
1010 :
1011 0 : if (ret > 0)
1012 0 : add_rchar(current, ret);
1013 0 : inc_syscr(current);
1014 0 : return ret;
1015 : }
1016 :
1017 2 : static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
1018 : unsigned long vlen, loff_t pos, rwf_t flags)
1019 : {
1020 2 : struct fd f;
1021 2 : ssize_t ret = -EBADF;
1022 :
1023 2 : if (pos < 0)
1024 : return -EINVAL;
1025 :
1026 2 : f = fdget(fd);
1027 2 : if (f.file) {
1028 2 : ret = -ESPIPE;
1029 2 : if (f.file->f_mode & FMODE_PWRITE)
1030 2 : ret = vfs_writev(f.file, vec, vlen, &pos, flags);
1031 2 : fdput(f);
1032 : }
1033 :
1034 2 : if (ret > 0)
1035 2 : add_wchar(current, ret);
1036 2 : inc_syscw(current);
1037 2 : return ret;
1038 : }
1039 :
1040 1366614 : SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
1041 : unsigned long, vlen)
1042 : {
1043 683307 : return do_readv(fd, vec, vlen, 0);
1044 : }
1045 :
1046 6627633 : SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
1047 : unsigned long, vlen)
1048 : {
1049 3313818 : return do_writev(fd, vec, vlen, 0);
1050 : }
1051 :
1052 0 : SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
1053 : unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1054 : {
1055 0 : loff_t pos = pos_from_hilo(pos_h, pos_l);
1056 :
1057 0 : return do_preadv(fd, vec, vlen, pos, 0);
1058 : }
1059 :
1060 0 : SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
1061 : unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1062 : rwf_t, flags)
1063 : {
1064 0 : loff_t pos = pos_from_hilo(pos_h, pos_l);
1065 :
1066 0 : if (pos == -1)
1067 0 : return do_readv(fd, vec, vlen, flags);
1068 :
1069 0 : return do_preadv(fd, vec, vlen, pos, flags);
1070 : }
1071 :
1072 4 : SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
1073 : unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1074 : {
1075 2 : loff_t pos = pos_from_hilo(pos_h, pos_l);
1076 :
1077 2 : return do_pwritev(fd, vec, vlen, pos, 0);
1078 : }
1079 :
1080 0 : SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
1081 : unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1082 : rwf_t, flags)
1083 : {
1084 0 : loff_t pos = pos_from_hilo(pos_h, pos_l);
1085 :
1086 0 : if (pos == -1)
1087 0 : return do_writev(fd, vec, vlen, flags);
1088 :
1089 0 : return do_pwritev(fd, vec, vlen, pos, flags);
1090 : }
1091 :
1092 : /*
1093 : * Various compat syscalls. Note that they all pretend to take a native
1094 : * iovec - import_iovec will properly treat those as compat_iovecs based on
1095 : * in_compat_syscall().
1096 : */
1097 : #ifdef CONFIG_COMPAT
1098 : #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
1099 : COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1100 : const struct iovec __user *, vec,
1101 : unsigned long, vlen, loff_t, pos)
1102 : {
1103 : return do_preadv(fd, vec, vlen, pos, 0);
1104 : }
1105 : #endif
1106 :
1107 : COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
1108 : const struct iovec __user *, vec,
1109 : compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1110 : {
1111 : loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1112 :
1113 : return do_preadv(fd, vec, vlen, pos, 0);
1114 : }
1115 :
1116 : #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
1117 : COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
1118 : const struct iovec __user *, vec,
1119 : unsigned long, vlen, loff_t, pos, rwf_t, flags)
1120 : {
1121 : if (pos == -1)
1122 : return do_readv(fd, vec, vlen, flags);
1123 : return do_preadv(fd, vec, vlen, pos, flags);
1124 : }
1125 : #endif
1126 :
1127 : COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
1128 : const struct iovec __user *, vec,
1129 : compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
1130 : rwf_t, flags)
1131 : {
1132 : loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1133 :
1134 : if (pos == -1)
1135 : return do_readv(fd, vec, vlen, flags);
1136 : return do_preadv(fd, vec, vlen, pos, flags);
1137 : }
1138 :
1139 : #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
1140 : COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1141 : const struct iovec __user *, vec,
1142 : unsigned long, vlen, loff_t, pos)
1143 : {
1144 : return do_pwritev(fd, vec, vlen, pos, 0);
1145 : }
1146 : #endif
1147 :
1148 : COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
1149 : const struct iovec __user *,vec,
1150 : compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1151 : {
1152 : loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1153 :
1154 : return do_pwritev(fd, vec, vlen, pos, 0);
1155 : }
1156 :
1157 : #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
1158 : COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
1159 : const struct iovec __user *, vec,
1160 : unsigned long, vlen, loff_t, pos, rwf_t, flags)
1161 : {
1162 : if (pos == -1)
1163 : return do_writev(fd, vec, vlen, flags);
1164 : return do_pwritev(fd, vec, vlen, pos, flags);
1165 : }
1166 : #endif
1167 :
1168 : COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
1169 : const struct iovec __user *,vec,
1170 : compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
1171 : {
1172 : loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1173 :
1174 : if (pos == -1)
1175 : return do_writev(fd, vec, vlen, flags);
1176 : return do_pwritev(fd, vec, vlen, pos, flags);
1177 : }
1178 : #endif /* CONFIG_COMPAT */
1179 :
1180 2 : static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1181 : size_t count, loff_t max)
1182 : {
1183 2 : struct fd in, out;
1184 2 : struct inode *in_inode, *out_inode;
1185 2 : struct pipe_inode_info *opipe;
1186 2 : loff_t pos;
1187 2 : loff_t out_pos;
1188 2 : ssize_t retval;
1189 2 : int fl;
1190 :
1191 : /*
1192 : * Get input file, and verify that it is ok..
1193 : */
1194 2 : retval = -EBADF;
1195 2 : in = fdget(in_fd);
1196 2 : if (!in.file)
1197 0 : goto out;
1198 2 : if (!(in.file->f_mode & FMODE_READ))
1199 0 : goto fput_in;
1200 2 : retval = -ESPIPE;
1201 2 : if (!ppos) {
1202 0 : pos = in.file->f_pos;
1203 : } else {
1204 2 : pos = *ppos;
1205 2 : if (!(in.file->f_mode & FMODE_PREAD))
1206 0 : goto fput_in;
1207 : }
1208 2 : retval = rw_verify_area(READ, in.file, &pos, count);
1209 2 : if (retval < 0)
1210 0 : goto fput_in;
1211 2 : if (count > MAX_RW_COUNT)
1212 : count = MAX_RW_COUNT;
1213 :
1214 : /*
1215 : * Get output file, and verify that it is ok..
1216 : */
1217 2 : retval = -EBADF;
1218 2 : out = fdget(out_fd);
1219 2 : if (!out.file)
1220 0 : goto fput_in;
1221 2 : if (!(out.file->f_mode & FMODE_WRITE))
1222 0 : goto fput_out;
1223 2 : in_inode = file_inode(in.file);
1224 2 : out_inode = file_inode(out.file);
1225 2 : out_pos = out.file->f_pos;
1226 :
1227 2 : if (!max)
1228 2 : max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1229 :
1230 2 : if (unlikely(pos + count > max)) {
1231 0 : retval = -EOVERFLOW;
1232 0 : if (pos >= max)
1233 0 : goto fput_out;
1234 0 : count = max - pos;
1235 : }
1236 :
1237 2 : fl = 0;
1238 : #if 0
1239 : /*
1240 : * We need to debate whether we can enable this or not. The
1241 : * man page documents EAGAIN return for the output at least,
1242 : * and the application is arguably buggy if it doesn't expect
1243 : * EAGAIN on a non-blocking file descriptor.
1244 : */
1245 : if (in.file->f_flags & O_NONBLOCK)
1246 : fl = SPLICE_F_NONBLOCK;
1247 : #endif
1248 2 : opipe = get_pipe_info(out.file, true);
1249 2 : if (!opipe) {
1250 2 : retval = rw_verify_area(WRITE, out.file, &out_pos, count);
1251 2 : if (retval < 0)
1252 0 : goto fput_out;
1253 2 : file_start_write(out.file);
1254 2 : retval = do_splice_direct(in.file, &pos, out.file, &out_pos,
1255 : count, fl);
1256 2 : file_end_write(out.file);
1257 : } else {
1258 0 : if (out.file->f_flags & O_NONBLOCK)
1259 0 : fl |= SPLICE_F_NONBLOCK;
1260 :
1261 0 : retval = splice_file_to_pipe(in.file, opipe, &pos, count, fl);
1262 : }
1263 :
1264 2 : if (retval > 0) {
1265 2 : add_rchar(current, retval);
1266 2 : add_wchar(current, retval);
1267 2 : fsnotify_access(in.file);
1268 2 : fsnotify_modify(out.file);
1269 2 : out.file->f_pos = out_pos;
1270 2 : if (ppos)
1271 2 : *ppos = pos;
1272 : else
1273 0 : in.file->f_pos = pos;
1274 : }
1275 :
1276 2 : inc_syscr(current);
1277 2 : inc_syscw(current);
1278 2 : if (pos > max)
1279 0 : retval = -EOVERFLOW;
1280 :
1281 2 : fput_out:
1282 2 : fdput(out);
1283 2 : fput_in:
1284 2 : fdput(in);
1285 2 : out:
1286 2 : return retval;
1287 : }
1288 :
1289 0 : SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
1290 : {
1291 0 : loff_t pos;
1292 0 : off_t off;
1293 0 : ssize_t ret;
1294 :
1295 0 : if (offset) {
1296 0 : if (unlikely(get_user(off, offset)))
1297 : return -EFAULT;
1298 0 : pos = off;
1299 0 : ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1300 0 : if (unlikely(put_user(pos, offset)))
1301 : return -EFAULT;
1302 0 : return ret;
1303 : }
1304 :
1305 0 : return do_sendfile(out_fd, in_fd, NULL, count, 0);
1306 : }
1307 :
1308 4 : SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
1309 : {
1310 2 : loff_t pos;
1311 2 : ssize_t ret;
1312 :
1313 2 : if (offset) {
1314 2 : if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1315 : return -EFAULT;
1316 2 : ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1317 2 : if (unlikely(put_user(pos, offset)))
1318 : return -EFAULT;
1319 2 : return ret;
1320 : }
1321 :
1322 0 : return do_sendfile(out_fd, in_fd, NULL, count, 0);
1323 : }
1324 :
1325 : #ifdef CONFIG_COMPAT
1326 : COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1327 : compat_off_t __user *, offset, compat_size_t, count)
1328 : {
1329 : loff_t pos;
1330 : off_t off;
1331 : ssize_t ret;
1332 :
1333 : if (offset) {
1334 : if (unlikely(get_user(off, offset)))
1335 : return -EFAULT;
1336 : pos = off;
1337 : ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1338 : if (unlikely(put_user(pos, offset)))
1339 : return -EFAULT;
1340 : return ret;
1341 : }
1342 :
1343 : return do_sendfile(out_fd, in_fd, NULL, count, 0);
1344 : }
1345 :
1346 : COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1347 : compat_loff_t __user *, offset, compat_size_t, count)
1348 : {
1349 : loff_t pos;
1350 : ssize_t ret;
1351 :
1352 : if (offset) {
1353 : if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1354 : return -EFAULT;
1355 : ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1356 : if (unlikely(put_user(pos, offset)))
1357 : return -EFAULT;
1358 : return ret;
1359 : }
1360 :
1361 : return do_sendfile(out_fd, in_fd, NULL, count, 0);
1362 : }
1363 : #endif
1364 :
1365 : /**
1366 : * generic_copy_file_range - copy data between two files
1367 : * @file_in: file structure to read from
1368 : * @pos_in: file offset to read from
1369 : * @file_out: file structure to write data to
1370 : * @pos_out: file offset to write data to
1371 : * @len: amount of data to copy
1372 : * @flags: copy flags
1373 : *
1374 : * This is a generic filesystem helper to copy data from one file to another.
1375 : * It has no constraints on the source or destination file owners - the files
1376 : * can belong to different superblocks and different filesystem types. Short
1377 : * copies are allowed.
1378 : *
1379 : * This should be called from the @file_out filesystem, as per the
1380 : * ->copy_file_range() method.
1381 : *
1382 : * Returns the number of bytes copied or a negative error indicating the
1383 : * failure.
1384 : */
1385 :
1386 0 : ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
1387 : struct file *file_out, loff_t pos_out,
1388 : size_t len, unsigned int flags)
1389 : {
1390 3405234 : lockdep_assert(sb_write_started(file_inode(file_out)->i_sb));
1391 :
1392 3405234 : return do_splice_direct(file_in, &pos_in, file_out, &pos_out,
1393 0 : len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
1394 : }
1395 : EXPORT_SYMBOL(generic_copy_file_range);
1396 :
1397 : /*
1398 : * Performs necessary checks before doing a file copy
1399 : *
1400 : * Can adjust amount of bytes to copy via @req_count argument.
1401 : * Returns appropriate error code that caller should return or
1402 : * zero in case the copy should be allowed.
1403 : */
1404 3459872 : static int generic_copy_file_checks(struct file *file_in, loff_t pos_in,
1405 : struct file *file_out, loff_t pos_out,
1406 : size_t *req_count, unsigned int flags)
1407 : {
1408 3459872 : struct inode *inode_in = file_inode(file_in);
1409 3459872 : struct inode *inode_out = file_inode(file_out);
1410 3459872 : uint64_t count = *req_count;
1411 3459872 : loff_t size_in;
1412 3459872 : int ret;
1413 :
1414 3459872 : ret = generic_file_rw_checks(file_in, file_out);
1415 3459872 : if (ret)
1416 : return ret;
1417 :
1418 : /*
1419 : * We allow some filesystems to handle cross sb copy, but passing
1420 : * a file of the wrong filesystem type to filesystem driver can result
1421 : * in an attempt to dereference the wrong type of ->private_data, so
1422 : * avoid doing that until we really have a good reason.
1423 : *
1424 : * nfs and cifs define several different file_system_type structures
1425 : * and several different sets of file_operations, but they all end up
1426 : * using the same ->copy_file_range() function pointer.
1427 : */
1428 3459844 : if (flags & COPY_FILE_SPLICE) {
1429 : /* cross sb splice is allowed */
1430 3459846 : } else if (file_out->f_op->copy_file_range) {
1431 0 : if (file_in->f_op->copy_file_range !=
1432 : file_out->f_op->copy_file_range)
1433 : return -EXDEV;
1434 3459846 : } else if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) {
1435 : return -EXDEV;
1436 : }
1437 :
1438 : /* Don't touch certain kinds of inodes */
1439 3459842 : if (IS_IMMUTABLE(inode_out))
1440 : return -EPERM;
1441 :
1442 3459840 : if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
1443 : return -ETXTBSY;
1444 :
1445 : /* Ensure offsets don't wrap. */
1446 3459838 : if (pos_in + count < pos_in || pos_out + count < pos_out)
1447 : return -EOVERFLOW;
1448 :
1449 : /* Shorten the copy to EOF */
1450 3459834 : size_in = i_size_read(inode_in);
1451 3459834 : if (pos_in >= size_in)
1452 98 : count = 0;
1453 : else
1454 3459736 : count = min(count, size_in - (uint64_t)pos_in);
1455 :
1456 3459834 : ret = generic_write_check_limits(file_out, pos_out, &count);
1457 3459832 : if (ret)
1458 : return ret;
1459 :
1460 : /* Don't allow overlapped copying within the same file. */
1461 3459828 : if (inode_in == inode_out &&
1462 718400 : pos_out + count > pos_in &&
1463 452356 : pos_out < pos_in + count)
1464 : return -EINVAL;
1465 :
1466 3459826 : *req_count = count;
1467 3459826 : return 0;
1468 : }
1469 :
1470 : /*
1471 : * copy_file_range() differs from regular file read and write in that it
1472 : * specifically allows return partial success. When it does so is up to
1473 : * the copy_file_range method.
1474 : */
1475 3459873 : ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
1476 : struct file *file_out, loff_t pos_out,
1477 : size_t len, unsigned int flags)
1478 : {
1479 3459873 : ssize_t ret;
1480 3459873 : bool splice = flags & COPY_FILE_SPLICE;
1481 :
1482 3459873 : if (flags & ~COPY_FILE_SPLICE)
1483 : return -EINVAL;
1484 :
1485 3459873 : ret = generic_copy_file_checks(file_in, pos_in, file_out, pos_out, &len,
1486 : flags);
1487 3459872 : if (unlikely(ret))
1488 : return ret;
1489 :
1490 3459828 : ret = rw_verify_area(READ, file_in, &pos_in, len);
1491 3459828 : if (unlikely(ret))
1492 : return ret;
1493 :
1494 3459828 : ret = rw_verify_area(WRITE, file_out, &pos_out, len);
1495 3459828 : if (unlikely(ret))
1496 : return ret;
1497 :
1498 3459828 : if (len == 0)
1499 : return 0;
1500 :
1501 3459729 : file_start_write(file_out);
1502 :
1503 : /*
1504 : * Cloning is supported by more file systems, so we implement copy on
1505 : * same sb using clone, but for filesystems where both clone and copy
1506 : * are supported (e.g. nfs,cifs), we only call the copy method.
1507 : */
1508 3459727 : if (!splice && file_out->f_op->copy_file_range) {
1509 0 : ret = file_out->f_op->copy_file_range(file_in, pos_in,
1510 : file_out, pos_out,
1511 : len, flags);
1512 0 : goto done;
1513 : }
1514 :
1515 3459727 : if (!splice && file_in->f_op->remap_file_range &&
1516 3459658 : file_inode(file_in)->i_sb == file_inode(file_out)->i_sb) {
1517 3459662 : ret = file_in->f_op->remap_file_range(file_in, pos_in,
1518 : file_out, pos_out,
1519 3459662 : min_t(loff_t, MAX_RW_COUNT, len),
1520 : REMAP_FILE_CAN_SHORTEN);
1521 3459662 : if (ret > 0)
1522 54493 : goto done;
1523 : }
1524 :
1525 : /*
1526 : * We can get here for same sb copy of filesystems that do not implement
1527 : * ->copy_file_range() in case filesystem does not support clone or in
1528 : * case filesystem supports clone but rejected the clone request (e.g.
1529 : * because it was not block aligned).
1530 : *
1531 : * In both cases, fall back to kernel copy so we are able to maintain a
1532 : * consistent story about which filesystems support copy_file_range()
1533 : * and which filesystems do not, that will allow userspace tools to
1534 : * make consistent desicions w.r.t using copy_file_range().
1535 : *
1536 : * We also get here if caller (e.g. nfsd) requested COPY_FILE_SPLICE.
1537 : */
1538 3405234 : ret = generic_copy_file_range(file_in, pos_in, file_out, pos_out, len,
1539 : flags);
1540 :
1541 3459734 : done:
1542 3459734 : if (ret > 0) {
1543 3378036 : fsnotify_access(file_in);
1544 3378038 : add_rchar(current, ret);
1545 3378038 : fsnotify_modify(file_out);
1546 3378037 : add_wchar(current, ret);
1547 : }
1548 :
1549 3459735 : inc_syscr(current);
1550 3459735 : inc_syscw(current);
1551 :
1552 3459735 : file_end_write(file_out);
1553 :
1554 3459735 : return ret;
1555 : }
1556 : EXPORT_SYMBOL(vfs_copy_file_range);
1557 :
1558 6919746 : SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
1559 : int, fd_out, loff_t __user *, off_out,
1560 : size_t, len, unsigned int, flags)
1561 : {
1562 3459876 : loff_t pos_in;
1563 3459876 : loff_t pos_out;
1564 3459876 : struct fd f_in;
1565 3459876 : struct fd f_out;
1566 3459876 : ssize_t ret = -EBADF;
1567 :
1568 3459876 : f_in = fdget(fd_in);
1569 3459875 : if (!f_in.file)
1570 0 : goto out2;
1571 :
1572 3459875 : f_out = fdget(fd_out);
1573 3459876 : if (!f_out.file)
1574 0 : goto out1;
1575 :
1576 3459876 : ret = -EFAULT;
1577 3459876 : if (off_in) {
1578 3459876 : if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
1579 0 : goto out;
1580 : } else {
1581 0 : pos_in = f_in.file->f_pos;
1582 : }
1583 :
1584 3459872 : if (off_out) {
1585 3459872 : if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
1586 0 : goto out;
1587 : } else {
1588 0 : pos_out = f_out.file->f_pos;
1589 : }
1590 :
1591 3459875 : ret = -EINVAL;
1592 3459875 : if (flags != 0)
1593 0 : goto out;
1594 :
1595 3459875 : ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len,
1596 : flags);
1597 3459864 : if (ret > 0) {
1598 3378027 : pos_in += ret;
1599 3378027 : pos_out += ret;
1600 :
1601 3378027 : if (off_in) {
1602 3378027 : if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
1603 0 : ret = -EFAULT;
1604 : } else {
1605 0 : f_in.file->f_pos = pos_in;
1606 : }
1607 :
1608 3378026 : if (off_out) {
1609 3378026 : if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
1610 0 : ret = -EFAULT;
1611 : } else {
1612 0 : f_out.file->f_pos = pos_out;
1613 : }
1614 : }
1615 :
1616 81837 : out:
1617 3459866 : fdput(f_out);
1618 3459866 : out1:
1619 3459866 : fdput(f_in);
1620 3459866 : out2:
1621 3459866 : return ret;
1622 : }
1623 :
1624 : /*
1625 : * Don't operate on ranges the page cache doesn't support, and don't exceed the
1626 : * LFS limits. If pos is under the limit it becomes a short access. If it
1627 : * exceeds the limit we return -EFBIG.
1628 : */
1629 446706805 : int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count)
1630 : {
1631 446706805 : struct inode *inode = file->f_mapping->host;
1632 446706805 : loff_t max_size = inode->i_sb->s_maxbytes;
1633 446706805 : loff_t limit = rlimit(RLIMIT_FSIZE);
1634 :
1635 446706805 : if (limit != RLIM_INFINITY) {
1636 18 : if (pos >= limit) {
1637 2 : send_sig(SIGXFSZ, current, 0);
1638 2 : return -EFBIG;
1639 : }
1640 16 : *count = min(*count, limit - pos);
1641 : }
1642 :
1643 446706803 : if (!(file->f_flags & O_LARGEFILE))
1644 0 : max_size = MAX_NON_LFS;
1645 :
1646 446706803 : if (unlikely(pos >= max_size))
1647 : return -EFBIG;
1648 :
1649 446706801 : *count = min(*count, max_size - pos);
1650 :
1651 446706801 : return 0;
1652 : }
1653 :
1654 : /* Like generic_write_checks(), but takes size of write instead of iter. */
1655 355987360 : int generic_write_checks_count(struct kiocb *iocb, loff_t *count)
1656 : {
1657 355987360 : struct file *file = iocb->ki_filp;
1658 355987360 : struct inode *inode = file->f_mapping->host;
1659 :
1660 355987360 : if (IS_SWAPFILE(inode))
1661 : return -ETXTBSY;
1662 :
1663 355987354 : if (!*count)
1664 : return 0;
1665 :
1666 355987354 : if (iocb->ki_flags & IOCB_APPEND)
1667 8219469 : iocb->ki_pos = i_size_read(inode);
1668 :
1669 355987354 : if ((iocb->ki_flags & IOCB_NOWAIT) &&
1670 0 : !((iocb->ki_flags & IOCB_DIRECT) ||
1671 0 : (file->f_mode & FMODE_BUF_WASYNC)))
1672 : return -EINVAL;
1673 :
1674 355987354 : return generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, count);
1675 : }
1676 : EXPORT_SYMBOL(generic_write_checks_count);
1677 :
1678 : /*
1679 : * Performs necessary checks before doing a write
1680 : *
1681 : * Can adjust writing position or amount of bytes to write.
1682 : * Returns appropriate error code that caller should return or
1683 : * zero in case that write should be allowed.
1684 : */
1685 355987109 : ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
1686 : {
1687 355987109 : loff_t count = iov_iter_count(from);
1688 355987109 : int ret;
1689 :
1690 355987109 : ret = generic_write_checks_count(iocb, &count);
1691 355989089 : if (ret)
1692 6 : return ret;
1693 :
1694 355989083 : iov_iter_truncate(from, count);
1695 355989083 : return iov_iter_count(from);
1696 : }
1697 : EXPORT_SYMBOL(generic_write_checks);
1698 :
1699 : /*
1700 : * Performs common checks before doing a file copy/clone
1701 : * from @file_in to @file_out.
1702 : */
1703 7146971 : int generic_file_rw_checks(struct file *file_in, struct file *file_out)
1704 : {
1705 7146971 : struct inode *inode_in = file_inode(file_in);
1706 7146971 : struct inode *inode_out = file_inode(file_out);
1707 :
1708 : /* Don't copy dirs, pipes, sockets... */
1709 7146971 : if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1710 : return -EISDIR;
1711 7146965 : if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1712 : return -EINVAL;
1713 :
1714 7146943 : if (!(file_in->f_mode & FMODE_READ) ||
1715 7146943 : !(file_out->f_mode & FMODE_WRITE) ||
1716 7146939 : (file_out->f_flags & O_APPEND))
1717 10 : return -EBADF;
1718 :
1719 : return 0;
1720 : }
|
