Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * linux/fs/read_write.c
4 : *
5 : * Copyright (C) 1991, 1992 Linus Torvalds
6 : */
7 :
8 : #include <linux/slab.h>
9 : #include <linux/stat.h>
10 : #include <linux/sched/xacct.h>
11 : #include <linux/fcntl.h>
12 : #include <linux/file.h>
13 : #include <linux/uio.h>
14 : #include <linux/fsnotify.h>
15 : #include <linux/security.h>
16 : #include <linux/export.h>
17 : #include <linux/syscalls.h>
18 : #include <linux/pagemap.h>
19 : #include <linux/splice.h>
20 : #include <linux/compat.h>
21 : #include <linux/mount.h>
22 : #include <linux/fs.h>
23 : #include "internal.h"
24 :
25 : #include <linux/uaccess.h>
26 : #include <asm/unistd.h>
27 :
28 : const struct file_operations generic_ro_fops = {
29 : .llseek = generic_file_llseek,
30 : .read_iter = generic_file_read_iter,
31 : .mmap = generic_file_readonly_mmap,
32 : .splice_read = filemap_splice_read,
33 : };
34 :
35 : EXPORT_SYMBOL(generic_ro_fops);
36 :
37 : static inline bool unsigned_offsets(struct file *file)
38 : {
39 74060 : return file->f_mode & FMODE_UNSIGNED_OFFSET;
40 : }
41 :
42 : /**
43 : * vfs_setpos - update the file offset for lseek
44 : * @file: file structure in question
45 : * @offset: file offset to seek to
46 : * @maxsize: maximum file size
47 : *
48 : * This is a low-level filesystem helper for updating the file offset to
49 : * the value specified by @offset if the given offset is valid and it is
50 : * not equal to the current file offset.
51 : *
52 : * Return the specified offset on success and -EINVAL on invalid offset.
53 : */
54 78725917 : loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
55 : {
56 78725917 : if (offset < 0 && !unsigned_offsets(file))
57 : return -EINVAL;
58 493278314 : if (offset > maxsize)
59 : return -EINVAL;
60 :
61 493278309 : if (offset != file->f_pos) {
62 407771188 : file->f_pos = offset;
63 407771188 : file->f_version = 0;
64 : }
65 : return offset;
66 : }
67 : EXPORT_SYMBOL(vfs_setpos);
68 :
69 : /**
70 : * generic_file_llseek_size - generic llseek implementation for regular files
71 : * @file: file structure to seek on
72 : * @offset: file offset to seek to
73 : * @whence: type of seek
74 : * @size: max size of this file in file system
75 : * @eof: offset used for SEEK_END position
76 : *
77 : * This is a variant of generic_file_llseek that allows passing in a custom
78 : * maximum file size and a custom EOF position, for e.g. hashed directories
79 : *
80 : * Synchronization:
81 : * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
82 : * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
83 : * read/writes behave like SEEK_SET against seeks.
84 : */
85 : loff_t
86 468222846 : generic_file_llseek_size(struct file *file, loff_t offset, int whence,
87 : loff_t maxsize, loff_t eof)
88 : {
89 468222846 : switch (whence) {
90 40998596 : case SEEK_END:
91 40998596 : offset += eof;
92 40998596 : break;
93 63378754 : case SEEK_CUR:
94 : /*
95 : * Here we special-case the lseek(fd, 0, SEEK_CUR)
96 : * position-querying operation. Avoid rewriting the "same"
97 : * f_pos value back to the file because a concurrent read(),
98 : * write() or lseek() might have altered it
99 : */
100 63378754 : if (offset == 0)
101 53670854 : return file->f_pos;
102 : /*
103 : * f_lock protects against read/modify/write race with other
104 : * SEEK_CURs. Note that parallel writes and reads behave
105 : * like SEEK_SET.
106 : */
107 9707900 : spin_lock(&file->f_lock);
108 9708305 : offset = vfs_setpos(file, file->f_pos + offset, maxsize);
109 9708305 : spin_unlock(&file->f_lock);
110 9708305 : return offset;
111 0 : case SEEK_DATA:
112 : /*
113 : * In the generic case the entire file is data, so as long as
114 : * offset isn't at the end of the file then the offset is data.
115 : */
116 0 : if ((unsigned long long)offset >= eof)
117 : return -ENXIO;
118 : break;
119 1689 : case SEEK_HOLE:
120 : /*
121 : * There is a virtual hole at the end of the file, so as long as
122 : * offset isn't i_size or larger, return i_size.
123 : */
124 1689 : if ((unsigned long long)offset >= eof)
125 : return -ENXIO;
126 : offset = eof;
127 : break;
128 : }
129 :
130 404844092 : return vfs_setpos(file, offset, maxsize);
131 : }
132 : EXPORT_SYMBOL(generic_file_llseek_size);
133 :
134 : /**
135 : * generic_file_llseek - generic llseek implementation for regular files
136 : * @file: file structure to seek on
137 : * @offset: file offset to seek to
138 : * @whence: type of seek
139 : *
140 : * This is a generic implemenation of ->llseek useable for all normal local
141 : * filesystems. It just updates the file offset to the value specified by
142 : * @offset and @whence.
143 : */
144 432088928 : loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
145 : {
146 432088928 : struct inode *inode = file->f_mapping->host;
147 :
148 432088928 : return generic_file_llseek_size(file, offset, whence,
149 432088928 : inode->i_sb->s_maxbytes,
150 : i_size_read(inode));
151 : }
152 : EXPORT_SYMBOL(generic_file_llseek);
153 :
154 : /**
155 : * fixed_size_llseek - llseek implementation for fixed-sized devices
156 : * @file: file structure to seek on
157 : * @offset: file offset to seek to
158 : * @whence: type of seek
159 : * @size: size of the file
160 : *
161 : */
162 12243625 : loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
163 : {
164 12243625 : switch (whence) {
165 12243625 : case SEEK_SET: case SEEK_CUR: case SEEK_END:
166 12243625 : return generic_file_llseek_size(file, offset, whence,
167 : size, size);
168 : default:
169 : return -EINVAL;
170 : }
171 : }
172 : EXPORT_SYMBOL(fixed_size_llseek);
173 :
174 : /**
175 : * no_seek_end_llseek - llseek implementation for fixed-sized devices
176 : * @file: file structure to seek on
177 : * @offset: file offset to seek to
178 : * @whence: type of seek
179 : *
180 : */
181 0 : loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
182 : {
183 0 : switch (whence) {
184 0 : case SEEK_SET: case SEEK_CUR:
185 0 : return generic_file_llseek_size(file, offset, whence,
186 : OFFSET_MAX, 0);
187 : default:
188 : return -EINVAL;
189 : }
190 : }
191 : EXPORT_SYMBOL(no_seek_end_llseek);
192 :
193 : /**
194 : * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
195 : * @file: file structure to seek on
196 : * @offset: file offset to seek to
197 : * @whence: type of seek
198 : * @size: maximal offset allowed
199 : *
200 : */
201 0 : loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
202 : {
203 0 : switch (whence) {
204 0 : case SEEK_SET: case SEEK_CUR:
205 0 : return generic_file_llseek_size(file, offset, whence,
206 : size, 0);
207 : default:
208 : return -EINVAL;
209 : }
210 : }
211 : EXPORT_SYMBOL(no_seek_end_llseek_size);
212 :
213 : /**
214 : * noop_llseek - No Operation Performed llseek implementation
215 : * @file: file structure to seek on
216 : * @offset: file offset to seek to
217 : * @whence: type of seek
218 : *
219 : * This is an implementation of ->llseek useable for the rare special case when
220 : * userspace expects the seek to succeed but the (device) file is actually not
221 : * able to perform the seek. In this case you use noop_llseek() instead of
222 : * falling back to the default implementation of ->llseek.
223 : */
224 29834 : loff_t noop_llseek(struct file *file, loff_t offset, int whence)
225 : {
226 29834 : return file->f_pos;
227 : }
228 : EXPORT_SYMBOL(noop_llseek);
229 :
230 0 : loff_t default_llseek(struct file *file, loff_t offset, int whence)
231 : {
232 0 : struct inode *inode = file_inode(file);
233 0 : loff_t retval;
234 :
235 0 : inode_lock(inode);
236 0 : switch (whence) {
237 : case SEEK_END:
238 0 : offset += i_size_read(inode);
239 0 : break;
240 0 : case SEEK_CUR:
241 0 : if (offset == 0) {
242 0 : retval = file->f_pos;
243 0 : goto out;
244 : }
245 0 : offset += file->f_pos;
246 0 : break;
247 0 : case SEEK_DATA:
248 : /*
249 : * In the generic case the entire file is data, so as
250 : * long as offset isn't at the end of the file then the
251 : * offset is data.
252 : */
253 0 : if (offset >= inode->i_size) {
254 0 : retval = -ENXIO;
255 0 : goto out;
256 : }
257 : break;
258 0 : case SEEK_HOLE:
259 : /*
260 : * There is a virtual hole at the end of the file, so
261 : * as long as offset isn't i_size or larger, return
262 : * i_size.
263 : */
264 0 : if (offset >= inode->i_size) {
265 0 : retval = -ENXIO;
266 0 : goto out;
267 : }
268 : offset = inode->i_size;
269 : break;
270 : }
271 0 : retval = -EINVAL;
272 0 : if (offset >= 0 || unsigned_offsets(file)) {
273 0 : if (offset != file->f_pos) {
274 0 : file->f_pos = offset;
275 0 : file->f_version = 0;
276 : }
277 : retval = offset;
278 : }
279 0 : out:
280 0 : inode_unlock(inode);
281 0 : return retval;
282 : }
283 : EXPORT_SYMBOL(default_llseek);
284 :
285 557838950 : loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
286 : {
287 557838950 : if (!(file->f_mode & FMODE_LSEEK))
288 : return -ESPIPE;
289 551605815 : return file->f_op->llseek(file, offset, whence);
290 : }
291 : EXPORT_SYMBOL(vfs_llseek);
292 :
293 480248733 : static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
294 : {
295 480248733 : off_t retval;
296 480248733 : struct fd f = fdget_pos(fd);
297 480252636 : if (!f.file)
298 : return -EBADF;
299 :
300 480250812 : retval = -EINVAL;
301 480250812 : if (whence <= SEEK_MAX) {
302 480251385 : loff_t res = vfs_llseek(f.file, offset, whence);
303 480251385 : retval = res;
304 480251385 : if (res != (loff_t)retval)
305 : retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
306 : }
307 480237615 : fdput_pos(f);
308 480237615 : return retval;
309 : }
310 :
311 960468570 : SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
312 : {
313 480249923 : return ksys_lseek(fd, offset, whence);
314 : }
315 :
316 : #ifdef CONFIG_COMPAT
317 0 : COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
318 : {
319 0 : return ksys_lseek(fd, offset, whence);
320 : }
321 : #endif
322 :
323 : #if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT) || \
324 : defined(__ARCH_WANT_SYS_LLSEEK)
325 0 : SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
326 : unsigned long, offset_low, loff_t __user *, result,
327 : unsigned int, whence)
328 : {
329 0 : int retval;
330 0 : struct fd f = fdget_pos(fd);
331 0 : loff_t offset;
332 :
333 0 : if (!f.file)
334 : return -EBADF;
335 :
336 0 : retval = -EINVAL;
337 0 : if (whence > SEEK_MAX)
338 0 : goto out_putf;
339 :
340 0 : offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
341 : whence);
342 :
343 0 : retval = (int)offset;
344 0 : if (offset >= 0) {
345 0 : retval = -EFAULT;
346 0 : if (!copy_to_user(result, &offset, sizeof(offset)))
347 0 : retval = 0;
348 : }
349 0 : out_putf:
350 0 : fdput_pos(f);
351 0 : return retval;
352 : }
353 : #endif
354 :
355 7229723741 : int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
356 : {
357 7229723741 : if (unlikely((ssize_t) count < 0))
358 : return -EINVAL;
359 :
360 7229723741 : if (ppos) {
361 5007587127 : loff_t pos = *ppos;
362 :
363 5007587127 : if (unlikely(pos < 0)) {
364 0 : if (!unsigned_offsets(file))
365 : return -EINVAL;
366 0 : if (count >= -pos) /* both values are in 0..LLONG_MAX */
367 0 : return -EOVERFLOW;
368 5007587127 : } else if (unlikely((loff_t) (pos + count) < 0)) {
369 74060 : if (!unsigned_offsets(file))
370 74060 : return -EINVAL;
371 : }
372 : }
373 :
374 : return security_file_permission(file,
375 : read_write == READ ? MAY_READ : MAY_WRITE);
376 : }
377 : EXPORT_SYMBOL(rw_verify_area);
378 :
379 5658075237 : static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
380 : {
381 5658075237 : struct kiocb kiocb;
382 5658075237 : struct iov_iter iter;
383 5658075237 : ssize_t ret;
384 :
385 5658075237 : init_sync_kiocb(&kiocb, filp);
386 5654934053 : kiocb.ki_pos = (ppos ? *ppos : 0);
387 5654934053 : iov_iter_ubuf(&iter, ITER_DEST, buf, len);
388 :
389 5654934053 : ret = call_read_iter(filp, &kiocb, &iter);
390 5656939895 : BUG_ON(ret == -EIOCBQUEUED);
391 5656939895 : if (ppos)
392 3535911762 : *ppos = kiocb.ki_pos;
393 5656939895 : return ret;
394 : }
395 :
396 0 : static int warn_unsupported(struct file *file, const char *op)
397 : {
398 0 : pr_warn_ratelimited(
399 : "kernel %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
400 : op, file, current->pid, current->comm);
401 0 : return -EINVAL;
402 : }
403 :
404 184838150 : ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
405 : {
406 184838150 : struct kvec iov = {
407 : .iov_base = buf,
408 184838150 : .iov_len = min_t(size_t, count, MAX_RW_COUNT),
409 : };
410 184838150 : struct kiocb kiocb;
411 184838150 : struct iov_iter iter;
412 184838150 : ssize_t ret;
413 :
414 184838150 : if (WARN_ON_ONCE(!(file->f_mode & FMODE_READ)))
415 : return -EINVAL;
416 184838150 : if (!(file->f_mode & FMODE_CAN_READ))
417 : return -EINVAL;
418 : /*
419 : * Also fail if ->read_iter and ->read are both wired up as that
420 : * implies very convoluted semantics.
421 : */
422 184838150 : if (unlikely(!file->f_op->read_iter || file->f_op->read))
423 0 : return warn_unsupported(file, "read");
424 :
425 184838150 : init_sync_kiocb(&kiocb, file);
426 184850241 : kiocb.ki_pos = pos ? *pos : 0;
427 184850241 : iov_iter_kvec(&iter, ITER_DEST, &iov, 1, iov.iov_len);
428 184850346 : ret = file->f_op->read_iter(&kiocb, &iter);
429 184900660 : if (ret > 0) {
430 184899988 : if (pos)
431 184899988 : *pos = kiocb.ki_pos;
432 184899988 : fsnotify_access(file);
433 184881369 : add_rchar(current, ret);
434 : }
435 184882041 : inc_syscr(current);
436 184882041 : return ret;
437 : }
438 :
439 184839417 : ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
440 : {
441 184839417 : ssize_t ret;
442 :
443 184839417 : ret = rw_verify_area(READ, file, pos, count);
444 184839417 : if (ret)
445 : return ret;
446 184822907 : return __kernel_read(file, buf, count, pos);
447 : }
448 : EXPORT_SYMBOL(kernel_read);
449 :
450 6073318270 : ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
451 : {
452 6073318270 : ssize_t ret;
453 :
454 6073318270 : if (!(file->f_mode & FMODE_READ))
455 : return -EBADF;
456 6073318270 : if (!(file->f_mode & FMODE_CAN_READ))
457 : return -EINVAL;
458 12146636540 : if (unlikely(!access_ok(buf, count)))
459 : return -EFAULT;
460 :
461 6073318270 : ret = rw_verify_area(READ, file, pos, count);
462 6073318270 : if (ret)
463 : return ret;
464 6073244320 : if (count > MAX_RW_COUNT)
465 : count = MAX_RW_COUNT;
466 :
467 6073244320 : if (file->f_op->read)
468 415734373 : ret = file->f_op->read(file, buf, count, pos);
469 5657509947 : else if (file->f_op->read_iter)
470 5657509947 : ret = new_sync_read(file, buf, count, pos);
471 : else
472 : ret = -EINVAL;
473 6068109770 : if (ret > 0) {
474 5339139625 : fsnotify_access(file);
475 5347108098 : add_rchar(current, ret);
476 : }
477 6076078243 : inc_syscr(current);
478 6076078243 : return ret;
479 : }
480 :
481 804263714 : static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
482 : {
483 804263714 : struct kiocb kiocb;
484 804263714 : struct iov_iter iter;
485 804263714 : ssize_t ret;
486 :
487 804263714 : init_sync_kiocb(&kiocb, filp);
488 803962612 : kiocb.ki_pos = (ppos ? *ppos : 0);
489 803962612 : iov_iter_ubuf(&iter, ITER_SOURCE, (void __user *)buf, len);
490 :
491 803962612 : ret = call_write_iter(filp, &kiocb, &iter);
492 804659951 : BUG_ON(ret == -EIOCBQUEUED);
493 804659951 : if (ret > 0 && ppos)
494 702413749 : *ppos = kiocb.ki_pos;
495 804659951 : return ret;
496 : }
497 :
498 : /* caller is responsible for file_start_write/file_end_write */
499 5498085 : ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos)
500 : {
501 5498085 : struct kiocb kiocb;
502 5498085 : ssize_t ret;
503 :
504 5498085 : if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE)))
505 : return -EBADF;
506 5498085 : if (!(file->f_mode & FMODE_CAN_WRITE))
507 : return -EINVAL;
508 : /*
509 : * Also fail if ->write_iter and ->write are both wired up as that
510 : * implies very convoluted semantics.
511 : */
512 5498085 : if (unlikely(!file->f_op->write_iter || file->f_op->write))
513 0 : return warn_unsupported(file, "write");
514 :
515 5498085 : init_sync_kiocb(&kiocb, file);
516 5498902 : kiocb.ki_pos = pos ? *pos : 0;
517 5498902 : ret = file->f_op->write_iter(&kiocb, from);
518 5498808 : if (ret > 0) {
519 5498808 : if (pos)
520 5498808 : *pos = kiocb.ki_pos;
521 5498808 : fsnotify_modify(file);
522 5499145 : add_wchar(current, ret);
523 : }
524 5499145 : inc_syscw(current);
525 5499145 : return ret;
526 : }
527 :
528 : /* caller is responsible for file_start_write/file_end_write */
529 5498031 : ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
530 : {
531 5498031 : struct kvec iov = {
532 : .iov_base = (void *)buf,
533 5498031 : .iov_len = min_t(size_t, count, MAX_RW_COUNT),
534 : };
535 5498031 : struct iov_iter iter;
536 5498031 : iov_iter_kvec(&iter, ITER_SOURCE, &iov, 1, iov.iov_len);
537 5497836 : return __kernel_write_iter(file, &iter, pos);
538 : }
539 : /*
540 : * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()",
541 : * but autofs is one of the few internal kernel users that actually
542 : * wants this _and_ can be built as a module. So we need to export
543 : * this symbol for autofs, even though it really isn't appropriate
544 : * for any other kernel modules.
545 : */
546 : EXPORT_SYMBOL_GPL(__kernel_write);
547 :
548 5497239 : ssize_t kernel_write(struct file *file, const void *buf, size_t count,
549 : loff_t *pos)
550 : {
551 5497239 : ssize_t ret;
552 :
553 5497239 : ret = rw_verify_area(WRITE, file, pos, count);
554 5497239 : if (ret)
555 : return ret;
556 :
557 5496765 : file_start_write(file);
558 5497722 : ret = __kernel_write(file, buf, count, pos);
559 5499331 : file_end_write(file);
560 5499331 : return ret;
561 : }
562 : EXPORT_SYMBOL(kernel_write);
563 :
564 816040881 : ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
565 : {
566 816040881 : ssize_t ret;
567 :
568 816040881 : if (!(file->f_mode & FMODE_WRITE))
569 : return -EBADF;
570 816040881 : if (!(file->f_mode & FMODE_CAN_WRITE))
571 : return -EINVAL;
572 1632081762 : if (unlikely(!access_ok(buf, count)))
573 : return -EFAULT;
574 :
575 816040881 : ret = rw_verify_area(WRITE, file, pos, count);
576 816040881 : if (ret)
577 : return ret;
578 815618789 : if (count > MAX_RW_COUNT)
579 : count = MAX_RW_COUNT;
580 815618789 : file_start_write(file);
581 815767968 : if (file->f_op->write)
582 11590453 : ret = file->f_op->write(file, buf, count, pos);
583 804177515 : else if (file->f_op->write_iter)
584 804177515 : ret = new_sync_write(file, buf, count, pos);
585 : else
586 : ret = -EINVAL;
587 816060365 : if (ret > 0) {
588 813445476 : fsnotify_modify(file);
589 813584247 : add_wchar(current, ret);
590 : }
591 816199136 : inc_syscw(current);
592 816199136 : file_end_write(file);
593 816199136 : return ret;
594 : }
595 :
596 : /* file_ppos returns &file->f_pos or NULL if file is stream */
597 : static inline loff_t *file_ppos(struct file *file)
598 : {
599 4923304248 : return file->f_mode & FMODE_STREAM ? NULL : &file->f_pos;
600 : }
601 :
602 3312856623 : ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
603 : {
604 3312856623 : struct fd f = fdget_pos(fd);
605 3312942637 : ssize_t ret = -EBADF;
606 :
607 3312942637 : if (f.file) {
608 3312942637 : loff_t pos, *ppos = file_ppos(f.file);
609 1191899760 : if (ppos) {
610 1191899760 : pos = *ppos;
611 1191899760 : ppos = &pos;
612 : }
613 3312942637 : ret = vfs_read(f.file, buf, count, ppos);
614 3312595765 : if (ret >= 0 && ppos)
615 870611274 : f.file->f_pos = pos;
616 3312595765 : fdput_pos(f);
617 : }
618 3312802621 : return ret;
619 : }
620 :
621 6625873271 : SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
622 : {
623 3313005572 : return ksys_read(fd, buf, count);
624 : }
625 :
626 249934400 : ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
627 : {
628 249934400 : struct fd f = fdget_pos(fd);
629 250022215 : ssize_t ret = -EBADF;
630 :
631 250022215 : if (f.file) {
632 250022215 : loff_t pos, *ppos = file_ppos(f.file);
633 148956562 : if (ppos) {
634 148956562 : pos = *ppos;
635 148956562 : ppos = &pos;
636 : }
637 250022215 : ret = vfs_write(f.file, buf, count, ppos);
638 249937935 : if (ret >= 0 && ppos)
639 147716963 : f.file->f_pos = pos;
640 249937935 : fdput_pos(f);
641 : }
642 :
643 249979135 : return ret;
644 : }
645 :
646 499979427 : SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
647 : size_t, count)
648 : {
649 249985372 : return ksys_write(fd, buf, count);
650 : }
651 :
652 2765699863 : ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
653 : loff_t pos)
654 : {
655 2765699863 : struct fd f;
656 2765699863 : ssize_t ret = -EBADF;
657 :
658 2765699863 : if (pos < 0)
659 : return -EINVAL;
660 :
661 2765699863 : f = fdget(fd);
662 2767449278 : if (f.file) {
663 2767449278 : ret = -ESPIPE;
664 2767449278 : if (f.file->f_mode & FMODE_PREAD)
665 2767235187 : ret = vfs_read(f.file, buf, count, &pos);
666 2763578809 : fdput(f);
667 : }
668 :
669 : return ret;
670 : }
671 :
672 5527563067 : SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
673 : size_t, count, loff_t, pos)
674 : {
675 2767755157 : return ksys_pread64(fd, buf, count, pos);
676 : }
677 :
678 : #if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PREAD64)
679 : COMPAT_SYSCALL_DEFINE5(pread64, unsigned int, fd, char __user *, buf,
680 : size_t, count, compat_arg_u64_dual(pos))
681 : {
682 : return ksys_pread64(fd, buf, count, compat_arg_u64_glue(pos));
683 : }
684 : #endif
685 :
686 566226663 : ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
687 : size_t count, loff_t pos)
688 : {
689 566226663 : struct fd f;
690 566226663 : ssize_t ret = -EBADF;
691 :
692 566226663 : if (pos < 0)
693 : return -EINVAL;
694 :
695 566226663 : f = fdget(fd);
696 566159524 : if (f.file) {
697 566159524 : ret = -ESPIPE;
698 566159524 : if (f.file->f_mode & FMODE_PWRITE)
699 566193291 : ret = vfs_write(f.file, buf, count, &pos);
700 566164879 : fdput(f);
701 : }
702 :
703 : return ret;
704 : }
705 :
706 1132581125 : SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
707 : size_t, count, loff_t, pos)
708 : {
709 566265950 : return ksys_pwrite64(fd, buf, count, pos);
710 : }
711 :
712 : #if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PWRITE64)
713 : COMPAT_SYSCALL_DEFINE5(pwrite64, unsigned int, fd, const char __user *, buf,
714 : size_t, count, compat_arg_u64_dual(pos))
715 : {
716 : return ksys_pwrite64(fd, buf, count, compat_arg_u64_glue(pos));
717 : }
718 : #endif
719 :
720 39288372 : static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
721 : loff_t *ppos, int type, rwf_t flags)
722 : {
723 39288372 : struct kiocb kiocb;
724 39288372 : ssize_t ret;
725 :
726 39288372 : init_sync_kiocb(&kiocb, filp);
727 39288291 : ret = kiocb_set_rw_flags(&kiocb, flags);
728 39288227 : if (ret)
729 : return ret;
730 39288227 : kiocb.ki_pos = (ppos ? *ppos : 0);
731 :
732 39288227 : if (type == READ)
733 14853365 : ret = call_read_iter(filp, &kiocb, iter);
734 : else
735 24434862 : ret = call_write_iter(filp, &kiocb, iter);
736 39287830 : BUG_ON(ret == -EIOCBQUEUED);
737 39287830 : if (ppos)
738 39217780 : *ppos = kiocb.ki_pos;
739 : return ret;
740 : }
741 :
742 : /* Do it by hand, with file-ops */
743 0 : static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
744 : loff_t *ppos, int type, rwf_t flags)
745 : {
746 0 : ssize_t ret = 0;
747 :
748 0 : if (flags & ~RWF_HIPRI)
749 : return -EOPNOTSUPP;
750 :
751 0 : while (iov_iter_count(iter)) {
752 0 : ssize_t nr;
753 :
754 0 : if (type == READ) {
755 0 : nr = filp->f_op->read(filp, iter_iov_addr(iter),
756 0 : iter_iov_len(iter), ppos);
757 : } else {
758 0 : nr = filp->f_op->write(filp, iter_iov_addr(iter),
759 0 : iter_iov_len(iter), ppos);
760 : }
761 :
762 0 : if (nr < 0) {
763 0 : if (!ret)
764 0 : ret = nr;
765 : break;
766 : }
767 0 : ret += nr;
768 0 : if (nr != iter_iov_len(iter))
769 : break;
770 0 : iov_iter_advance(iter, nr);
771 : }
772 :
773 : return ret;
774 : }
775 :
776 14853486 : static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,
777 : loff_t *pos, rwf_t flags)
778 : {
779 14853486 : size_t tot_len;
780 14853486 : ssize_t ret = 0;
781 :
782 14853486 : if (!(file->f_mode & FMODE_READ))
783 : return -EBADF;
784 14853486 : if (!(file->f_mode & FMODE_CAN_READ))
785 : return -EINVAL;
786 :
787 14853486 : tot_len = iov_iter_count(iter);
788 14853486 : if (!tot_len)
789 0 : goto out;
790 14853486 : ret = rw_verify_area(READ, file, pos, tot_len);
791 14853486 : if (ret < 0)
792 : return ret;
793 :
794 14853486 : if (file->f_op->read_iter)
795 14853486 : ret = do_iter_readv_writev(file, iter, pos, READ, flags);
796 : else
797 0 : ret = do_loop_readv_writev(file, iter, pos, READ, flags);
798 14853602 : out:
799 14853602 : if (ret >= 0)
800 14853575 : fsnotify_access(file);
801 : return ret;
802 : }
803 :
804 0 : ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
805 : struct iov_iter *iter)
806 : {
807 0 : size_t tot_len;
808 0 : ssize_t ret = 0;
809 :
810 0 : if (!file->f_op->read_iter)
811 : return -EINVAL;
812 0 : if (!(file->f_mode & FMODE_READ))
813 : return -EBADF;
814 0 : if (!(file->f_mode & FMODE_CAN_READ))
815 : return -EINVAL;
816 :
817 0 : tot_len = iov_iter_count(iter);
818 0 : if (!tot_len)
819 0 : goto out;
820 0 : ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len);
821 0 : if (ret < 0)
822 : return ret;
823 :
824 0 : ret = call_read_iter(file, iocb, iter);
825 : out:
826 0 : if (ret >= 0)
827 0 : fsnotify_access(file);
828 : return ret;
829 : }
830 : EXPORT_SYMBOL(vfs_iocb_iter_read);
831 :
832 13241953 : ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
833 : rwf_t flags)
834 : {
835 13241953 : if (!file->f_op->read_iter)
836 : return -EINVAL;
837 13241953 : return do_iter_read(file, iter, ppos, flags);
838 : }
839 : EXPORT_SYMBOL(vfs_iter_read);
840 :
841 24435059 : static ssize_t do_iter_write(struct file *file, struct iov_iter *iter,
842 : loff_t *pos, rwf_t flags)
843 : {
844 24435059 : size_t tot_len;
845 24435059 : ssize_t ret = 0;
846 :
847 24435059 : if (!(file->f_mode & FMODE_WRITE))
848 : return -EBADF;
849 24435059 : if (!(file->f_mode & FMODE_CAN_WRITE))
850 : return -EINVAL;
851 :
852 24435059 : tot_len = iov_iter_count(iter);
853 24435059 : if (!tot_len)
854 : return 0;
855 24435059 : ret = rw_verify_area(WRITE, file, pos, tot_len);
856 24435059 : if (ret < 0)
857 : return ret;
858 :
859 24435059 : if (file->f_op->write_iter)
860 24435059 : ret = do_iter_readv_writev(file, iter, pos, WRITE, flags);
861 : else
862 0 : ret = do_loop_readv_writev(file, iter, pos, WRITE, flags);
863 24434574 : if (ret > 0)
864 23678556 : fsnotify_modify(file);
865 : return ret;
866 : }
867 :
868 0 : ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
869 : struct iov_iter *iter)
870 : {
871 0 : size_t tot_len;
872 0 : ssize_t ret = 0;
873 :
874 0 : if (!file->f_op->write_iter)
875 : return -EINVAL;
876 0 : if (!(file->f_mode & FMODE_WRITE))
877 : return -EBADF;
878 0 : if (!(file->f_mode & FMODE_CAN_WRITE))
879 : return -EINVAL;
880 :
881 0 : tot_len = iov_iter_count(iter);
882 0 : if (!tot_len)
883 : return 0;
884 0 : ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len);
885 0 : if (ret < 0)
886 : return ret;
887 :
888 0 : ret = call_write_iter(file, iocb, iter);
889 0 : if (ret > 0)
890 0 : fsnotify_modify(file);
891 :
892 : return ret;
893 : }
894 : EXPORT_SYMBOL(vfs_iocb_iter_write);
895 :
896 16270229 : ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
897 : rwf_t flags)
898 : {
899 16270229 : if (!file->f_op->write_iter)
900 : return -EINVAL;
901 16270229 : return do_iter_write(file, iter, ppos, flags);
902 : }
903 : EXPORT_SYMBOL(vfs_iter_write);
904 :
905 1611564 : static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
906 : unsigned long vlen, loff_t *pos, rwf_t flags)
907 : {
908 1611564 : struct iovec iovstack[UIO_FASTIOV];
909 1611564 : struct iovec *iov = iovstack;
910 1611564 : struct iov_iter iter;
911 1611564 : ssize_t ret;
912 :
913 1611564 : ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
914 1611572 : if (ret >= 0) {
915 1611572 : ret = do_iter_read(file, &iter, pos, flags);
916 1611564 : kfree(iov);
917 : }
918 :
919 1611563 : return ret;
920 : }
921 :
922 8165014 : static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
923 : unsigned long vlen, loff_t *pos, rwf_t flags)
924 : {
925 8165014 : struct iovec iovstack[UIO_FASTIOV];
926 8165014 : struct iovec *iov = iovstack;
927 8165014 : struct iov_iter iter;
928 8165014 : ssize_t ret;
929 :
930 8165014 : ret = import_iovec(ITER_SOURCE, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
931 8165095 : if (ret >= 0) {
932 8165091 : file_start_write(file);
933 8165059 : ret = do_iter_write(file, &iter, pos, flags);
934 8164810 : file_end_write(file);
935 8164882 : kfree(iov);
936 : }
937 8164988 : return ret;
938 : }
939 :
940 1611567 : static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
941 : unsigned long vlen, rwf_t flags)
942 : {
943 1611567 : struct fd f = fdget_pos(fd);
944 1611566 : ssize_t ret = -EBADF;
945 :
946 1611566 : if (f.file) {
947 1611566 : loff_t pos, *ppos = file_ppos(f.file);
948 1577716 : if (ppos) {
949 1577716 : pos = *ppos;
950 1577716 : ppos = &pos;
951 : }
952 1611566 : ret = vfs_readv(f.file, vec, vlen, ppos, flags);
953 1611561 : if (ret >= 0 && ppos)
954 1577684 : f.file->f_pos = pos;
955 1611561 : fdput_pos(f);
956 : }
957 :
958 1611567 : if (ret > 0)
959 1611540 : add_rchar(current, ret);
960 1611567 : inc_syscr(current);
961 1611567 : return ret;
962 : }
963 :
964 8164980 : static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
965 : unsigned long vlen, rwf_t flags)
966 : {
967 8164980 : struct fd f = fdget_pos(fd);
968 8164996 : ssize_t ret = -EBADF;
969 :
970 8164996 : if (f.file) {
971 8164996 : loff_t pos, *ppos = file_ppos(f.file);
972 8128796 : if (ppos) {
973 8128796 : pos = *ppos;
974 8128796 : ppos = &pos;
975 : }
976 8164996 : ret = vfs_writev(f.file, vec, vlen, ppos, flags);
977 8164870 : if (ret >= 0 && ppos)
978 7775066 : f.file->f_pos = pos;
979 8164870 : fdput_pos(f);
980 : }
981 :
982 8164934 : if (ret > 0)
983 7811281 : add_wchar(current, ret);
984 8164934 : inc_syscw(current);
985 8164934 : return ret;
986 : }
987 :
988 : static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
989 : {
990 : #define HALF_LONG_BITS (BITS_PER_LONG / 2)
991 7 : return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
992 : }
993 :
994 0 : static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
995 : unsigned long vlen, loff_t pos, rwf_t flags)
996 : {
997 0 : struct fd f;
998 0 : ssize_t ret = -EBADF;
999 :
1000 0 : if (pos < 0)
1001 : return -EINVAL;
1002 :
1003 0 : f = fdget(fd);
1004 0 : if (f.file) {
1005 0 : ret = -ESPIPE;
1006 0 : if (f.file->f_mode & FMODE_PREAD)
1007 0 : ret = vfs_readv(f.file, vec, vlen, &pos, flags);
1008 0 : fdput(f);
1009 : }
1010 :
1011 0 : if (ret > 0)
1012 0 : add_rchar(current, ret);
1013 0 : inc_syscr(current);
1014 0 : return ret;
1015 : }
1016 :
1017 60 : static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
1018 : unsigned long vlen, loff_t pos, rwf_t flags)
1019 : {
1020 60 : struct fd f;
1021 60 : ssize_t ret = -EBADF;
1022 :
1023 60 : if (pos < 0)
1024 : return -EINVAL;
1025 :
1026 60 : f = fdget(fd);
1027 60 : if (f.file) {
1028 60 : ret = -ESPIPE;
1029 60 : if (f.file->f_mode & FMODE_PWRITE)
1030 60 : ret = vfs_writev(f.file, vec, vlen, &pos, flags);
1031 60 : fdput(f);
1032 : }
1033 :
1034 60 : if (ret > 0)
1035 54 : add_wchar(current, ret);
1036 60 : inc_syscw(current);
1037 60 : return ret;
1038 : }
1039 :
1040 3223137 : SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
1041 : unsigned long, vlen)
1042 : {
1043 1611569 : return do_readv(fd, vec, vlen, 0);
1044 : }
1045 :
1046 16329914 : SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
1047 : unsigned long, vlen)
1048 : {
1049 8165004 : return do_writev(fd, vec, vlen, 0);
1050 : }
1051 :
1052 0 : SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
1053 : unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1054 : {
1055 0 : loff_t pos = pos_from_hilo(pos_h, pos_l);
1056 :
1057 0 : return do_preadv(fd, vec, vlen, pos, 0);
1058 : }
1059 :
1060 0 : SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
1061 : unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1062 : rwf_t, flags)
1063 : {
1064 0 : loff_t pos = pos_from_hilo(pos_h, pos_l);
1065 :
1066 0 : if (pos == -1)
1067 0 : return do_readv(fd, vec, vlen, flags);
1068 :
1069 0 : return do_preadv(fd, vec, vlen, pos, flags);
1070 : }
1071 :
1072 106 : SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
1073 : unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1074 : {
1075 53 : loff_t pos = pos_from_hilo(pos_h, pos_l);
1076 :
1077 53 : return do_pwritev(fd, vec, vlen, pos, 0);
1078 : }
1079 :
1080 14 : SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
1081 : unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1082 : rwf_t, flags)
1083 : {
1084 7 : loff_t pos = pos_from_hilo(pos_h, pos_l);
1085 :
1086 7 : if (pos == -1)
1087 0 : return do_writev(fd, vec, vlen, flags);
1088 :
1089 7 : return do_pwritev(fd, vec, vlen, pos, flags);
1090 : }
1091 :
1092 : /*
1093 : * Various compat syscalls. Note that they all pretend to take a native
1094 : * iovec - import_iovec will properly treat those as compat_iovecs based on
1095 : * in_compat_syscall().
1096 : */
1097 : #ifdef CONFIG_COMPAT
1098 : #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
1099 0 : COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1100 : const struct iovec __user *, vec,
1101 : unsigned long, vlen, loff_t, pos)
1102 : {
1103 0 : return do_preadv(fd, vec, vlen, pos, 0);
1104 : }
1105 : #endif
1106 :
1107 0 : COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
1108 : const struct iovec __user *, vec,
1109 : compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1110 : {
1111 0 : loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1112 :
1113 0 : return do_preadv(fd, vec, vlen, pos, 0);
1114 : }
1115 :
1116 : #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
1117 0 : COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
1118 : const struct iovec __user *, vec,
1119 : unsigned long, vlen, loff_t, pos, rwf_t, flags)
1120 : {
1121 0 : if (pos == -1)
1122 0 : return do_readv(fd, vec, vlen, flags);
1123 0 : return do_preadv(fd, vec, vlen, pos, flags);
1124 : }
1125 : #endif
1126 :
1127 0 : COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
1128 : const struct iovec __user *, vec,
1129 : compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
1130 : rwf_t, flags)
1131 : {
1132 0 : loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1133 :
1134 0 : if (pos == -1)
1135 0 : return do_readv(fd, vec, vlen, flags);
1136 0 : return do_preadv(fd, vec, vlen, pos, flags);
1137 : }
1138 :
1139 : #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
1140 0 : COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1141 : const struct iovec __user *, vec,
1142 : unsigned long, vlen, loff_t, pos)
1143 : {
1144 0 : return do_pwritev(fd, vec, vlen, pos, 0);
1145 : }
1146 : #endif
1147 :
1148 0 : COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
1149 : const struct iovec __user *,vec,
1150 : compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1151 : {
1152 0 : loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1153 :
1154 0 : return do_pwritev(fd, vec, vlen, pos, 0);
1155 : }
1156 :
1157 : #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
1158 0 : COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
1159 : const struct iovec __user *, vec,
1160 : unsigned long, vlen, loff_t, pos, rwf_t, flags)
1161 : {
1162 0 : if (pos == -1)
1163 0 : return do_writev(fd, vec, vlen, flags);
1164 0 : return do_pwritev(fd, vec, vlen, pos, flags);
1165 : }
1166 : #endif
1167 :
1168 0 : COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
1169 : const struct iovec __user *,vec,
1170 : compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
1171 : {
1172 0 : loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1173 :
1174 0 : if (pos == -1)
1175 0 : return do_writev(fd, vec, vlen, flags);
1176 0 : return do_pwritev(fd, vec, vlen, pos, flags);
1177 : }
1178 : #endif /* CONFIG_COMPAT */
1179 :
1180 13 : static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1181 : size_t count, loff_t max)
1182 : {
1183 13 : struct fd in, out;
1184 13 : struct inode *in_inode, *out_inode;
1185 13 : struct pipe_inode_info *opipe;
1186 13 : loff_t pos;
1187 13 : loff_t out_pos;
1188 13 : ssize_t retval;
1189 13 : int fl;
1190 :
1191 : /*
1192 : * Get input file, and verify that it is ok..
1193 : */
1194 13 : retval = -EBADF;
1195 13 : in = fdget(in_fd);
1196 13 : if (!in.file)
1197 0 : goto out;
1198 13 : if (!(in.file->f_mode & FMODE_READ))
1199 0 : goto fput_in;
1200 13 : retval = -ESPIPE;
1201 13 : if (!ppos) {
1202 0 : pos = in.file->f_pos;
1203 : } else {
1204 13 : pos = *ppos;
1205 13 : if (!(in.file->f_mode & FMODE_PREAD))
1206 0 : goto fput_in;
1207 : }
1208 13 : retval = rw_verify_area(READ, in.file, &pos, count);
1209 13 : if (retval < 0)
1210 0 : goto fput_in;
1211 13 : if (count > MAX_RW_COUNT)
1212 : count = MAX_RW_COUNT;
1213 :
1214 : /*
1215 : * Get output file, and verify that it is ok..
1216 : */
1217 13 : retval = -EBADF;
1218 13 : out = fdget(out_fd);
1219 13 : if (!out.file)
1220 0 : goto fput_in;
1221 13 : if (!(out.file->f_mode & FMODE_WRITE))
1222 0 : goto fput_out;
1223 13 : in_inode = file_inode(in.file);
1224 13 : out_inode = file_inode(out.file);
1225 13 : out_pos = out.file->f_pos;
1226 :
1227 13 : if (!max)
1228 13 : max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1229 :
1230 13 : if (unlikely(pos + count > max)) {
1231 0 : retval = -EOVERFLOW;
1232 0 : if (pos >= max)
1233 0 : goto fput_out;
1234 0 : count = max - pos;
1235 : }
1236 :
1237 13 : fl = 0;
1238 : #if 0
1239 : /*
1240 : * We need to debate whether we can enable this or not. The
1241 : * man page documents EAGAIN return for the output at least,
1242 : * and the application is arguably buggy if it doesn't expect
1243 : * EAGAIN on a non-blocking file descriptor.
1244 : */
1245 : if (in.file->f_flags & O_NONBLOCK)
1246 : fl = SPLICE_F_NONBLOCK;
1247 : #endif
1248 13 : opipe = get_pipe_info(out.file, true);
1249 13 : if (!opipe) {
1250 13 : retval = rw_verify_area(WRITE, out.file, &out_pos, count);
1251 13 : if (retval < 0)
1252 0 : goto fput_out;
1253 13 : file_start_write(out.file);
1254 13 : retval = do_splice_direct(in.file, &pos, out.file, &out_pos,
1255 : count, fl);
1256 13 : file_end_write(out.file);
1257 : } else {
1258 0 : if (out.file->f_flags & O_NONBLOCK)
1259 0 : fl |= SPLICE_F_NONBLOCK;
1260 :
1261 0 : retval = splice_file_to_pipe(in.file, opipe, &pos, count, fl);
1262 : }
1263 :
1264 13 : if (retval > 0) {
1265 13 : add_rchar(current, retval);
1266 13 : add_wchar(current, retval);
1267 13 : fsnotify_access(in.file);
1268 13 : fsnotify_modify(out.file);
1269 13 : out.file->f_pos = out_pos;
1270 13 : if (ppos)
1271 13 : *ppos = pos;
1272 : else
1273 0 : in.file->f_pos = pos;
1274 : }
1275 :
1276 13 : inc_syscr(current);
1277 13 : inc_syscw(current);
1278 13 : if (pos > max)
1279 0 : retval = -EOVERFLOW;
1280 :
1281 13 : fput_out:
1282 13 : fdput(out);
1283 13 : fput_in:
1284 13 : fdput(in);
1285 13 : out:
1286 13 : return retval;
1287 : }
1288 :
1289 0 : SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
1290 : {
1291 0 : loff_t pos;
1292 0 : off_t off;
1293 0 : ssize_t ret;
1294 :
1295 0 : if (offset) {
1296 0 : if (unlikely(get_user(off, offset)))
1297 : return -EFAULT;
1298 0 : pos = off;
1299 0 : ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1300 0 : if (unlikely(put_user(pos, offset)))
1301 : return -EFAULT;
1302 0 : return ret;
1303 : }
1304 :
1305 0 : return do_sendfile(out_fd, in_fd, NULL, count, 0);
1306 : }
1307 :
1308 26 : SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
1309 : {
1310 13 : loff_t pos;
1311 13 : ssize_t ret;
1312 :
1313 13 : if (offset) {
1314 13 : if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1315 : return -EFAULT;
1316 13 : ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1317 13 : if (unlikely(put_user(pos, offset)))
1318 : return -EFAULT;
1319 13 : return ret;
1320 : }
1321 :
1322 0 : return do_sendfile(out_fd, in_fd, NULL, count, 0);
1323 : }
1324 :
1325 : #ifdef CONFIG_COMPAT
1326 0 : COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1327 : compat_off_t __user *, offset, compat_size_t, count)
1328 : {
1329 0 : loff_t pos;
1330 0 : off_t off;
1331 0 : ssize_t ret;
1332 :
1333 0 : if (offset) {
1334 0 : if (unlikely(get_user(off, offset)))
1335 : return -EFAULT;
1336 0 : pos = off;
1337 0 : ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1338 0 : if (unlikely(put_user(pos, offset)))
1339 : return -EFAULT;
1340 0 : return ret;
1341 : }
1342 :
1343 0 : return do_sendfile(out_fd, in_fd, NULL, count, 0);
1344 : }
1345 :
1346 0 : COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1347 : compat_loff_t __user *, offset, compat_size_t, count)
1348 : {
1349 0 : loff_t pos;
1350 0 : ssize_t ret;
1351 :
1352 0 : if (offset) {
1353 0 : if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1354 : return -EFAULT;
1355 0 : ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1356 0 : if (unlikely(put_user(pos, offset)))
1357 : return -EFAULT;
1358 0 : return ret;
1359 : }
1360 :
1361 0 : return do_sendfile(out_fd, in_fd, NULL, count, 0);
1362 : }
1363 : #endif
1364 :
1365 : /**
1366 : * generic_copy_file_range - copy data between two files
1367 : * @file_in: file structure to read from
1368 : * @pos_in: file offset to read from
1369 : * @file_out: file structure to write data to
1370 : * @pos_out: file offset to write data to
1371 : * @len: amount of data to copy
1372 : * @flags: copy flags
1373 : *
1374 : * This is a generic filesystem helper to copy data from one file to another.
1375 : * It has no constraints on the source or destination file owners - the files
1376 : * can belong to different superblocks and different filesystem types. Short
1377 : * copies are allowed.
1378 : *
1379 : * This should be called from the @file_out filesystem, as per the
1380 : * ->copy_file_range() method.
1381 : *
1382 : * Returns the number of bytes copied or a negative error indicating the
1383 : * failure.
1384 : */
1385 :
1386 0 : ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
1387 : struct file *file_out, loff_t pos_out,
1388 : size_t len, unsigned int flags)
1389 : {
1390 9190507 : lockdep_assert(sb_write_started(file_inode(file_out)->i_sb));
1391 :
1392 9190507 : return do_splice_direct(file_in, &pos_in, file_out, &pos_out,
1393 0 : len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
1394 : }
1395 : EXPORT_SYMBOL(generic_copy_file_range);
1396 :
1397 : /*
1398 : * Performs necessary checks before doing a file copy
1399 : *
1400 : * Can adjust amount of bytes to copy via @req_count argument.
1401 : * Returns appropriate error code that caller should return or
1402 : * zero in case the copy should be allowed.
1403 : */
1404 9810061 : static int generic_copy_file_checks(struct file *file_in, loff_t pos_in,
1405 : struct file *file_out, loff_t pos_out,
1406 : size_t *req_count, unsigned int flags)
1407 : {
1408 9810061 : struct inode *inode_in = file_inode(file_in);
1409 9810061 : struct inode *inode_out = file_inode(file_out);
1410 9810061 : uint64_t count = *req_count;
1411 9810061 : loff_t size_in;
1412 9810061 : int ret;
1413 :
1414 9810061 : ret = generic_file_rw_checks(file_in, file_out);
1415 9810061 : if (ret)
1416 : return ret;
1417 :
1418 : /*
1419 : * We allow some filesystems to handle cross sb copy, but passing
1420 : * a file of the wrong filesystem type to filesystem driver can result
1421 : * in an attempt to dereference the wrong type of ->private_data, so
1422 : * avoid doing that until we really have a good reason.
1423 : *
1424 : * nfs and cifs define several different file_system_type structures
1425 : * and several different sets of file_operations, but they all end up
1426 : * using the same ->copy_file_range() function pointer.
1427 : */
1428 9809879 : if (flags & COPY_FILE_SPLICE) {
1429 : /* cross sb splice is allowed */
1430 9809910 : } else if (file_out->f_op->copy_file_range) {
1431 0 : if (file_in->f_op->copy_file_range !=
1432 : file_out->f_op->copy_file_range)
1433 : return -EXDEV;
1434 9809910 : } else if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) {
1435 : return -EXDEV;
1436 : }
1437 :
1438 : /* Don't touch certain kinds of inodes */
1439 9809866 : if (IS_IMMUTABLE(inode_out))
1440 : return -EPERM;
1441 :
1442 9809853 : if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
1443 : return -ETXTBSY;
1444 :
1445 : /* Ensure offsets don't wrap. */
1446 9809840 : if (pos_in + count < pos_in || pos_out + count < pos_out)
1447 : return -EOVERFLOW;
1448 :
1449 : /* Shorten the copy to EOF */
1450 9809814 : size_in = i_size_read(inode_in);
1451 9809814 : if (pos_in >= size_in)
1452 601 : count = 0;
1453 : else
1454 9809213 : count = min(count, size_in - (uint64_t)pos_in);
1455 :
1456 9809814 : ret = generic_write_check_limits(file_out, pos_out, &count);
1457 9810084 : if (ret)
1458 : return ret;
1459 :
1460 : /* Don't allow overlapped copying within the same file. */
1461 9810058 : if (inode_in == inode_out &&
1462 2922331 : pos_out + count > pos_in &&
1463 1739833 : pos_out < pos_in + count)
1464 : return -EINVAL;
1465 :
1466 9810045 : *req_count = count;
1467 9810045 : return 0;
1468 : }
1469 :
1470 : /*
1471 : * copy_file_range() differs from regular file read and write in that it
1472 : * specifically allows return partial success. When it does so is up to
1473 : * the copy_file_range method.
1474 : */
1475 9810443 : ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
1476 : struct file *file_out, loff_t pos_out,
1477 : size_t len, unsigned int flags)
1478 : {
1479 9810443 : ssize_t ret;
1480 9810443 : bool splice = flags & COPY_FILE_SPLICE;
1481 :
1482 9810443 : if (flags & ~COPY_FILE_SPLICE)
1483 : return -EINVAL;
1484 :
1485 9810443 : ret = generic_copy_file_checks(file_in, pos_in, file_out, pos_out, &len,
1486 : flags);
1487 9810246 : if (unlikely(ret))
1488 : return ret;
1489 :
1490 9809960 : ret = rw_verify_area(READ, file_in, &pos_in, len);
1491 9809960 : if (unlikely(ret))
1492 : return ret;
1493 :
1494 9809960 : ret = rw_verify_area(WRITE, file_out, &pos_out, len);
1495 9809960 : if (unlikely(ret))
1496 : return ret;
1497 :
1498 9809960 : if (len == 0)
1499 : return 0;
1500 :
1501 9809393 : file_start_write(file_out);
1502 :
1503 : /*
1504 : * Cloning is supported by more file systems, so we implement copy on
1505 : * same sb using clone, but for filesystems where both clone and copy
1506 : * are supported (e.g. nfs,cifs), we only call the copy method.
1507 : */
1508 9809517 : if (!splice && file_out->f_op->copy_file_range) {
1509 0 : ret = file_out->f_op->copy_file_range(file_in, pos_in,
1510 : file_out, pos_out,
1511 : len, flags);
1512 0 : goto done;
1513 : }
1514 :
1515 9809517 : if (!splice && file_in->f_op->remap_file_range &&
1516 9570867 : file_inode(file_in)->i_sb == file_inode(file_out)->i_sb) {
1517 9570851 : ret = file_in->f_op->remap_file_range(file_in, pos_in,
1518 : file_out, pos_out,
1519 9570851 : min_t(loff_t, MAX_RW_COUNT, len),
1520 : REMAP_FILE_CAN_SHORTEN);
1521 9570918 : if (ret > 0)
1522 619077 : goto done;
1523 : }
1524 :
1525 : /*
1526 : * We can get here for same sb copy of filesystems that do not implement
1527 : * ->copy_file_range() in case filesystem does not support clone or in
1528 : * case filesystem supports clone but rejected the clone request (e.g.
1529 : * because it was not block aligned).
1530 : *
1531 : * In both cases, fall back to kernel copy so we are able to maintain a
1532 : * consistent story about which filesystems support copy_file_range()
1533 : * and which filesystems do not, that will allow userspace tools to
1534 : * make consistent desicions w.r.t using copy_file_range().
1535 : *
1536 : * We also get here if caller (e.g. nfsd) requested COPY_FILE_SPLICE.
1537 : */
1538 9190507 : ret = generic_copy_file_range(file_in, pos_in, file_out, pos_out, len,
1539 : flags);
1540 :
1541 9809599 : done:
1542 9809599 : if (ret > 0) {
1543 9506916 : fsnotify_access(file_in);
1544 9506928 : add_rchar(current, ret);
1545 9506928 : fsnotify_modify(file_out);
1546 9506935 : add_wchar(current, ret);
1547 : }
1548 :
1549 9809618 : inc_syscr(current);
1550 9809618 : inc_syscw(current);
1551 :
1552 9809618 : file_end_write(file_out);
1553 :
1554 9809618 : return ret;
1555 : }
1556 : EXPORT_SYMBOL(vfs_copy_file_range);
1557 :
1558 19620922 : SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
1559 : int, fd_out, loff_t __user *, off_out,
1560 : size_t, len, unsigned int, flags)
1561 : {
1562 9810392 : loff_t pos_in;
1563 9810392 : loff_t pos_out;
1564 9810392 : struct fd f_in;
1565 9810392 : struct fd f_out;
1566 9810392 : ssize_t ret = -EBADF;
1567 :
1568 9810392 : f_in = fdget(fd_in);
1569 9810449 : if (!f_in.file)
1570 0 : goto out2;
1571 :
1572 9810449 : f_out = fdget(fd_out);
1573 9810493 : if (!f_out.file)
1574 0 : goto out1;
1575 :
1576 9810493 : ret = -EFAULT;
1577 9810493 : if (off_in) {
1578 9810493 : if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
1579 0 : goto out;
1580 : } else {
1581 0 : pos_in = f_in.file->f_pos;
1582 : }
1583 :
1584 9810444 : if (off_out) {
1585 9810444 : if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
1586 0 : goto out;
1587 : } else {
1588 0 : pos_out = f_out.file->f_pos;
1589 : }
1590 :
1591 9810426 : ret = -EINVAL;
1592 9810426 : if (flags != 0)
1593 0 : goto out;
1594 :
1595 9810426 : ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len,
1596 : flags);
1597 9810463 : if (ret > 0) {
1598 9506915 : pos_in += ret;
1599 9506915 : pos_out += ret;
1600 :
1601 9506915 : if (off_in) {
1602 9506915 : if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
1603 0 : ret = -EFAULT;
1604 : } else {
1605 0 : f_in.file->f_pos = pos_in;
1606 : }
1607 :
1608 9506924 : if (off_out) {
1609 9506924 : if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
1610 0 : ret = -EFAULT;
1611 : } else {
1612 0 : f_out.file->f_pos = pos_out;
1613 : }
1614 : }
1615 :
1616 303548 : out:
1617 9810476 : fdput(f_out);
1618 9810476 : out1:
1619 9810476 : fdput(f_in);
1620 9810476 : out2:
1621 9810476 : return ret;
1622 : }
1623 :
1624 : /*
1625 : * Don't operate on ranges the page cache doesn't support, and don't exceed the
1626 : * LFS limits. If pos is under the limit it becomes a short access. If it
1627 : * exceeds the limit we return -EFBIG.
1628 : */
1629 1129612278 : int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count)
1630 : {
1631 1129612278 : struct inode *inode = file->f_mapping->host;
1632 1129612278 : loff_t max_size = inode->i_sb->s_maxbytes;
1633 1129612278 : loff_t limit = rlimit(RLIMIT_FSIZE);
1634 :
1635 1129612278 : if (limit != RLIM_INFINITY) {
1636 341 : if (pos >= limit) {
1637 13 : send_sig(SIGXFSZ, current, 0);
1638 13 : return -EFBIG;
1639 : }
1640 328 : *count = min(*count, limit - pos);
1641 : }
1642 :
1643 1129612265 : if (!(file->f_flags & O_LARGEFILE))
1644 0 : max_size = MAX_NON_LFS;
1645 :
1646 1129612265 : if (unlikely(pos >= max_size))
1647 : return -EFBIG;
1648 :
1649 1129612250 : *count = min(*count, max_size - pos);
1650 :
1651 1129612250 : return 0;
1652 : }
1653 : EXPORT_SYMBOL(generic_write_check_limits);
1654 :
1655 : /* Like generic_write_checks(), but takes size of write instead of iter. */
1656 779302450 : int generic_write_checks_count(struct kiocb *iocb, loff_t *count)
1657 : {
1658 779302450 : struct file *file = iocb->ki_filp;
1659 779302450 : struct inode *inode = file->f_mapping->host;
1660 :
1661 779302450 : if (IS_SWAPFILE(inode))
1662 : return -ETXTBSY;
1663 :
1664 779302414 : if (!*count)
1665 : return 0;
1666 :
1667 779302359 : if (iocb->ki_flags & IOCB_APPEND)
1668 49785754 : iocb->ki_pos = i_size_read(inode);
1669 :
1670 779302359 : if ((iocb->ki_flags & IOCB_NOWAIT) &&
1671 7 : !((iocb->ki_flags & IOCB_DIRECT) ||
1672 0 : (file->f_mode & FMODE_BUF_WASYNC)))
1673 : return -EINVAL;
1674 :
1675 779302359 : return generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, count);
1676 : }
1677 : EXPORT_SYMBOL(generic_write_checks_count);
1678 :
1679 : /*
1680 : * Performs necessary checks before doing a write
1681 : *
1682 : * Can adjust writing position or amount of bytes to write.
1683 : * Returns appropriate error code that caller should return or
1684 : * zero in case that write should be allowed.
1685 : */
1686 779408390 : ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
1687 : {
1688 779408390 : loff_t count = iov_iter_count(from);
1689 779408390 : int ret;
1690 :
1691 779408390 : ret = generic_write_checks_count(iocb, &count);
1692 779992656 : if (ret)
1693 38 : return ret;
1694 :
1695 779992618 : iov_iter_truncate(from, count);
1696 779992618 : return iov_iter_count(from);
1697 : }
1698 : EXPORT_SYMBOL(generic_write_checks);
1699 :
1700 : /*
1701 : * Performs common checks before doing a file copy/clone
1702 : * from @file_in to @file_out.
1703 : */
1704 30911806 : int generic_file_rw_checks(struct file *file_in, struct file *file_out)
1705 : {
1706 30911806 : struct inode *inode_in = file_inode(file_in);
1707 30911806 : struct inode *inode_out = file_inode(file_out);
1708 :
1709 : /* Don't copy dirs, pipes, sockets... */
1710 30911806 : if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1711 : return -EISDIR;
1712 30911769 : if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1713 : return -EINVAL;
1714 :
1715 30911632 : if (!(file_in->f_mode & FMODE_READ) ||
1716 27806650 : !(file_out->f_mode & FMODE_WRITE) ||
1717 27806614 : (file_out->f_flags & O_APPEND))
1718 3105055 : return -EBADF;
1719 :
1720 : return 0;
1721 : }
1722 : EXPORT_SYMBOL(generic_file_rw_checks);
|
