Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : #include <linux/slab.h>
3 : #include <linux/stat.h>
4 : #include <linux/sched/xacct.h>
5 : #include <linux/fcntl.h>
6 : #include <linux/file.h>
7 : #include <linux/uio.h>
8 : #include <linux/fsnotify.h>
9 : #include <linux/security.h>
10 : #include <linux/export.h>
11 : #include <linux/syscalls.h>
12 : #include <linux/pagemap.h>
13 : #include <linux/splice.h>
14 : #include <linux/compat.h>
15 : #include <linux/mount.h>
16 : #include <linux/fs.h>
17 : #include <linux/dax.h>
18 : #include <linux/overflow.h>
19 : #include "internal.h"
20 :
21 : #include <linux/uaccess.h>
22 : #include <asm/unistd.h>
23 :
24 : /*
25 : * Performs necessary checks before doing a clone.
26 : *
27 : * Can adjust amount of bytes to clone via @req_count argument.
28 : * Returns appropriate error code that caller should return or
29 : * zero in case the clone should be allowed.
30 : */
31 387399905 : static int generic_remap_checks(struct file *file_in, loff_t pos_in,
32 : struct file *file_out, loff_t pos_out,
33 : loff_t *req_count, unsigned int remap_flags,
34 : unsigned int blocksize)
35 : {
36 387399905 : struct inode *inode_in = file_in->f_mapping->host;
37 387399905 : struct inode *inode_out = file_out->f_mapping->host;
38 387399905 : uint64_t count = *req_count;
39 387399905 : uint64_t bcount;
40 387399905 : loff_t size_in, size_out;
41 387399905 : int ret;
42 :
43 : /* The start of both ranges must be aligned to an fs block. */
44 387399905 : if (!IS_ALIGNED(pos_in, blocksize) || !IS_ALIGNED(pos_out, blocksize))
45 : return -EINVAL;
46 :
47 : /* Ensure offsets don't wrap. */
48 379264874 : if (pos_in + count < pos_in || pos_out + count < pos_out)
49 : return -EINVAL;
50 :
51 379264874 : size_in = i_size_read(inode_in);
52 379264874 : size_out = i_size_read(inode_out);
53 :
54 : /* Dedupe requires both ranges to be within EOF. */
55 379264874 : if ((remap_flags & REMAP_FILE_DEDUP) &&
56 368745106 : (pos_in >= size_in || pos_in + count > size_in ||
57 326724950 : pos_out >= size_out || pos_out + count > size_out))
58 : return -EINVAL;
59 :
60 : /* Ensure the infile range is within the infile. */
61 336314714 : if (pos_in >= size_in)
62 : return -EINVAL;
63 336314703 : count = min(count, size_in - (uint64_t)pos_in);
64 :
65 336314703 : ret = generic_write_check_limits(file_out, pos_out, &count);
66 336317122 : if (ret)
67 : return ret;
68 :
69 : /*
70 : * If the user wanted us to link to the infile's EOF, round up to the
71 : * next block boundary for this check.
72 : *
73 : * Otherwise, make sure the count is also block-aligned, having
74 : * already confirmed the starting offsets' block alignment.
75 : */
76 336317122 : if (pos_in + count == size_in &&
77 238913 : (!(remap_flags & REMAP_FILE_DEDUP) || pos_out + count == size_out)) {
78 465233 : bcount = ALIGN(size_in, blocksize) - pos_in;
79 : } else {
80 335851889 : if (!IS_ALIGNED(count, blocksize))
81 18616032 : count = ALIGN_DOWN(count, blocksize);
82 335851889 : bcount = count;
83 : }
84 :
85 : /* Don't allow overlapped cloning within the same file. */
86 336317122 : if (inode_in == inode_out &&
87 6430522 : pos_out + bcount > pos_in &&
88 3505963 : pos_out < pos_in + bcount)
89 : return -EINVAL;
90 :
91 : /*
92 : * We shortened the request but the caller can't deal with that, so
93 : * bounce the request back to userspace.
94 : */
95 336303639 : if (*req_count != count && !(remap_flags & REMAP_FILE_CAN_SHORTEN))
96 : return -EINVAL;
97 :
98 336303573 : *req_count = count;
99 336303573 : return 0;
100 : }
101 :
102 3724129 : int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write)
103 : {
104 895905834 : loff_t tmp;
105 :
106 3724129 : if (unlikely(pos < 0 || len < 0))
107 : return -EINVAL;
108 :
109 895905834 : if (unlikely(check_add_overflow(pos, len, &tmp)))
110 22 : return -EINVAL;
111 :
112 : return security_file_permission(file, write ? MAY_WRITE : MAY_READ);
113 : }
114 : EXPORT_SYMBOL(remap_verify_area);
115 :
116 : /*
117 : * Ensure that we don't remap a partial EOF block in the middle of something
118 : * else. Assume that the offsets have already been checked for block
119 : * alignment.
120 : *
121 : * For clone we only link a partial EOF block above or at the destination file's
122 : * EOF. For deduplication we accept a partial EOF block only if it ends at the
123 : * destination file's EOF (can not link it into the middle of a file).
124 : *
125 : * Shorten the request if possible.
126 : */
127 244877485 : static int generic_remap_check_len(struct inode *inode_in,
128 : struct inode *inode_out,
129 : loff_t pos_out,
130 : loff_t *len,
131 : unsigned int remap_flags,
132 : unsigned int blocksize)
133 : {
134 244877485 : u64 blkmask = blocksize - 1;
135 244877485 : loff_t new_len = *len;
136 :
137 244877485 : if ((*len & blkmask) == 0)
138 : return 0;
139 :
140 29324 : if (pos_out + *len < i_size_read(inode_out))
141 7014 : new_len &= ~blkmask;
142 :
143 29324 : if (new_len == *len)
144 : return 0;
145 :
146 7014 : if (remap_flags & REMAP_FILE_CAN_SHORTEN) {
147 0 : *len = new_len;
148 0 : return 0;
149 : }
150 :
151 7014 : return (remap_flags & REMAP_FILE_DEDUP) ? -EBADE : -EINVAL;
152 : }
153 :
154 : /* Read a page's worth of file data into the page cache. */
155 8028801940 : static struct folio *vfs_dedupe_get_folio(struct file *file, loff_t pos)
156 : {
157 8028801940 : return read_mapping_folio(file->f_mapping, pos >> PAGE_SHIFT, file);
158 : }
159 :
160 : /*
161 : * Lock two folios, ensuring that we lock in offset order if the folios
162 : * are from the same file.
163 : */
164 4016277418 : static void vfs_lock_two_folios(struct folio *folio1, struct folio *folio2)
165 : {
166 : /* Always lock in order of increasing index. */
167 4016277418 : if (folio1->index > folio2->index)
168 2007295132 : swap(folio1, folio2);
169 :
170 4016277418 : folio_lock(folio1);
171 4016615598 : if (folio1 != folio2)
172 4016614397 : folio_lock(folio2);
173 4016481088 : }
174 :
175 : /* Unlock two folios, being careful not to unlock the same folio twice. */
176 4016437704 : static void vfs_unlock_two_folios(struct folio *folio1, struct folio *folio2)
177 : {
178 4016437704 : folio_unlock(folio1);
179 4017348705 : if (folio1 != folio2)
180 4017347504 : folio_unlock(folio2);
181 4016944809 : }
182 :
183 : /*
184 : * Compare extents of two files to see if they are the same.
185 : * Caller must have locked both inodes to prevent write races.
186 : */
187 309611124 : static int vfs_dedupe_file_range_compare(struct file *src, loff_t srcoff,
188 : struct file *dest, loff_t dstoff,
189 : loff_t len, bool *is_same)
190 : {
191 309611124 : bool same = true;
192 309611124 : int error = -EINVAL;
193 :
194 4250840564 : while (len) {
195 4016480782 : struct folio *src_folio, *dst_folio;
196 4016480782 : void *src_addr, *dst_addr;
197 4016480782 : loff_t cmp_len = min(PAGE_SIZE - offset_in_page(srcoff),
198 : PAGE_SIZE - offset_in_page(dstoff));
199 :
200 4016480782 : cmp_len = min(cmp_len, len);
201 4016480782 : if (cmp_len <= 0)
202 0 : goto out_error;
203 :
204 4016480782 : src_folio = vfs_dedupe_get_folio(src, srcoff);
205 4016043253 : if (IS_ERR(src_folio)) {
206 854 : error = PTR_ERR(src_folio);
207 854 : goto out_error;
208 : }
209 4016042399 : dst_folio = vfs_dedupe_get_folio(dest, dstoff);
210 4016615374 : if (IS_ERR(dst_folio)) {
211 1579 : error = PTR_ERR(dst_folio);
212 1579 : folio_put(src_folio);
213 1579 : goto out_error;
214 : }
215 :
216 4016613795 : vfs_lock_two_folios(src_folio, dst_folio);
217 :
218 : /*
219 : * Now that we've locked both folios, make sure they're still
220 : * mapped to the file data we're interested in. If not,
221 : * someone is invalidating pages on us and we lose.
222 : */
223 12048351717 : if (!folio_test_uptodate(src_folio) || !folio_test_uptodate(dst_folio) ||
224 4015781341 : src_folio->mapping != src->f_mapping ||
225 4015781341 : dst_folio->mapping != dest->f_mapping) {
226 0 : same = false;
227 0 : goto unlock;
228 : }
229 :
230 8031158616 : src_addr = kmap_local_folio(src_folio,
231 4015781341 : offset_in_folio(src_folio, srcoff));
232 8030876996 : dst_addr = kmap_local_folio(dst_folio,
233 4015377275 : offset_in_folio(dst_folio, dstoff));
234 :
235 4015499721 : flush_dcache_folio(src_folio);
236 4015499721 : flush_dcache_folio(dst_folio);
237 :
238 8030999442 : if (memcmp(src_addr, dst_addr, cmp_len))
239 75276920 : same = false;
240 :
241 : kunmap_local(dst_addr);
242 4015499721 : kunmap_local(src_addr);
243 4015499721 : unlock:
244 4015499721 : vfs_unlock_two_folios(src_folio, dst_folio);
245 4016819680 : folio_put(dst_folio);
246 4016778040 : folio_put(src_folio);
247 :
248 4016506144 : if (!same)
249 : break;
250 :
251 3941229440 : srcoff += cmp_len;
252 3941229440 : dstoff += cmp_len;
253 3941229440 : len -= cmp_len;
254 : }
255 :
256 309636486 : *is_same = same;
257 309636486 : return 0;
258 :
259 : out_error:
260 : return error;
261 : }
262 :
263 : /*
264 : * Check that the two inodes are eligible for cloning, the ranges make
265 : * sense, and then flush all dirty data. Caller must ensure that the
266 : * inodes have been locked against any other modifications.
267 : *
268 : * If there's an error, then the usual negative error code is returned.
269 : * Otherwise returns 0 with *len set to the request length.
270 : */
271 : int
272 388319232 : __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
273 : struct file *file_out, loff_t pos_out,
274 : loff_t *len, unsigned int remap_flags,
275 : const struct iomap_ops *dax_read_ops,
276 : unsigned int blocksize)
277 : {
278 388319232 : struct inode *inode_in = file_inode(file_in);
279 388319232 : struct inode *inode_out = file_inode(file_out);
280 388319232 : bool same_inode = (inode_in == inode_out);
281 388319232 : int ret;
282 :
283 : /* Don't touch certain kinds of inodes */
284 388319232 : if (IS_IMMUTABLE(inode_out))
285 : return -EPERM;
286 :
287 388319232 : if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
288 : return -ETXTBSY;
289 :
290 : /* Don't reflink dirs, pipes, sockets... */
291 388319199 : if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
292 : return -EISDIR;
293 388319199 : if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
294 : return -EINVAL;
295 :
296 : /* Zero length dedupe exits immediately; reflink goes to EOF. */
297 388319199 : if (*len == 0) {
298 1062833 : loff_t isize = i_size_read(inode_in);
299 :
300 1062833 : if ((remap_flags & REMAP_FILE_DEDUP) || pos_in == isize)
301 : return 0;
302 126365 : if (pos_in > isize)
303 : return -EINVAL;
304 126365 : *len = isize - pos_in;
305 126365 : if (*len == 0)
306 : return 0;
307 : }
308 :
309 : /* Check that we don't violate system file offset limits. */
310 387382731 : ret = generic_remap_checks(file_in, pos_in, file_out, pos_out, len,
311 : remap_flags, blocksize);
312 387388703 : if (ret || *len == 0)
313 : return ret;
314 :
315 : /* Wait for the completion of any pending IOs on both files */
316 320113479 : inode_dio_wait(inode_in);
317 320110160 : if (!same_inode)
318 313783721 : inode_dio_wait(inode_out);
319 :
320 320110795 : ret = filemap_write_and_wait_range(inode_in->i_mapping,
321 320110795 : pos_in, pos_in + *len - 1);
322 320109956 : if (ret)
323 : return ret;
324 :
325 320112120 : ret = filemap_write_and_wait_range(inode_out->i_mapping,
326 320112120 : pos_out, pos_out + *len - 1);
327 320136604 : if (ret)
328 : return ret;
329 :
330 : /*
331 : * Check that the extents are the same.
332 : */
333 320135473 : if (remap_flags & REMAP_FILE_DEDUP) {
334 309615023 : bool is_same = false;
335 :
336 309615023 : if (!IS_DAX(inode_in))
337 309615023 : ret = vfs_dedupe_file_range_compare(file_in, pos_in,
338 : file_out, pos_out, *len, &is_same);
339 0 : else if (dax_read_ops)
340 0 : ret = dax_dedupe_file_range_compare(inode_in, pos_in,
341 : inode_out, pos_out, *len, &is_same,
342 : dax_read_ops);
343 : else
344 75278918 : return -EINVAL;
345 309636846 : if (ret)
346 : return ret;
347 309634413 : if (!is_same)
348 : return -EBADE;
349 : }
350 :
351 244878378 : ret = generic_remap_check_len(inode_in, inode_out, pos_out, len,
352 : remap_flags, blocksize);
353 244877334 : if (ret || *len == 0)
354 : return ret;
355 :
356 : /* If can't alter the file contents, we're done. */
357 244870320 : if (!(remap_flags & REMAP_FILE_DEDUP))
358 10513309 : ret = file_modified(file_out);
359 :
360 : return ret;
361 : }
362 : EXPORT_SYMBOL(__generic_remap_file_range_prep);
363 :
364 2374424 : int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
365 : struct file *file_out, loff_t pos_out,
366 : loff_t *len, unsigned int remap_flags)
367 : {
368 2374424 : unsigned int blocksize = file_inode(file_out)->i_sb->s_blocksize;
369 :
370 2374424 : return __generic_remap_file_range_prep(file_in, pos_in, file_out,
371 : pos_out, len, remap_flags, NULL,
372 : blocksize);
373 : }
374 : EXPORT_SYMBOL(generic_remap_file_range_prep);
375 :
376 11167925 : loff_t do_clone_file_range(struct file *file_in, loff_t pos_in,
377 : struct file *file_out, loff_t pos_out,
378 : loff_t len, unsigned int remap_flags)
379 : {
380 11167925 : loff_t ret;
381 :
382 11167925 : WARN_ON_ONCE(remap_flags & REMAP_FILE_DEDUP);
383 :
384 11167925 : if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
385 : return -EXDEV;
386 :
387 11167872 : ret = generic_file_rw_checks(file_in, file_out);
388 11167696 : if (ret < 0)
389 : return ret;
390 :
391 11167641 : if (!file_in->f_op->remap_file_range)
392 : return -EOPNOTSUPP;
393 :
394 11100099 : ret = remap_verify_area(file_in, pos_in, len, false);
395 11100099 : if (ret)
396 : return ret;
397 :
398 11100088 : ret = remap_verify_area(file_out, pos_out, len, true);
399 11100088 : if (ret)
400 : return ret;
401 :
402 11100077 : ret = file_in->f_op->remap_file_range(file_in, pos_in,
403 : file_out, pos_out, len, remap_flags);
404 11100492 : if (ret < 0)
405 : return ret;
406 :
407 10745979 : fsnotify_access(file_in);
408 10745879 : fsnotify_modify(file_out);
409 10745879 : return ret;
410 : }
411 : EXPORT_SYMBOL(do_clone_file_range);
412 :
413 11168000 : loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
414 : struct file *file_out, loff_t pos_out,
415 : loff_t len, unsigned int remap_flags)
416 : {
417 11168000 : loff_t ret;
418 :
419 11168000 : file_start_write(file_out);
420 11167918 : ret = do_clone_file_range(file_in, pos_in, file_out, pos_out, len,
421 : remap_flags);
422 11168178 : file_end_write(file_out);
423 :
424 11168111 : return ret;
425 : }
426 : EXPORT_SYMBOL(vfs_clone_file_range);
427 :
428 : /* Check whether we are allowed to dedupe the destination file */
429 429880521 : static bool allow_file_dedupe(struct file *file)
430 : {
431 429880521 : struct mnt_idmap *idmap = file_mnt_idmap(file);
432 429877959 : struct inode *inode = file_inode(file);
433 :
434 429877959 : if (capable(CAP_SYS_ADMIN))
435 : return true;
436 3454898 : if (file->f_mode & FMODE_WRITE)
437 : return true;
438 0 : if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), current_fsuid()))
439 : return true;
440 0 : if (!inode_permission(idmap, inode, MAY_WRITE))
441 0 : return true;
442 : return false;
443 : }
444 :
445 429883788 : loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
446 : struct file *dst_file, loff_t dst_pos,
447 : loff_t len, unsigned int remap_flags)
448 : {
449 429883788 : loff_t ret;
450 :
451 429883788 : WARN_ON_ONCE(remap_flags & ~(REMAP_FILE_DEDUP |
452 : REMAP_FILE_CAN_SHORTEN));
453 :
454 429883788 : ret = mnt_want_write_file(dst_file);
455 429887823 : if (ret)
456 : return ret;
457 :
458 : /*
459 : * This is redundant if called from vfs_dedupe_file_range(), but other
460 : * callers need it and it's not performance sesitive...
461 : */
462 429887823 : ret = remap_verify_area(src_file, src_pos, len, false);
463 429887823 : if (ret)
464 0 : goto out_drop_write;
465 :
466 429887823 : ret = remap_verify_area(dst_file, dst_pos, len, true);
467 429887823 : if (ret)
468 0 : goto out_drop_write;
469 :
470 429887823 : ret = -EPERM;
471 429887823 : if (!allow_file_dedupe(dst_file))
472 0 : goto out_drop_write;
473 :
474 429870689 : ret = -EXDEV;
475 429870689 : if (file_inode(src_file)->i_sb != file_inode(dst_file)->i_sb)
476 11 : goto out_drop_write;
477 :
478 429870678 : ret = -EISDIR;
479 429870678 : if (S_ISDIR(file_inode(dst_file)->i_mode))
480 0 : goto out_drop_write;
481 :
482 429870678 : ret = -EINVAL;
483 429870678 : if (!dst_file->f_op->remap_file_range)
484 22 : goto out_drop_write;
485 :
486 429870656 : if (len == 0) {
487 49631860 : ret = 0;
488 49631860 : goto out_drop_write;
489 : }
490 :
491 380238796 : ret = dst_file->f_op->remap_file_range(src_file, src_pos, dst_file,
492 : dst_pos, len, remap_flags | REMAP_FILE_DEDUP);
493 429889865 : out_drop_write:
494 429889865 : mnt_drop_write_file(dst_file);
495 :
496 429889865 : return ret;
497 : }
498 : EXPORT_SYMBOL(vfs_dedupe_file_range_one);
499 :
500 10273071 : int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same)
501 : {
502 10273071 : struct file_dedupe_range_info *info;
503 10273071 : struct inode *src = file_inode(file);
504 10273071 : u64 off;
505 10273071 : u64 len;
506 10273071 : int i;
507 10273071 : int ret;
508 10273071 : u16 count = same->dest_count;
509 10273071 : loff_t deduped;
510 :
511 10273071 : if (!(file->f_mode & FMODE_READ))
512 : return -EINVAL;
513 :
514 10273071 : if (same->reserved1 || same->reserved2)
515 : return -EINVAL;
516 :
517 10273071 : off = same->src_offset;
518 10273071 : len = same->src_length;
519 :
520 10273071 : if (S_ISDIR(src->i_mode))
521 : return -EISDIR;
522 :
523 10273060 : if (!S_ISREG(src->i_mode))
524 : return -EINVAL;
525 :
526 10273049 : if (!file->f_op->remap_file_range)
527 : return -EOPNOTSUPP;
528 :
529 10205872 : ret = remap_verify_area(file, off, len, false);
530 10205839 : if (ret < 0)
531 : return ret;
532 10205839 : ret = 0;
533 :
534 10205839 : if (off + len > i_size_read(src))
535 : return -EINVAL;
536 :
537 : /* Arbitrary 1G limit on a single dedupe request, can be raised. */
538 10205817 : len = min_t(u64, len, 1 << 30);
539 :
540 : /* pre-format output fields to sane values */
541 440182337 : for (i = 0; i < count; i++) {
542 429976520 : same->info[i].bytes_deduped = 0ULL;
543 429976520 : same->info[i].status = FILE_DEDUPE_RANGE_SAME;
544 : }
545 :
546 440003686 : for (i = 0, info = same->info; i < count; i++, info++) {
547 429799138 : struct fd dst_fd = fdget(info->dest_fd);
548 429894124 : struct file *dst_file = dst_fd.file;
549 :
550 429894124 : if (!dst_file) {
551 0 : info->status = -EBADF;
552 0 : goto next_loop;
553 : }
554 :
555 429894124 : if (info->reserved) {
556 0 : info->status = -EINVAL;
557 0 : goto next_fdput;
558 : }
559 :
560 429894124 : deduped = vfs_dedupe_file_range_one(file, off, dst_file,
561 429894124 : info->dest_offset, len,
562 : REMAP_FILE_CAN_SHORTEN);
563 429886760 : if (deduped == -EBADE)
564 77416352 : info->status = FILE_DEDUPE_RANGE_DIFFERS;
565 352470408 : else if (deduped < 0)
566 53659988 : info->status = deduped;
567 : else
568 298810420 : info->bytes_deduped = len;
569 :
570 429886760 : next_fdput:
571 429886760 : fdput(dst_fd);
572 429886766 : next_loop:
573 429886766 : if (fatal_signal_pending(current))
574 : break;
575 : }
576 : return ret;
577 : }
578 : EXPORT_SYMBOL(vfs_dedupe_file_range);
|
