Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : #include <linux/slab.h>
3 : #include <linux/stat.h>
4 : #include <linux/sched/xacct.h>
5 : #include <linux/fcntl.h>
6 : #include <linux/file.h>
7 : #include <linux/uio.h>
8 : #include <linux/fsnotify.h>
9 : #include <linux/security.h>
10 : #include <linux/export.h>
11 : #include <linux/syscalls.h>
12 : #include <linux/pagemap.h>
13 : #include <linux/splice.h>
14 : #include <linux/compat.h>
15 : #include <linux/mount.h>
16 : #include <linux/fs.h>
17 : #include <linux/dax.h>
18 : #include <linux/overflow.h>
19 : #include "internal.h"
20 :
21 : #include <linux/uaccess.h>
22 : #include <asm/unistd.h>
23 :
24 : /*
25 : * Performs necessary checks before doing a clone.
26 : *
27 : * Can adjust amount of bytes to clone via @req_count argument.
28 : * Returns appropriate error code that caller should return or
29 : * zero in case the clone should be allowed.
30 : */
31 192025341 : static int generic_remap_checks(struct file *file_in, loff_t pos_in,
32 : struct file *file_out, loff_t pos_out,
33 : loff_t *req_count, unsigned int remap_flags)
34 : {
35 192025341 : struct inode *inode_in = file_in->f_mapping->host;
36 192025341 : struct inode *inode_out = file_out->f_mapping->host;
37 192025341 : uint64_t count = *req_count;
38 192025341 : uint64_t bcount;
39 192025341 : loff_t size_in, size_out;
40 192025341 : loff_t bs = inode_out->i_sb->s_blocksize;
41 192025341 : int ret;
42 :
43 : /* The start of both ranges must be aligned to an fs block. */
44 192025341 : if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_out, bs))
45 : return -EINVAL;
46 :
47 : /* Ensure offsets don't wrap. */
48 187473073 : if (pos_in + count < pos_in || pos_out + count < pos_out)
49 : return -EINVAL;
50 :
51 187473073 : size_in = i_size_read(inode_in);
52 187473073 : size_out = i_size_read(inode_out);
53 :
54 : /* Dedupe requires both ranges to be within EOF. */
55 187473073 : if ((remap_flags & REMAP_FILE_DEDUP) &&
56 181443934 : (pos_in >= size_in || pos_in + count > size_in ||
57 156484684 : pos_out >= size_out || pos_out + count > size_out))
58 : return -EINVAL;
59 :
60 : /* Ensure the infile range is within the infile. */
61 161909225 : if (pos_in >= size_in)
62 : return -EINVAL;
63 161909218 : count = min(count, size_in - (uint64_t)pos_in);
64 :
65 161909218 : ret = generic_write_check_limits(file_out, pos_out, &count);
66 161910313 : if (ret)
67 : return ret;
68 :
69 : /*
70 : * If the user wanted us to link to the infile's EOF, round up to the
71 : * next block boundary for this check.
72 : *
73 : * Otherwise, make sure the count is also block-aligned, having
74 : * already confirmed the starting offsets' block alignment.
75 : */
76 161910313 : if (pos_in + count == size_in &&
77 161550 : (!(remap_flags & REMAP_FILE_DEDUP) || pos_out + count == size_out)) {
78 273381 : bcount = ALIGN(size_in, bs) - pos_in;
79 : } else {
80 161636932 : if (!IS_ALIGNED(count, bs))
81 6471067 : count = ALIGN_DOWN(count, bs);
82 161636932 : bcount = count;
83 : }
84 :
85 : /* Don't allow overlapped cloning within the same file. */
86 161910313 : if (inode_in == inode_out &&
87 4327953 : pos_out + bcount > pos_in &&
88 2357100 : pos_out < pos_in + bcount)
89 : return -EINVAL;
90 :
91 : /*
92 : * We shortened the request but the caller can't deal with that, so
93 : * bounce the request back to userspace.
94 : */
95 161900388 : if (*req_count != count && !(remap_flags & REMAP_FILE_CAN_SHORTEN))
96 : return -EINVAL;
97 :
98 161900374 : *req_count = count;
99 161900374 : return 0;
100 : }
101 :
102 2485680 : int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write)
103 : {
104 662453438 : loff_t tmp;
105 :
106 2485680 : if (unlikely(pos < 0 || len < 0))
107 : return -EINVAL;
108 :
109 662453438 : if (unlikely(check_add_overflow(pos, len, &tmp)))
110 14 : return -EINVAL;
111 :
112 : return security_file_permission(file, write ? MAY_WRITE : MAY_READ);
113 : }
114 : EXPORT_SYMBOL(remap_verify_area);
115 :
116 : /*
117 : * Ensure that we don't remap a partial EOF block in the middle of something
118 : * else. Assume that the offsets have already been checked for block
119 : * alignment.
120 : *
121 : * For clone we only link a partial EOF block above or at the destination file's
122 : * EOF. For deduplication we accept a partial EOF block only if it ends at the
123 : * destination file's EOF (can not link it into the middle of a file).
124 : *
125 : * Shorten the request if possible.
126 : */
127 118174179 : static int generic_remap_check_len(struct inode *inode_in,
128 : struct inode *inode_out,
129 : loff_t pos_out,
130 : loff_t *len,
131 : unsigned int remap_flags)
132 : {
133 118174179 : u64 blkmask = i_blocksize(inode_in) - 1;
134 118173324 : loff_t new_len = *len;
135 :
136 118173324 : if ((*len & blkmask) == 0)
137 : return 0;
138 :
139 18280 : if (pos_out + *len < i_size_read(inode_out))
140 4474 : new_len &= ~blkmask;
141 :
142 18280 : if (new_len == *len)
143 : return 0;
144 :
145 4474 : if (remap_flags & REMAP_FILE_CAN_SHORTEN) {
146 0 : *len = new_len;
147 0 : return 0;
148 : }
149 :
150 4474 : return (remap_flags & REMAP_FILE_DEDUP) ? -EBADE : -EINVAL;
151 : }
152 :
153 : /* Read a page's worth of file data into the page cache. */
154 3859806596 : static struct folio *vfs_dedupe_get_folio(struct file *file, loff_t pos)
155 : {
156 3859806596 : return read_mapping_folio(file->f_mapping, pos >> PAGE_SHIFT, file);
157 : }
158 :
159 : /*
160 : * Lock two folios, ensuring that we lock in offset order if the folios
161 : * are from the same file.
162 : */
163 1930588719 : static void vfs_lock_two_folios(struct folio *folio1, struct folio *folio2)
164 : {
165 : /* Always lock in order of increasing index. */
166 1930588719 : if (folio1->index > folio2->index)
167 963464836 : swap(folio1, folio2);
168 :
169 1930588719 : folio_lock(folio1);
170 1930623153 : if (folio1 != folio2)
171 1930622321 : folio_lock(folio2);
172 1930410790 : }
173 :
174 : /* Unlock two folios, being careful not to unlock the same folio twice. */
175 1930753752 : static void vfs_unlock_two_folios(struct folio *folio1, struct folio *folio2)
176 : {
177 1930753752 : folio_unlock(folio1);
178 1931083334 : if (folio1 != folio2)
179 1931082502 : folio_unlock(folio2);
180 1930750554 : }
181 :
182 : /*
183 : * Compare extents of two files to see if they are the same.
184 : * Caller must have locked both inodes to prevent write races.
185 : */
186 150984309 : static int vfs_dedupe_file_range_compare(struct file *src, loff_t srcoff,
187 : struct file *dest, loff_t dstoff,
188 : loff_t len, bool *is_same)
189 : {
190 150984309 : bool same = true;
191 150984309 : int error = -EINVAL;
192 :
193 2043082142 : while (len) {
194 1930936733 : struct folio *src_folio, *dst_folio;
195 1930936733 : void *src_addr, *dst_addr;
196 1930936733 : loff_t cmp_len = min(PAGE_SIZE - offset_in_page(srcoff),
197 : PAGE_SIZE - offset_in_page(dstoff));
198 :
199 1930936733 : cmp_len = min(cmp_len, len);
200 1930936733 : if (cmp_len <= 0)
201 0 : goto out_error;
202 :
203 1930936733 : src_folio = vfs_dedupe_get_folio(src, srcoff);
204 1930593743 : if (IS_ERR(src_folio)) {
205 874 : error = PTR_ERR(src_folio);
206 874 : goto out_error;
207 : }
208 1930592869 : dst_folio = vfs_dedupe_get_folio(dest, dstoff);
209 1930692917 : if (IS_ERR(dst_folio)) {
210 1582 : error = PTR_ERR(dst_folio);
211 1582 : folio_put(src_folio);
212 1582 : goto out_error;
213 : }
214 :
215 1930691335 : vfs_lock_two_folios(src_folio, dst_folio);
216 :
217 : /*
218 : * Now that we've locked both folios, make sure they're still
219 : * mapped to the file data we're interested in. If not,
220 : * someone is invalidating pages on us and we lose.
221 : */
222 5790881235 : if (!folio_test_uptodate(src_folio) || !folio_test_uptodate(dst_folio) ||
223 1930246065 : src_folio->mapping != src->f_mapping ||
224 1930246065 : dst_folio->mapping != dest->f_mapping) {
225 0 : same = false;
226 0 : goto unlock;
227 : }
228 :
229 3860237778 : src_addr = kmap_local_folio(src_folio,
230 1930246065 : offset_in_folio(src_folio, srcoff));
231 3860064653 : dst_addr = kmap_local_folio(dst_folio,
232 1929991713 : offset_in_folio(dst_folio, dstoff));
233 :
234 1930072940 : flush_dcache_folio(src_folio);
235 1930072940 : flush_dcache_folio(dst_folio);
236 :
237 3860145880 : if (memcmp(src_addr, dst_addr, cmp_len))
238 38847765 : same = false;
239 :
240 : kunmap_local(dst_addr);
241 1930072940 : kunmap_local(src_addr);
242 1930072940 : unlock:
243 1930072940 : vfs_unlock_two_folios(src_folio, dst_folio);
244 1930696676 : folio_put(dst_folio);
245 1930753078 : folio_put(src_folio);
246 :
247 1930945549 : if (!same)
248 : break;
249 :
250 1892097833 : srcoff += cmp_len;
251 1892097833 : dstoff += cmp_len;
252 1892097833 : len -= cmp_len;
253 : }
254 :
255 150993125 : *is_same = same;
256 150993125 : return 0;
257 :
258 : out_error:
259 : return error;
260 : }
261 :
262 : /*
263 : * Check that the two inodes are eligible for cloning, the ranges make
264 : * sense, and then flush all dirty data. Caller must ensure that the
265 : * inodes have been locked against any other modifications.
266 : *
267 : * If there's an error, then the usual negative error code is returned.
268 : * Otherwise returns 0 with *len set to the request length.
269 : */
270 : int
271 192606560 : __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
272 : struct file *file_out, loff_t pos_out,
273 : loff_t *len, unsigned int remap_flags,
274 : const struct iomap_ops *dax_read_ops)
275 : {
276 192606560 : struct inode *inode_in = file_inode(file_in);
277 192606560 : struct inode *inode_out = file_inode(file_out);
278 192606560 : bool same_inode = (inode_in == inode_out);
279 192606560 : int ret;
280 :
281 : /* Don't touch certain kinds of inodes */
282 192606560 : if (IS_IMMUTABLE(inode_out))
283 : return -EPERM;
284 :
285 192606560 : if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
286 : return -ETXTBSY;
287 :
288 : /* Don't reflink dirs, pipes, sockets... */
289 192606539 : if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
290 : return -EISDIR;
291 192606539 : if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
292 : return -EINVAL;
293 :
294 : /* Zero length dedupe exits immediately; reflink goes to EOF. */
295 192606539 : if (*len == 0) {
296 640694 : loff_t isize = i_size_read(inode_in);
297 :
298 640694 : if ((remap_flags & REMAP_FILE_DEDUP) || pos_in == isize)
299 : return 0;
300 52845 : if (pos_in > isize)
301 : return -EINVAL;
302 52845 : *len = isize - pos_in;
303 52845 : if (*len == 0)
304 : return 0;
305 : }
306 :
307 : /* Check that we don't violate system file offset limits. */
308 192018690 : ret = generic_remap_checks(file_in, pos_in, file_out, pos_out, len,
309 : remap_flags);
310 192024533 : if (ret || *len == 0)
311 : return ret;
312 :
313 : /* Wait for the completion of any pending IOs on both files */
314 157011596 : inode_dio_wait(inode_in);
315 157008016 : if (!same_inode)
316 152713928 : inode_dio_wait(inode_out);
317 :
318 157006800 : ret = filemap_write_and_wait_range(inode_in->i_mapping,
319 157006800 : pos_in, pos_in + *len - 1);
320 157007166 : if (ret)
321 : return ret;
322 :
323 157007932 : ret = filemap_write_and_wait_range(inode_out->i_mapping,
324 157007932 : pos_out, pos_out + *len - 1);
325 157016665 : if (ret)
326 : return ret;
327 :
328 : /*
329 : * Check that the extents are the same.
330 : */
331 157015273 : if (remap_flags & REMAP_FILE_DEDUP) {
332 150985502 : bool is_same = false;
333 :
334 150985502 : if (!IS_DAX(inode_in))
335 150985502 : ret = vfs_dedupe_file_range_compare(file_in, pos_in,
336 : file_out, pos_out, *len, &is_same);
337 0 : else if (dax_read_ops)
338 0 : ret = dax_dedupe_file_range_compare(inode_in, pos_in,
339 : inode_out, pos_out, *len, &is_same,
340 : dax_read_ops);
341 : else
342 38850056 : return -EINVAL;
343 150995410 : if (ret)
344 : return ret;
345 150992954 : if (!is_same)
346 : return -EBADE;
347 : }
348 :
349 118175125 : ret = generic_remap_check_len(inode_in, inode_out, pos_out, len,
350 : remap_flags);
351 118173069 : if (ret || *len == 0)
352 : return ret;
353 :
354 : /* If can't alter the file contents, we're done. */
355 118168595 : if (!(remap_flags & REMAP_FILE_DEDUP))
356 6025213 : ret = file_modified(file_out);
357 :
358 : return ret;
359 : }
360 :
361 192614278 : int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
362 : struct file *file_out, loff_t pos_out,
363 : loff_t *len, unsigned int remap_flags)
364 : {
365 192614278 : return __generic_remap_file_range_prep(file_in, pos_in, file_out,
366 : pos_out, len, remap_flags, NULL);
367 : }
368 : EXPORT_SYMBOL(generic_remap_file_range_prep);
369 :
370 8139115 : loff_t do_clone_file_range(struct file *file_in, loff_t pos_in,
371 : struct file *file_out, loff_t pos_out,
372 : loff_t len, unsigned int remap_flags)
373 : {
374 8139115 : loff_t ret;
375 :
376 8139115 : WARN_ON_ONCE(remap_flags & REMAP_FILE_DEDUP);
377 :
378 8139115 : if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
379 : return -EXDEV;
380 :
381 8139069 : ret = generic_file_rw_checks(file_in, file_out);
382 8138886 : if (ret < 0)
383 : return ret;
384 :
385 8138851 : if (!file_in->f_op->remap_file_range)
386 : return -EOPNOTSUPP;
387 :
388 8088169 : ret = remap_verify_area(file_in, pos_in, len, false);
389 8088169 : if (ret)
390 : return ret;
391 :
392 8088162 : ret = remap_verify_area(file_out, pos_out, len, true);
393 8088162 : if (ret)
394 : return ret;
395 :
396 8088155 : ret = file_in->f_op->remap_file_range(file_in, pos_in,
397 : file_out, pos_out, len, remap_flags);
398 8088397 : if (ret < 0)
399 : return ret;
400 :
401 6083036 : fsnotify_access(file_in);
402 6082996 : fsnotify_modify(file_out);
403 6082996 : return ret;
404 : }
405 : EXPORT_SYMBOL(do_clone_file_range);
406 :
407 8139231 : loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
408 : struct file *file_out, loff_t pos_out,
409 : loff_t len, unsigned int remap_flags)
410 : {
411 8139231 : loff_t ret;
412 :
413 8139231 : file_start_write(file_out);
414 8139103 : ret = do_clone_file_range(file_in, pos_in, file_out, pos_out, len,
415 : remap_flags);
416 8139164 : file_end_write(file_out);
417 :
418 8139187 : return ret;
419 : }
420 : EXPORT_SYMBOL(vfs_clone_file_range);
421 :
422 : /* Check whether we are allowed to dedupe the destination file */
423 318105072 : static bool allow_file_dedupe(struct file *file)
424 : {
425 318105072 : struct mnt_idmap *idmap = file_mnt_idmap(file);
426 318104494 : struct inode *inode = file_inode(file);
427 :
428 318104494 : if (capable(CAP_SYS_ADMIN))
429 : return true;
430 230715 : if (file->f_mode & FMODE_WRITE)
431 : return true;
432 0 : if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), current_fsuid()))
433 : return true;
434 0 : if (!inode_permission(idmap, inode, MAY_WRITE))
435 0 : return true;
436 : return false;
437 : }
438 :
439 318104836 : loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
440 : struct file *dst_file, loff_t dst_pos,
441 : loff_t len, unsigned int remap_flags)
442 : {
443 318104836 : loff_t ret;
444 :
445 318104836 : WARN_ON_ONCE(remap_flags & ~(REMAP_FILE_DEDUP |
446 : REMAP_FILE_CAN_SHORTEN));
447 :
448 318104836 : ret = mnt_want_write_file(dst_file);
449 318107165 : if (ret)
450 : return ret;
451 :
452 : /*
453 : * This is redundant if called from vfs_dedupe_file_range(), but other
454 : * callers need it and it's not performance sesitive...
455 : */
456 318107165 : ret = remap_verify_area(src_file, src_pos, len, false);
457 318107165 : if (ret)
458 0 : goto out_drop_write;
459 :
460 318107165 : ret = remap_verify_area(dst_file, dst_pos, len, true);
461 318107165 : if (ret)
462 0 : goto out_drop_write;
463 :
464 318107165 : ret = -EPERM;
465 318107165 : if (!allow_file_dedupe(dst_file))
466 0 : goto out_drop_write;
467 :
468 318106073 : ret = -EXDEV;
469 318106073 : if (file_inode(src_file)->i_sb != file_inode(dst_file)->i_sb)
470 7 : goto out_drop_write;
471 :
472 318106066 : ret = -EISDIR;
473 318106066 : if (S_ISDIR(file_inode(dst_file)->i_mode))
474 0 : goto out_drop_write;
475 :
476 318106066 : ret = -EINVAL;
477 318106066 : if (!dst_file->f_op->remap_file_range)
478 14 : goto out_drop_write;
479 :
480 318106052 : if (len == 0) {
481 40656046 : ret = 0;
482 40656046 : goto out_drop_write;
483 : }
484 :
485 277450006 : ret = dst_file->f_op->remap_file_range(src_file, src_pos, dst_file,
486 : dst_pos, len, remap_flags | REMAP_FILE_DEDUP);
487 318109558 : out_drop_write:
488 318109558 : mnt_drop_write_file(dst_file);
489 :
490 318109558 : return ret;
491 : }
492 : EXPORT_SYMBOL(vfs_dedupe_file_range_one);
493 :
494 7627276 : int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same)
495 : {
496 7627276 : struct file_dedupe_range_info *info;
497 7627276 : struct inode *src = file_inode(file);
498 7627276 : u64 off;
499 7627276 : u64 len;
500 7627276 : int i;
501 7627276 : int ret;
502 7627276 : u16 count = same->dest_count;
503 7627276 : loff_t deduped;
504 :
505 7627276 : if (!(file->f_mode & FMODE_READ))
506 : return -EINVAL;
507 :
508 7627276 : if (same->reserved1 || same->reserved2)
509 : return -EINVAL;
510 :
511 7627276 : off = same->src_offset;
512 7627276 : len = same->src_length;
513 :
514 7627276 : if (S_ISDIR(src->i_mode))
515 : return -EISDIR;
516 :
517 7627269 : if (!S_ISREG(src->i_mode))
518 : return -EINVAL;
519 :
520 7627262 : if (!file->f_op->remap_file_range)
521 : return -EOPNOTSUPP;
522 :
523 7577097 : ret = remap_verify_area(file, off, len, false);
524 7577076 : if (ret < 0)
525 : return ret;
526 7577076 : ret = 0;
527 :
528 7577076 : if (off + len > i_size_read(src))
529 : return -EINVAL;
530 :
531 : /* Arbitrary 1G limit on a single dedupe request, can be raised. */
532 7577062 : len = min_t(u64, len, 1 << 30);
533 :
534 : /* pre-format output fields to sane values */
535 325731123 : for (i = 0; i < count; i++) {
536 318154061 : same->info[i].bytes_deduped = 0ULL;
537 318154061 : same->info[i].status = FILE_DEDUPE_RANGE_SAME;
538 : }
539 :
540 325641946 : for (i = 0, info = same->info; i < count; i++, info++) {
541 318065669 : struct fd dst_fd = fdget(info->dest_fd);
542 318109866 : struct file *dst_file = dst_fd.file;
543 :
544 318109866 : if (!dst_file) {
545 0 : info->status = -EBADF;
546 0 : goto next_loop;
547 : }
548 :
549 318109866 : if (info->reserved) {
550 0 : info->status = -EINVAL;
551 0 : goto next_fdput;
552 : }
553 :
554 318109866 : deduped = vfs_dedupe_file_range_one(file, off, dst_file,
555 318109866 : info->dest_offset, len,
556 : REMAP_FILE_CAN_SHORTEN);
557 318103711 : if (deduped == -EBADE)
558 38847428 : info->status = FILE_DEDUPE_RANGE_DIFFERS;
559 279256283 : else if (deduped < 0)
560 122400381 : info->status = deduped;
561 : else
562 156855902 : info->bytes_deduped = len;
563 :
564 318103711 : next_fdput:
565 318103711 : fdput(dst_fd);
566 318103720 : next_loop:
567 318103720 : if (fatal_signal_pending(current))
568 : break;
569 : }
570 : return ret;
571 : }
572 : EXPORT_SYMBOL(vfs_dedupe_file_range);
|
