Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * linux/fs/file_table.c
4 : *
5 : * Copyright (C) 1991, 1992 Linus Torvalds
6 : * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
7 : */
8 :
9 : #include <linux/string.h>
10 : #include <linux/slab.h>
11 : #include <linux/file.h>
12 : #include <linux/fdtable.h>
13 : #include <linux/init.h>
14 : #include <linux/module.h>
15 : #include <linux/fs.h>
16 : #include <linux/filelock.h>
17 : #include <linux/security.h>
18 : #include <linux/cred.h>
19 : #include <linux/eventpoll.h>
20 : #include <linux/rcupdate.h>
21 : #include <linux/mount.h>
22 : #include <linux/capability.h>
23 : #include <linux/cdev.h>
24 : #include <linux/fsnotify.h>
25 : #include <linux/sysctl.h>
26 : #include <linux/percpu_counter.h>
27 : #include <linux/percpu.h>
28 : #include <linux/task_work.h>
29 : #include <linux/ima.h>
30 : #include <linux/swap.h>
31 : #include <linux/kmemleak.h>
32 :
33 : #include <linux/atomic.h>
34 :
35 : #include "internal.h"
36 :
37 : /* sysctl tunables... */
38 : static struct files_stat_struct files_stat = {
39 : .max_files = NR_FILE
40 : };
41 :
42 : /* SLAB cache for file structures */
43 : static struct kmem_cache *filp_cachep __read_mostly;
44 :
45 : static struct percpu_counter nr_files __cacheline_aligned_in_smp;
46 :
47 : /* Container for backing file with optional real path */
48 : struct backing_file {
49 : struct file file;
50 : struct path real_path;
51 : };
52 :
53 : static inline struct backing_file *backing_file(struct file *f)
54 : {
55 : return container_of(f, struct backing_file, file);
56 : }
57 :
58 42027283 : struct path *backing_file_real_path(struct file *f)
59 : {
60 18233336 : return &backing_file(f)->real_path;
61 : }
62 : EXPORT_SYMBOL_GPL(backing_file_real_path);
63 :
64 909741091 : static void file_free_rcu(struct rcu_head *head)
65 : {
66 909741091 : struct file *f = container_of(head, struct file, f_rcuhead);
67 :
68 909741091 : put_cred(f->f_cred);
69 916311776 : if (unlikely(f->f_mode & FMODE_BACKING))
70 18238334 : kfree(backing_file(f));
71 : else
72 898073442 : kmem_cache_free(filp_cachep, f);
73 915119592 : }
74 :
75 923434855 : static inline void file_free(struct file *f)
76 : {
77 923434855 : security_file_free(f);
78 923434855 : if (unlikely(f->f_mode & FMODE_BACKING))
79 18240666 : path_put(backing_file_real_path(f));
80 923435086 : if (likely(!(f->f_mode & FMODE_NOACCOUNT)))
81 905176981 : percpu_counter_dec(&nr_files);
82 923506181 : call_rcu(&f->f_rcuhead, file_free_rcu);
83 923467405 : }
84 :
85 : /*
86 : * Return the total number of open files in the system
87 : */
88 : static long get_nr_files(void)
89 : {
90 904893224 : return percpu_counter_read_positive(&nr_files);
91 : }
92 :
93 : /*
94 : * Return the maximum number of open files in the system
95 : */
96 17961608 : unsigned long get_max_files(void)
97 : {
98 17961608 : return files_stat.max_files;
99 : }
100 : EXPORT_SYMBOL_GPL(get_max_files);
101 :
102 : #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
103 :
104 : /*
105 : * Handle nr_files sysctl
106 : */
107 379 : static int proc_nr_files(struct ctl_table *table, int write, void *buffer,
108 : size_t *lenp, loff_t *ppos)
109 : {
110 379 : files_stat.nr_files = get_nr_files();
111 379 : return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
112 : }
113 :
114 : static struct ctl_table fs_stat_sysctls[] = {
115 : {
116 : .procname = "file-nr",
117 : .data = &files_stat,
118 : .maxlen = sizeof(files_stat),
119 : .mode = 0444,
120 : .proc_handler = proc_nr_files,
121 : },
122 : {
123 : .procname = "file-max",
124 : .data = &files_stat.max_files,
125 : .maxlen = sizeof(files_stat.max_files),
126 : .mode = 0644,
127 : .proc_handler = proc_doulongvec_minmax,
128 : .extra1 = SYSCTL_LONG_ZERO,
129 : .extra2 = SYSCTL_LONG_MAX,
130 : },
131 : {
132 : .procname = "nr_open",
133 : .data = &sysctl_nr_open,
134 : .maxlen = sizeof(unsigned int),
135 : .mode = 0644,
136 : .proc_handler = proc_dointvec_minmax,
137 : .extra1 = &sysctl_nr_open_min,
138 : .extra2 = &sysctl_nr_open_max,
139 : },
140 : { }
141 : };
142 :
143 0 : static int __init init_fs_stat_sysctls(void)
144 : {
145 0 : register_sysctl_init("fs", fs_stat_sysctls);
146 0 : if (IS_ENABLED(CONFIG_BINFMT_MISC)) {
147 0 : struct ctl_table_header *hdr;
148 0 : hdr = register_sysctl_mount_point("fs/binfmt_misc");
149 0 : kmemleak_not_leak(hdr);
150 : }
151 0 : return 0;
152 : }
153 : fs_initcall(init_fs_stat_sysctls);
154 : #endif
155 :
156 923294588 : static int init_file(struct file *f, int flags, const struct cred *cred)
157 : {
158 923294588 : int error;
159 :
160 923294588 : f->f_cred = get_cred(cred);
161 923292084 : error = security_file_alloc(f);
162 923292084 : if (unlikely(error)) {
163 : put_cred(f->f_cred);
164 : return error;
165 : }
166 :
167 923292084 : atomic_long_set(&f->f_count, 1);
168 923292084 : rwlock_init(&f->f_owner.lock);
169 923229493 : spin_lock_init(&f->f_lock);
170 923189788 : mutex_init(&f->f_pos_lock);
171 923204223 : f->f_flags = flags;
172 923204223 : f->f_mode = OPEN_FMODE(flags);
173 : /* f->f_version: 0 */
174 :
175 923204223 : return 0;
176 : }
177 :
178 : /* Find an unused file structure and return a pointer to it.
179 : * Returns an error pointer if some error happend e.g. we over file
180 : * structures limit, run out of memory or operation is not permitted.
181 : *
182 : * Be very careful using this. You are responsible for
183 : * getting write access to any mount that you might assign
184 : * to this filp, if it is opened for write. If this is not
185 : * done, you will imbalance int the mount's writer count
186 : * and a warning at __fput() time.
187 : */
188 904892845 : struct file *alloc_empty_file(int flags, const struct cred *cred)
189 : {
190 904892845 : static long old_max;
191 904892845 : struct file *f;
192 904892845 : int error;
193 :
194 : /*
195 : * Privileged users can go above max_files
196 : */
197 904892845 : if (get_nr_files() >= files_stat.max_files && !capable(CAP_SYS_ADMIN)) {
198 : /*
199 : * percpu_counters are inaccurate. Do an expensive check before
200 : * we go and fail.
201 : */
202 0 : if (percpu_counter_sum_positive(&nr_files) >= files_stat.max_files)
203 0 : goto over;
204 : }
205 :
206 904892845 : f = kmem_cache_zalloc(filp_cachep, GFP_KERNEL);
207 905032994 : if (unlikely(!f))
208 : return ERR_PTR(-ENOMEM);
209 :
210 905032994 : error = init_file(f, flags, cred);
211 904954185 : if (unlikely(error)) {
212 0 : kmem_cache_free(filp_cachep, f);
213 0 : return ERR_PTR(error);
214 : }
215 :
216 904954185 : percpu_counter_inc(&nr_files);
217 :
218 904954185 : return f;
219 :
220 : over:
221 : /* Ran out of filps - report that */
222 0 : if (get_nr_files() > old_max) {
223 0 : pr_info("VFS: file-max limit %lu reached\n", get_max_files());
224 0 : old_max = get_nr_files();
225 : }
226 : return ERR_PTR(-ENFILE);
227 : }
228 :
229 : /*
230 : * Variant of alloc_empty_file() that doesn't check and modify nr_files.
231 : *
232 : * This is only for kernel internal use, and the allocate file must not be
233 : * installed into file tables or such.
234 : */
235 17063 : struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred)
236 : {
237 17063 : struct file *f;
238 17063 : int error;
239 :
240 17063 : f = kmem_cache_zalloc(filp_cachep, GFP_KERNEL);
241 17066 : if (unlikely(!f))
242 : return ERR_PTR(-ENOMEM);
243 :
244 17066 : error = init_file(f, flags, cred);
245 17064 : if (unlikely(error)) {
246 0 : kmem_cache_free(filp_cachep, f);
247 0 : return ERR_PTR(error);
248 : }
249 :
250 17064 : f->f_mode |= FMODE_NOACCOUNT;
251 :
252 17064 : return f;
253 : }
254 :
255 : /*
256 : * Variant of alloc_empty_file() that allocates a backing_file container
257 : * and doesn't check and modify nr_files.
258 : *
259 : * This is only for kernel internal use, and the allocate file must not be
260 : * installed into file tables or such.
261 : */
262 18199702 : struct file *alloc_empty_backing_file(int flags, const struct cred *cred)
263 : {
264 18199702 : struct backing_file *ff;
265 18199702 : int error;
266 :
267 18199702 : ff = kzalloc(sizeof(struct backing_file), GFP_KERNEL);
268 18168259 : if (unlikely(!ff))
269 : return ERR_PTR(-ENOMEM);
270 :
271 18168259 : error = init_file(&ff->file, flags, cred);
272 18195859 : if (unlikely(error)) {
273 0 : kfree(ff);
274 0 : return ERR_PTR(error);
275 : }
276 :
277 18195859 : ff->file.f_mode |= FMODE_BACKING | FMODE_NOACCOUNT;
278 18195859 : return &ff->file;
279 : }
280 :
281 : /**
282 : * alloc_file - allocate and initialize a 'struct file'
283 : *
284 : * @path: the (dentry, vfsmount) pair for the new file
285 : * @flags: O_... flags with which the new file will be opened
286 : * @fop: the 'struct file_operations' for the new file
287 : */
288 284541751 : static struct file *alloc_file(const struct path *path, int flags,
289 : const struct file_operations *fop)
290 : {
291 284541751 : struct file *file;
292 :
293 284541751 : file = alloc_empty_file(flags, current_cred());
294 284560153 : if (IS_ERR(file))
295 : return file;
296 :
297 284560153 : file->f_path = *path;
298 284560153 : file->f_inode = path->dentry->d_inode;
299 284560153 : file->f_mapping = path->dentry->d_inode->i_mapping;
300 284560153 : file->f_wb_err = filemap_sample_wb_err(file->f_mapping);
301 284567973 : file->f_sb_err = file_sample_sb_err(file);
302 284565940 : if (fop->llseek)
303 256993272 : file->f_mode |= FMODE_LSEEK;
304 284565940 : if ((file->f_mode & FMODE_READ) &&
305 275599827 : likely(fop->read || fop->read_iter))
306 275070408 : file->f_mode |= FMODE_CAN_READ;
307 284565940 : if ((file->f_mode & FMODE_WRITE) &&
308 275600009 : likely(fop->write || fop->write_iter))
309 274971435 : file->f_mode |= FMODE_CAN_WRITE;
310 284565940 : file->f_iocb_flags = iocb_flags(file);
311 284565940 : file->f_mode |= FMODE_OPENED;
312 284565940 : file->f_op = fop;
313 284565940 : if ((file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ)
314 8981152 : i_readcount_inc(path->dentry->d_inode);
315 : return file;
316 : }
317 :
318 275565119 : struct file *alloc_file_pseudo(struct inode *inode, struct vfsmount *mnt,
319 : const char *name, int flags,
320 : const struct file_operations *fops)
321 : {
322 275565119 : static const struct dentry_operations anon_ops = {
323 : .d_dname = simple_dname
324 : };
325 275565119 : struct qstr this = QSTR_INIT(name, strlen(name));
326 275565119 : struct path path;
327 275565119 : struct file *file;
328 :
329 275565119 : path.dentry = d_alloc_pseudo(mnt->mnt_sb, &this);
330 275516778 : if (!path.dentry)
331 : return ERR_PTR(-ENOMEM);
332 275516778 : if (!mnt->mnt_sb->s_d_op)
333 257157942 : d_set_d_op(path.dentry, &anon_ops);
334 275538280 : path.mnt = mntget(mnt);
335 275552061 : d_instantiate(path.dentry, inode);
336 275557684 : file = alloc_file(&path, flags, fops);
337 275554434 : if (IS_ERR(file)) {
338 0 : ihold(inode);
339 0 : path_put(&path);
340 : }
341 : return file;
342 : }
343 : EXPORT_SYMBOL(alloc_file_pseudo);
344 :
345 8979514 : struct file *alloc_file_clone(struct file *base, int flags,
346 : const struct file_operations *fops)
347 : {
348 8979514 : struct file *f = alloc_file(&base->f_path, flags, fops);
349 8978940 : if (!IS_ERR(f)) {
350 8978940 : path_get(&f->f_path);
351 8978879 : f->f_mapping = base->f_mapping;
352 : }
353 8978879 : return f;
354 : }
355 :
356 : /* the real guts of fput() - releasing the last reference to file
357 : */
358 923382725 : static void __fput(struct file *file)
359 : {
360 923382725 : struct dentry *dentry = file->f_path.dentry;
361 923382725 : struct vfsmount *mnt = file->f_path.mnt;
362 923382725 : struct inode *inode = file->f_inode;
363 923382725 : fmode_t mode = file->f_mode;
364 :
365 923382725 : if (unlikely(!(file->f_mode & FMODE_OPENED)))
366 51719261 : goto out;
367 :
368 871663464 : might_sleep();
369 :
370 871662529 : fsnotify_close(file);
371 : /*
372 : * The function eventpoll_release() should be the first called
373 : * in the file cleanup chain.
374 : */
375 871458128 : eventpoll_release(file);
376 871458500 : locks_remove_file(file);
377 :
378 871576878 : ima_file_free(file);
379 871576878 : if (unlikely(file->f_flags & FASYNC)) {
380 0 : if (file->f_op->fasync)
381 0 : file->f_op->fasync(-1, file, 0);
382 : }
383 871576878 : if (file->f_op->release)
384 551721275 : file->f_op->release(inode, file);
385 871631444 : if (unlikely(S_ISCHR(inode->i_mode) && inode->i_cdev != NULL &&
386 : !(mode & FMODE_PATH))) {
387 3468957 : cdev_put(inode->i_cdev);
388 : }
389 871629182 : fops_put(file->f_op);
390 871642177 : put_pid(file->f_owner.pid);
391 871618372 : put_file_access(file);
392 871613702 : dput(dentry);
393 871761594 : if (unlikely(mode & FMODE_NEED_UNMOUNT))
394 20248 : dissolve_on_fput(mnt);
395 871761594 : mntput(mnt);
396 923467321 : out:
397 923467321 : file_free(file);
398 923485264 : }
399 :
400 : static LLIST_HEAD(delayed_fput_list);
401 45343 : static void delayed_fput(struct work_struct *unused)
402 : {
403 45343 : struct llist_node *node = llist_del_all(&delayed_fput_list);
404 45343 : struct file *f, *t;
405 :
406 475447 : llist_for_each_entry_safe(f, t, node, f_llist)
407 430104 : __fput(f);
408 45343 : }
409 :
410 922906958 : static void ____fput(struct callback_head *work)
411 : {
412 922906958 : __fput(container_of(work, struct file, f_rcuhead));
413 923031989 : }
414 :
415 : /*
416 : * If kernel thread really needs to have the final fput() it has done
417 : * to complete, call this. The only user right now is the boot - we
418 : * *do* need to make sure our writes to binaries on initramfs has
419 : * not left us with opened struct file waiting for __fput() - execve()
420 : * won't work without that. Please, don't add more callers without
421 : * very good reasons; in particular, never call that with locks
422 : * held and never call that from a thread that might need to do
423 : * some work on any kind of umount.
424 : */
425 0 : void flush_delayed_fput(void)
426 : {
427 0 : delayed_fput(NULL);
428 0 : }
429 : EXPORT_SYMBOL_GPL(flush_delayed_fput);
430 :
431 : static DECLARE_DELAYED_WORK(delayed_fput_work, delayed_fput);
432 :
433 3316655014 : void fput(struct file *file)
434 : {
435 6633984454 : if (atomic_long_dec_and_test(&file->f_count)) {
436 923382399 : struct task_struct *task = current;
437 :
438 923382399 : if (likely(!in_interrupt() && !(task->flags & PF_KTHREAD))) {
439 922952331 : init_task_work(&file->f_rcuhead, ____fput);
440 922952331 : if (!task_work_add(task, &file->f_rcuhead, TWA_RESUME))
441 : return;
442 : /*
443 : * After this task has run exit_task_work(),
444 : * task_work_add() will fail. Fall through to delayed
445 : * fput to avoid leaking *file.
446 : */
447 : }
448 :
449 436680 : if (llist_add(&file->f_llist, &delayed_fput_list))
450 45343 : schedule_delayed_work(&delayed_fput_work, 1);
451 : }
452 : }
453 :
454 : /*
455 : * synchronous analog of fput(); for kernel threads that might be needed
456 : * in some umount() (and thus can't use flush_delayed_fput() without
457 : * risking deadlocks), need to wait for completion of __fput() and know
458 : * for this specific struct file it won't involve anything that would
459 : * need them. Use only if you really need it - at the very least,
460 : * don't blindly convert fput() by kernel thread to that.
461 : */
462 0 : void __fput_sync(struct file *file)
463 : {
464 0 : if (atomic_long_dec_and_test(&file->f_count)) {
465 0 : struct task_struct *task = current;
466 0 : BUG_ON(!(task->flags & PF_KTHREAD));
467 0 : __fput(file);
468 : }
469 0 : }
470 :
471 : EXPORT_SYMBOL(fput);
472 : EXPORT_SYMBOL(__fput_sync);
473 :
474 0 : void __init files_init(void)
475 : {
476 0 : filp_cachep = kmem_cache_create("filp", sizeof(struct file), 0,
477 : SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT, NULL);
478 0 : percpu_counter_init(&nr_files, 0, GFP_KERNEL);
479 0 : }
480 :
481 : /*
482 : * One file with associated inode and dcache is very roughly 1K. Per default
483 : * do not use more than 10% of our memory for files.
484 : */
485 0 : void __init files_maxfiles_init(void)
486 : {
487 0 : unsigned long n;
488 0 : unsigned long nr_pages = totalram_pages();
489 0 : unsigned long memreserve = (nr_pages - nr_free_pages()) * 3/2;
490 :
491 0 : memreserve = min(memreserve, nr_pages - 1);
492 0 : n = ((nr_pages - memreserve) * (PAGE_SIZE / 1024)) / 10;
493 :
494 0 : files_stat.max_files = max_t(unsigned long, n, NR_FILE);
495 0 : }
|
