Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-or-later
2 : /*
3 : * Copyright (C) 2019 Oracle. All Rights Reserved.
4 : * Author: Darrick J. Wong <darrick.wong@oracle.com>
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_mount.h"
13 : #include "xfs_inode.h"
14 : #include "xfs_btree.h"
15 : #include "xfs_ialloc.h"
16 : #include "xfs_ialloc_btree.h"
17 : #include "xfs_iwalk.h"
18 : #include "xfs_error.h"
19 : #include "xfs_trace.h"
20 : #include "xfs_icache.h"
21 : #include "xfs_health.h"
22 : #include "xfs_trans.h"
23 : #include "xfs_pwork.h"
24 : #include "xfs_ag.h"
25 : #include "xfs_bit.h"
26 :
27 : /*
28 : * Walking Inodes in the Filesystem
29 : * ================================
30 : *
31 : * This iterator function walks a subset of filesystem inodes in increasing
32 : * order from @startino until there are no more inodes. For each allocated
33 : * inode it finds, it calls a walk function with the relevant inode number and
34 : * a pointer to caller-provided data. The walk function can return the usual
35 : * negative error code to stop the iteration; 0 to continue the iteration; or
36 : * -ECANCELED to stop the iteration. This return value is returned to the
37 : * caller.
38 : *
39 : * Internally, we allow the walk function to do anything, which means that we
40 : * cannot maintain the inobt cursor or our lock on the AGI buffer. We
41 : * therefore cache the inobt records in kernel memory and only call the walk
42 : * function when our memory buffer is full. @nr_recs is the number of records
43 : * that we've cached, and @sz_recs is the size of our cache.
44 : *
45 : * It is the responsibility of the walk function to ensure it accesses
46 : * allocated inodes, as the inobt records may be stale by the time they are
47 : * acted upon.
48 : */
49 :
50 : struct xfs_iwalk_ag {
51 : /* parallel work control data; will be null if single threaded */
52 : struct xfs_pwork pwork;
53 :
54 : struct xfs_mount *mp;
55 : struct xfs_trans *tp;
56 : struct xfs_perag *pag;
57 :
58 : /* Where do we start the traversal? */
59 : xfs_ino_t startino;
60 :
61 : /* What was the last inode number we saw when iterating the inobt? */
62 : xfs_ino_t lastino;
63 :
64 : /* Array of inobt records we cache. */
65 : struct xfs_inobt_rec_incore *recs;
66 :
67 : /* Number of entries allocated for the @recs array. */
68 : unsigned int sz_recs;
69 :
70 : /* Number of entries in the @recs array that are in use. */
71 : unsigned int nr_recs;
72 :
73 : /* Inode walk function and data pointer. */
74 : xfs_iwalk_fn iwalk_fn;
75 : xfs_inobt_walk_fn inobt_walk_fn;
76 : void *data;
77 :
78 : /*
79 : * Make it look like the inodes up to startino are free so that
80 : * bulkstat can start its inode iteration at the correct place without
81 : * needing to special case everywhere.
82 : */
83 : unsigned int trim_start:1;
84 :
85 : /* Skip empty inobt records? */
86 : unsigned int skip_empty:1;
87 :
88 : /* Drop the (hopefully empty) transaction when calling iwalk_fn. */
89 : unsigned int drop_trans:1;
90 : };
91 :
92 : /*
93 : * Loop over all clusters in a chunk for a given incore inode allocation btree
94 : * record. Do a readahead if there are any allocated inodes in that cluster.
95 : */
96 : STATIC void
97 2425804968 : xfs_iwalk_ichunk_ra(
98 : struct xfs_mount *mp,
99 : struct xfs_perag *pag,
100 : struct xfs_inobt_rec_incore *irec)
101 : {
102 2425804968 : struct xfs_ino_geometry *igeo = M_IGEO(mp);
103 2425804968 : xfs_agblock_t agbno;
104 2425804968 : struct blk_plug plug;
105 2425804968 : int i; /* inode chunk index */
106 :
107 2425804968 : agbno = XFS_AGINO_TO_AGBNO(mp, irec->ir_startino);
108 :
109 2425804968 : blk_start_plug(&plug);
110 9702647089 : for (i = 0; i < XFS_INODES_PER_CHUNK; i += igeo->inodes_per_cluster) {
111 4851017478 : xfs_inofree_t imask;
112 :
113 4851017478 : imask = xfs_inobt_maskn(i, igeo->inodes_per_cluster);
114 4850662129 : if (imask & ~irec->ir_free) {
115 3663569919 : xfs_btree_reada_bufs(mp, pag->pag_agno, agbno,
116 : igeo->blocks_per_cluster,
117 : &xfs_inode_buf_ops);
118 : }
119 4851058904 : agbno += igeo->blocks_per_cluster;
120 : }
121 2425824643 : blk_finish_plug(&plug);
122 2425748816 : }
123 :
124 : /*
125 : * Set the bits in @irec's free mask that correspond to the inodes before
126 : * @agino so that we skip them. This is how we restart an inode walk that was
127 : * interrupted in the middle of an inode record.
128 : */
129 : STATIC void
130 478858185 : xfs_iwalk_adjust_start(
131 : xfs_agino_t agino, /* starting inode of chunk */
132 : struct xfs_inobt_rec_incore *irec) /* btree record */
133 : {
134 478858185 : int idx; /* index into inode chunk */
135 :
136 478858185 : idx = agino - irec->ir_startino;
137 :
138 478858185 : irec->ir_free |= xfs_inobt_maskn(0, idx);
139 478858185 : irec->ir_freecount = hweight64(irec->ir_free);
140 478858185 : }
141 :
142 : /* Allocate memory for a walk. */
143 : STATIC int
144 492758551 : xfs_iwalk_alloc(
145 : struct xfs_iwalk_ag *iwag)
146 : {
147 492758551 : size_t size;
148 :
149 492758551 : ASSERT(iwag->recs == NULL);
150 492758551 : iwag->nr_recs = 0;
151 :
152 : /* Allocate a prefetch buffer for inobt records. */
153 492758551 : size = iwag->sz_recs * sizeof(struct xfs_inobt_rec_incore);
154 492758551 : iwag->recs = kmem_alloc(size, KM_MAYFAIL);
155 492726709 : if (iwag->recs == NULL)
156 0 : return -ENOMEM;
157 :
158 : return 0;
159 : }
160 :
161 : /* Free memory we allocated for a walk. */
162 : STATIC void
163 492753257 : xfs_iwalk_free(
164 : struct xfs_iwalk_ag *iwag)
165 : {
166 492753257 : kmem_free(iwag->recs);
167 492732945 : iwag->recs = NULL;
168 492732945 : }
169 :
170 : /* For each inuse inode in each cached inobt record, call our function. */
171 : STATIC int
172 641096208 : xfs_iwalk_ag_recs(
173 : struct xfs_iwalk_ag *iwag)
174 : {
175 641096208 : struct xfs_mount *mp = iwag->mp;
176 641096208 : struct xfs_trans *tp = iwag->tp;
177 641096208 : struct xfs_perag *pag = iwag->pag;
178 641096208 : xfs_ino_t ino;
179 641096208 : unsigned int i, j;
180 641096208 : int error;
181 :
182 2154707323 : for (i = 0; i < iwag->nr_recs; i++) {
183 2002278827 : struct xfs_inobt_rec_incore *irec = &iwag->recs[i];
184 :
185 2002278827 : trace_xfs_iwalk_ag_rec(mp, pag->pag_agno, irec);
186 :
187 4003261114 : if (xfs_pwork_want_abort(&iwag->pwork))
188 : return 0;
189 :
190 2001630557 : if (iwag->inobt_walk_fn) {
191 8116080 : error = iwag->inobt_walk_fn(mp, tp, pag->pag_agno, irec,
192 : iwag->data);
193 8125738 : if (error)
194 5140932 : return error;
195 : }
196 :
197 1996499283 : if (!iwag->iwalk_fn)
198 2978178 : continue;
199 :
200 >11241*10^7 : for (j = 0; j < XFS_INODES_PER_CHUNK; j++) {
201 >22181*10^7 : if (xfs_pwork_want_abort(&iwag->pwork))
202 : return 0;
203 :
204 : /* Skip if this inode is free */
205 >11090*10^7 : if (XFS_INOBT_MASK(j) & irec->ir_free)
206 38876475133 : continue;
207 :
208 : /* Otherwise call our function. */
209 72031384538 : ino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
210 : irec->ir_startino + j);
211 72031384538 : error = iwag->iwalk_fn(mp, tp, ino, iwag->data);
212 72031979588 : if (error)
213 483483218 : return error;
214 : }
215 : }
216 :
217 : return 0;
218 : }
219 :
220 : /* Delete cursor and let go of AGI. */
221 : static inline void
222 1140200537 : xfs_iwalk_del_inobt(
223 : struct xfs_trans *tp,
224 : struct xfs_btree_cur **curpp,
225 : struct xfs_buf **agi_bpp,
226 : int error)
227 : {
228 1140200537 : if (*curpp) {
229 651596596 : xfs_btree_del_cursor(*curpp, error);
230 651607671 : *curpp = NULL;
231 : }
232 1140211612 : if (*agi_bpp) {
233 651607585 : xfs_trans_brelse(tp, *agi_bpp);
234 651617960 : *agi_bpp = NULL;
235 : }
236 1140221987 : }
237 :
238 : /*
239 : * Set ourselves up for walking inobt records starting from a given point in
240 : * the filesystem.
241 : *
242 : * If caller passed in a nonzero start inode number, load the record from the
243 : * inobt and make the record look like all the inodes before agino are free so
244 : * that we skip them, and then move the cursor to the next inobt record. This
245 : * is how we support starting an iwalk in the middle of an inode chunk.
246 : *
247 : * If the caller passed in a start number of zero, move the cursor to the first
248 : * inobt record.
249 : *
250 : * The caller is responsible for cleaning up the cursor and buffer pointer
251 : * regardless of the error status.
252 : */
253 : STATIC int
254 499098065 : xfs_iwalk_ag_start(
255 : struct xfs_iwalk_ag *iwag,
256 : xfs_agino_t agino,
257 : struct xfs_btree_cur **curpp,
258 : struct xfs_buf **agi_bpp,
259 : int *has_more)
260 : {
261 499098065 : struct xfs_mount *mp = iwag->mp;
262 499098065 : struct xfs_trans *tp = iwag->tp;
263 499098065 : struct xfs_perag *pag = iwag->pag;
264 499098065 : struct xfs_inobt_rec_incore *irec;
265 499098065 : int error;
266 :
267 : /* Set up a fresh cursor and empty the inobt cache. */
268 499098065 : iwag->nr_recs = 0;
269 499098065 : error = xfs_inobt_cur(pag, tp, XFS_BTNUM_INO, curpp, agi_bpp);
270 499151605 : if (error)
271 : return error;
272 :
273 : /* Starting at the beginning of the AG? That's easy! */
274 499150991 : if (agino == 0)
275 8631704 : return xfs_inobt_lookup(*curpp, 0, XFS_LOOKUP_GE, has_more);
276 :
277 : /*
278 : * Otherwise, we have to grab the inobt record where we left off, stuff
279 : * the record into our cache, and then see if there are more records.
280 : * We require a lookup cache of at least two elements so that the
281 : * caller doesn't have to deal with tearing down the cursor to walk the
282 : * records.
283 : */
284 490519287 : error = xfs_inobt_lookup(*curpp, agino, XFS_LOOKUP_LE, has_more);
285 490521249 : if (error)
286 : return error;
287 :
288 : /*
289 : * If the LE lookup at @agino yields no records, jump ahead to the
290 : * inobt cursor increment to see if there are more records to process.
291 : */
292 490521176 : if (!*has_more)
293 51 : goto out_advance;
294 :
295 : /* Get the record, should always work */
296 490521125 : irec = &iwag->recs[iwag->nr_recs];
297 490521125 : error = xfs_inobt_get_rec(*curpp, irec, has_more);
298 490528362 : if (error)
299 : return error;
300 490528362 : if (XFS_IS_CORRUPT(mp, *has_more != 1)) {
301 0 : xfs_btree_mark_sick(*curpp);
302 0 : return -EFSCORRUPTED;
303 : }
304 :
305 490528362 : iwag->lastino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
306 : irec->ir_startino + XFS_INODES_PER_CHUNK - 1);
307 :
308 : /*
309 : * If the LE lookup yielded an inobt record before the cursor position,
310 : * skip it and see if there's another one after it.
311 : */
312 490528362 : if (irec->ir_startino + XFS_INODES_PER_CHUNK <= agino)
313 8406711 : goto out_advance;
314 :
315 : /*
316 : * If agino fell in the middle of the inode record, make it look like
317 : * the inodes up to agino are free so that we don't return them again.
318 : */
319 482121651 : if (iwag->trim_start)
320 478867291 : xfs_iwalk_adjust_start(agino, irec);
321 :
322 : /*
323 : * The prefetch calculation is supposed to give us a large enough inobt
324 : * record cache that grab_ichunk can stage a partial first record and
325 : * the loop body can cache a record without having to check for cache
326 : * space until after it reads an inobt record.
327 : */
328 482107382 : iwag->nr_recs++;
329 482107382 : ASSERT(iwag->nr_recs < iwag->sz_recs);
330 :
331 482107382 : out_advance:
332 490514144 : return xfs_btree_increment(*curpp, 0, has_more);
333 : }
334 :
335 : /*
336 : * The inobt record cache is full, so preserve the inobt cursor state and
337 : * run callbacks on the cached inobt records. When we're done, restore the
338 : * cursor state to wherever the cursor would have been had the cache not been
339 : * full (and therefore we could've just incremented the cursor) if *@has_more
340 : * is true. On exit, *@has_more will indicate whether or not the caller should
341 : * try for more inode records.
342 : */
343 : STATIC int
344 641104462 : xfs_iwalk_run_callbacks(
345 : struct xfs_iwalk_ag *iwag,
346 : struct xfs_btree_cur **curpp,
347 : struct xfs_buf **agi_bpp,
348 : int *has_more)
349 : {
350 641104462 : struct xfs_mount *mp = iwag->mp;
351 641104462 : struct xfs_inobt_rec_incore *irec;
352 641104462 : xfs_agino_t next_agino;
353 641104462 : int error;
354 :
355 641104462 : next_agino = XFS_INO_TO_AGINO(mp, iwag->lastino) + 1;
356 :
357 641104462 : ASSERT(iwag->nr_recs > 0);
358 :
359 : /* Delete cursor but remember the last record we cached... */
360 641104462 : xfs_iwalk_del_inobt(iwag->tp, curpp, agi_bpp, 0);
361 641103936 : irec = &iwag->recs[iwag->nr_recs - 1];
362 641103936 : ASSERT(next_agino >= irec->ir_startino + XFS_INODES_PER_CHUNK);
363 :
364 641103936 : if (iwag->drop_trans) {
365 7565 : xfs_trans_cancel(iwag->tp);
366 7564 : iwag->tp = NULL;
367 : }
368 :
369 641103935 : error = xfs_iwalk_ag_recs(iwag);
370 641049936 : if (error)
371 : return error;
372 :
373 : /* ...empty the cache... */
374 152429141 : iwag->nr_recs = 0;
375 :
376 152429141 : if (!has_more)
377 : return 0;
378 :
379 152429141 : if (iwag->drop_trans) {
380 7565 : error = xfs_trans_alloc_empty(mp, &iwag->tp);
381 7565 : if (error)
382 : return error;
383 : }
384 :
385 : /* ...and recreate the cursor just past where we left off. */
386 152429141 : error = xfs_inobt_cur(iwag->pag, iwag->tp, XFS_BTNUM_INO, curpp,
387 : agi_bpp);
388 152435151 : if (error)
389 : return error;
390 :
391 152429723 : return xfs_inobt_lookup(*curpp, next_agino, XFS_LOOKUP_GE, has_more);
392 : }
393 :
394 : /* Walk all inodes in a single AG, from @iwag->startino to the end of the AG. */
395 : STATIC int
396 499094100 : xfs_iwalk_ag(
397 : struct xfs_iwalk_ag *iwag)
398 : {
399 499094100 : struct xfs_mount *mp = iwag->mp;
400 499094100 : struct xfs_perag *pag = iwag->pag;
401 499094100 : struct xfs_buf *agi_bp = NULL;
402 499094100 : struct xfs_btree_cur *cur = NULL;
403 499094100 : xfs_agino_t agino;
404 499094100 : int has_more;
405 499094100 : int error = 0;
406 :
407 : /* Set up our cursor at the right place in the inode btree. */
408 499094100 : ASSERT(pag->pag_agno == XFS_INO_TO_AGNO(mp, iwag->startino));
409 499094100 : agino = XFS_INO_TO_AGINO(mp, iwag->startino);
410 499094100 : error = xfs_iwalk_ag_start(iwag, agino, &cur, &agi_bp, &has_more);
411 :
412 2926272182 : while (!error && has_more) {
413 2435611206 : struct xfs_inobt_rec_incore *irec;
414 2435611206 : xfs_ino_t rec_fsino;
415 :
416 2435611206 : cond_resched();
417 4871230320 : if (xfs_pwork_want_abort(&iwag->pwork))
418 0 : goto out;
419 :
420 : /* Fetch the inobt record. */
421 2435615160 : irec = &iwag->recs[iwag->nr_recs];
422 2435615160 : error = xfs_inobt_get_rec(cur, irec, &has_more);
423 2435728950 : if (error || !has_more)
424 : break;
425 :
426 : /* Make sure that we always move forward. */
427 2435728950 : rec_fsino = XFS_AGINO_TO_INO(mp, pag->pag_agno, irec->ir_startino);
428 2435728950 : if (iwag->lastino != NULLFSINO &&
429 2433756440 : XFS_IS_CORRUPT(mp, iwag->lastino >= rec_fsino)) {
430 0 : xfs_btree_mark_sick(cur);
431 0 : error = -EFSCORRUPTED;
432 0 : goto out;
433 : }
434 2435728950 : iwag->lastino = rec_fsino + XFS_INODES_PER_CHUNK - 1;
435 :
436 : /* No allocated inodes in this chunk; skip it. */
437 2435728950 : if (iwag->skip_empty && irec->ir_freecount == irec->ir_count) {
438 0 : error = xfs_btree_increment(cur, 0, &has_more);
439 0 : if (error)
440 : break;
441 0 : continue;
442 : }
443 :
444 : /*
445 : * Start readahead for this inode chunk in anticipation of
446 : * walking the inodes.
447 : */
448 2435728950 : if (iwag->iwalk_fn)
449 2425848063 : xfs_iwalk_ichunk_ra(mp, pag, irec);
450 :
451 : /*
452 : * If there's space in the buffer for more records, increment
453 : * the btree cursor and grab more.
454 : */
455 2435817783 : if (++iwag->nr_recs < iwag->sz_recs) {
456 1806750814 : error = xfs_btree_increment(cur, 0, &has_more);
457 1806729086 : if (error || !has_more)
458 : break;
459 1798321161 : continue;
460 : }
461 :
462 : /*
463 : * Otherwise, we need to save cursor state and run the callback
464 : * function on the cached records. The run_callbacks function
465 : * is supposed to return a cursor pointing to the record where
466 : * we would be if we had been able to increment like above.
467 : */
468 629066969 : ASSERT(has_more);
469 629066969 : error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more);
470 : }
471 :
472 499130444 : if (iwag->nr_recs == 0 || error)
473 487095338 : goto out;
474 :
475 : /* Walk the unprocessed records in the cache. */
476 12035106 : error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more);
477 :
478 499129651 : out:
479 499129651 : xfs_iwalk_del_inobt(iwag->tp, &cur, &agi_bp, error);
480 499106812 : return error;
481 : }
482 :
483 : /*
484 : * We experimentally determined that the reduction in ioctl call overhead
485 : * diminishes when userspace asks for more than 2048 inodes, so we'll cap
486 : * prefetch at this point.
487 : */
488 : #define IWALK_MAX_INODE_PREFETCH (2048U)
489 :
490 : /*
491 : * Given the number of inodes to prefetch, set the number of inobt records that
492 : * we cache in memory, which controls the number of inodes we try to read
493 : * ahead. Set the maximum if @inodes == 0.
494 : */
495 : static inline unsigned int
496 : xfs_iwalk_prefetch(
497 : unsigned int inodes)
498 : {
499 487030255 : unsigned int inobt_records;
500 :
501 : /*
502 : * If the caller didn't tell us the number of inodes they wanted,
503 : * assume the maximum prefetch possible for best performance.
504 : * Otherwise, cap prefetch at that maximum so that we don't start an
505 : * absurd amount of prefetch.
506 : */
507 487030255 : if (inodes == 0)
508 60939 : inodes = IWALK_MAX_INODE_PREFETCH;
509 487030255 : inodes = min(inodes, IWALK_MAX_INODE_PREFETCH);
510 :
511 : /* Round the inode count up to a full chunk. */
512 487030255 : inodes = round_up(inodes, XFS_INODES_PER_CHUNK);
513 :
514 : /*
515 : * In order to convert the number of inodes to prefetch into an
516 : * estimate of the number of inobt records to cache, we require a
517 : * conversion factor that reflects our expectations of the average
518 : * loading factor of an inode chunk. Based on data gathered, most
519 : * (but not all) filesystems manage to keep the inode chunks totally
520 : * full, so we'll underestimate slightly so that our readahead will
521 : * still deliver the performance we want on aging filesystems:
522 : *
523 : * inobt = inodes / (INODES_PER_CHUNK * (4 / 5));
524 : *
525 : * The funny math is to avoid integer division.
526 : */
527 487030255 : inobt_records = (inodes * 5) / (4 * XFS_INODES_PER_CHUNK);
528 :
529 : /*
530 : * Allocate enough space to prefetch at least two inobt records so that
531 : * we can cache both the record where the iwalk started and the next
532 : * record. This simplifies the AG inode walk loop setup code.
533 : */
534 487030255 : return max(inobt_records, 2U);
535 : }
536 :
537 : /*
538 : * Walk all inodes in the filesystem starting from @startino. The @iwalk_fn
539 : * will be called for each allocated inode, being passed the inode's number and
540 : * @data. @max_prefetch controls how many inobt records' worth of inodes we
541 : * try to readahead.
542 : */
543 : int
544 486969316 : xfs_iwalk(
545 : struct xfs_mount *mp,
546 : struct xfs_trans *tp,
547 : xfs_ino_t startino,
548 : unsigned int flags,
549 : xfs_iwalk_fn iwalk_fn,
550 : unsigned int inode_records,
551 : void *data)
552 : {
553 486969316 : struct xfs_iwalk_ag iwag = {
554 : .mp = mp,
555 : .tp = tp,
556 : .iwalk_fn = iwalk_fn,
557 : .data = data,
558 : .startino = startino,
559 : .sz_recs = xfs_iwalk_prefetch(inode_records),
560 : .trim_start = 1,
561 : .skip_empty = 1,
562 : .pwork = XFS_PWORK_SINGLE_THREADED,
563 : .lastino = NULLFSINO,
564 : };
565 486969316 : struct xfs_perag *pag;
566 486969316 : xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
567 486969316 : int error;
568 :
569 486969316 : ASSERT(agno < mp->m_sb.sb_agcount);
570 486969316 : ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
571 :
572 486969316 : error = xfs_iwalk_alloc(&iwag);
573 487005555 : if (error)
574 : return error;
575 :
576 496888314 : for_each_perag_from(mp, agno, pag) {
577 493348058 : iwag.pag = pag;
578 493348058 : error = xfs_iwalk_ag(&iwag);
579 493368451 : if (error)
580 : break;
581 9887715 : iwag.startino = XFS_AGINO_TO_INO(mp, agno + 1, 0);
582 9887715 : if (flags & XFS_INOBT_WALK_SAME_AG)
583 : break;
584 9882787 : iwag.pag = NULL;
585 : }
586 :
587 487025336 : if (iwag.pag)
588 483477659 : xfs_perag_rele(pag);
589 487038881 : xfs_iwalk_free(&iwag);
590 487038881 : return error;
591 : }
592 :
593 : /* Run per-thread iwalk work. */
594 : static int
595 60938 : xfs_iwalk_ag_work(
596 : struct xfs_mount *mp,
597 : struct xfs_pwork *pwork)
598 : {
599 60938 : struct xfs_iwalk_ag *iwag;
600 60938 : int error = 0;
601 :
602 60938 : iwag = container_of(pwork, struct xfs_iwalk_ag, pwork);
603 121876 : if (xfs_pwork_want_abort(pwork))
604 0 : goto out;
605 :
606 60938 : error = xfs_iwalk_alloc(iwag);
607 60938 : if (error)
608 0 : goto out;
609 : /*
610 : * Grab an empty transaction so that we can use its recursive buffer
611 : * locking abilities to detect cycles in the inobt without deadlocking.
612 : */
613 60938 : error = xfs_trans_alloc_empty(mp, &iwag->tp);
614 60928 : if (error)
615 0 : goto out;
616 60928 : iwag->drop_trans = 1;
617 :
618 60928 : error = xfs_iwalk_ag(iwag);
619 60907 : if (iwag->tp)
620 60907 : xfs_trans_cancel(iwag->tp);
621 60846 : xfs_iwalk_free(iwag);
622 60822 : out:
623 60822 : xfs_perag_put(iwag->pag);
624 60926 : kmem_free(iwag);
625 60878 : return error;
626 : }
627 :
628 : /*
629 : * Walk all the inodes in the filesystem using multiple threads to process each
630 : * AG.
631 : */
632 : int
633 7318 : xfs_iwalk_threaded(
634 : struct xfs_mount *mp,
635 : xfs_ino_t startino,
636 : unsigned int flags,
637 : xfs_iwalk_fn iwalk_fn,
638 : unsigned int inode_records,
639 : bool polled,
640 : void *data)
641 : {
642 7318 : struct xfs_pwork_ctl pctl;
643 7318 : struct xfs_perag *pag;
644 7318 : xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
645 7318 : int error;
646 :
647 7318 : ASSERT(agno < mp->m_sb.sb_agcount);
648 7318 : ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
649 :
650 7318 : error = xfs_pwork_init(mp, &pctl, xfs_iwalk_ag_work, "xfs_iwalk");
651 7318 : if (error)
652 : return error;
653 :
654 68257 : for_each_perag_from(mp, agno, pag) {
655 60939 : struct xfs_iwalk_ag *iwag;
656 :
657 121878 : if (xfs_pwork_ctl_want_abort(&pctl))
658 : break;
659 :
660 60939 : iwag = kmem_zalloc(sizeof(struct xfs_iwalk_ag), 0);
661 60939 : iwag->mp = mp;
662 :
663 : /*
664 : * perag is being handed off to async work, so take a passive
665 : * reference for the async work to release.
666 : */
667 60939 : iwag->pag = xfs_perag_hold(pag);
668 60939 : iwag->iwalk_fn = iwalk_fn;
669 60939 : iwag->data = data;
670 60939 : iwag->startino = startino;
671 60939 : iwag->sz_recs = xfs_iwalk_prefetch(inode_records);
672 60939 : iwag->lastino = NULLFSINO;
673 60939 : xfs_pwork_queue(&pctl, &iwag->pwork);
674 60939 : startino = XFS_AGINO_TO_INO(mp, pag->pag_agno + 1, 0);
675 60939 : if (flags & XFS_INOBT_WALK_SAME_AG)
676 : break;
677 : }
678 7318 : if (pag)
679 0 : xfs_perag_rele(pag);
680 7318 : if (polled)
681 7318 : xfs_pwork_poll(&pctl);
682 7318 : return xfs_pwork_destroy(&pctl);
683 : }
684 :
685 : /*
686 : * Allow callers to cache up to a page's worth of inobt records. This reflects
687 : * the existing inumbers prefetching behavior. Since the inobt walk does not
688 : * itself do anything with the inobt records, we can set a fairly high limit
689 : * here.
690 : */
691 : #define MAX_INOBT_WALK_PREFETCH \
692 : (PAGE_SIZE / sizeof(struct xfs_inobt_rec_incore))
693 :
694 : /*
695 : * Given the number of records that the user wanted, set the number of inobt
696 : * records that we buffer in memory. Set the maximum if @inobt_records == 0.
697 : */
698 : static inline unsigned int
699 : xfs_inobt_walk_prefetch(
700 : unsigned int inobt_records)
701 : {
702 : /*
703 : * If the caller didn't tell us the number of inobt records they
704 : * wanted, assume the maximum prefetch possible for best performance.
705 : */
706 5668089 : if (inobt_records == 0)
707 0 : inobt_records = MAX_INOBT_WALK_PREFETCH;
708 :
709 : /*
710 : * Allocate enough space to prefetch at least two inobt records so that
711 : * we can cache both the record where the iwalk started and the next
712 : * record. This simplifies the AG inode walk loop setup code.
713 : */
714 5668089 : inobt_records = max(inobt_records, 2U);
715 :
716 : /*
717 : * Cap prefetch at that maximum so that we don't use an absurd amount
718 : * of memory.
719 : */
720 5668089 : return min_t(unsigned int, inobt_records, MAX_INOBT_WALK_PREFETCH);
721 : }
722 :
723 : /*
724 : * Walk all inode btree records in the filesystem starting from @startino. The
725 : * @inobt_walk_fn will be called for each btree record, being passed the incore
726 : * record and @data. @max_prefetch controls how many inobt records we try to
727 : * cache ahead of time.
728 : */
729 : int
730 5668089 : xfs_inobt_walk(
731 : struct xfs_mount *mp,
732 : struct xfs_trans *tp,
733 : xfs_ino_t startino,
734 : unsigned int flags,
735 : xfs_inobt_walk_fn inobt_walk_fn,
736 : unsigned int inobt_records,
737 : void *data)
738 : {
739 5668089 : struct xfs_iwalk_ag iwag = {
740 : .mp = mp,
741 : .tp = tp,
742 : .inobt_walk_fn = inobt_walk_fn,
743 : .data = data,
744 : .startino = startino,
745 : .sz_recs = xfs_inobt_walk_prefetch(inobt_records),
746 : .pwork = XFS_PWORK_SINGLE_THREADED,
747 : .lastino = NULLFSINO,
748 : };
749 5668089 : struct xfs_perag *pag;
750 5668089 : xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
751 5668089 : int error;
752 :
753 5668089 : ASSERT(agno < mp->m_sb.sb_agcount);
754 5668089 : ASSERT(!(flags & ~XFS_INOBT_WALK_FLAGS_ALL));
755 :
756 5668089 : error = xfs_iwalk_alloc(&iwag);
757 5664312 : if (error)
758 : return error;
759 :
760 5708368 : for_each_perag_from(mp, agno, pag) {
761 5711535 : iwag.pag = pag;
762 5711535 : error = xfs_iwalk_ag(&iwag);
763 5703032 : if (error)
764 : break;
765 563123 : iwag.startino = XFS_AGINO_TO_INO(mp, pag->pag_agno + 1, 0);
766 563123 : if (flags & XFS_INOBT_WALK_SAME_AG)
767 : break;
768 44414 : iwag.pag = NULL;
769 : }
770 :
771 5666110 : if (iwag.pag)
772 5656776 : xfs_perag_rele(pag);
773 5672383 : xfs_iwalk_free(&iwag);
774 5672383 : return error;
775 : }
|
