Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
4 : * All Rights Reserved.
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_bit.h"
13 : #include "xfs_mount.h"
14 : #include "xfs_inode.h"
15 : #include "xfs_btree.h"
16 : #include "xfs_ialloc.h"
17 : #include "xfs_ialloc_btree.h"
18 : #include "xfs_alloc.h"
19 : #include "xfs_errortag.h"
20 : #include "xfs_error.h"
21 : #include "xfs_bmap.h"
22 : #include "xfs_trans.h"
23 : #include "xfs_buf_item.h"
24 : #include "xfs_icreate_item.h"
25 : #include "xfs_icache.h"
26 : #include "xfs_trace.h"
27 : #include "xfs_log.h"
28 : #include "xfs_rmap.h"
29 : #include "xfs_ag.h"
30 : #include "xfs_health.h"
31 :
32 : /*
33 : * Lookup a record by ino in the btree given by cur.
34 : */
35 : int /* error */
36 4743487978 : xfs_inobt_lookup(
37 : struct xfs_btree_cur *cur, /* btree cursor */
38 : xfs_agino_t ino, /* starting inode of chunk */
39 : xfs_lookup_t dir, /* <=, >=, == */
40 : int *stat) /* success/failure */
41 : {
42 10378487548 : cur->bc_rec.i.ir_startino = ino;
43 10378487548 : cur->bc_rec.i.ir_holemask = 0;
44 10378487548 : cur->bc_rec.i.ir_count = 0;
45 10378487548 : cur->bc_rec.i.ir_freecount = 0;
46 10378487548 : cur->bc_rec.i.ir_free = 0;
47 4743487978 : return xfs_btree_lookup(cur, dir, stat);
48 : }
49 :
50 : /*
51 : * Update the record referred to by cur to the value given.
52 : * This either works (return 0) or gets an EFSCORRUPTED error.
53 : */
54 : STATIC int /* error */
55 356718010 : xfs_inobt_update(
56 : struct xfs_btree_cur *cur, /* btree cursor */
57 : xfs_inobt_rec_incore_t *irec) /* btree record */
58 : {
59 356718010 : union xfs_btree_rec rec;
60 :
61 356718010 : rec.inobt.ir_startino = cpu_to_be32(irec->ir_startino);
62 356718010 : if (xfs_has_sparseinodes(cur->bc_mp)) {
63 356716575 : rec.inobt.ir_u.sp.ir_holemask = cpu_to_be16(irec->ir_holemask);
64 356716575 : rec.inobt.ir_u.sp.ir_count = irec->ir_count;
65 356716575 : rec.inobt.ir_u.sp.ir_freecount = irec->ir_freecount;
66 : } else {
67 : /* ir_holemask/ir_count not supported on-disk */
68 1435 : rec.inobt.ir_u.f.ir_freecount = cpu_to_be32(irec->ir_freecount);
69 : }
70 356718010 : rec.inobt.ir_free = cpu_to_be64(irec->ir_free);
71 356718010 : return xfs_btree_update(cur, &rec);
72 : }
73 :
74 : /* Convert on-disk btree record to incore inobt record. */
75 : void
76 30655298111 : xfs_inobt_btrec_to_irec(
77 : struct xfs_mount *mp,
78 : const union xfs_btree_rec *rec,
79 : struct xfs_inobt_rec_incore *irec)
80 : {
81 30655298111 : irec->ir_startino = be32_to_cpu(rec->inobt.ir_startino);
82 30655298111 : if (xfs_has_sparseinodes(mp)) {
83 30655291354 : irec->ir_holemask = be16_to_cpu(rec->inobt.ir_u.sp.ir_holemask);
84 30655291354 : irec->ir_count = rec->inobt.ir_u.sp.ir_count;
85 30655291354 : irec->ir_freecount = rec->inobt.ir_u.sp.ir_freecount;
86 : } else {
87 : /*
88 : * ir_holemask/ir_count not supported on-disk. Fill in hardcoded
89 : * values for full inode chunks.
90 : */
91 6757 : irec->ir_holemask = XFS_INOBT_HOLEMASK_FULL;
92 6757 : irec->ir_count = XFS_INODES_PER_CHUNK;
93 6757 : irec->ir_freecount =
94 6757 : be32_to_cpu(rec->inobt.ir_u.f.ir_freecount);
95 : }
96 30655298111 : irec->ir_free = be64_to_cpu(rec->inobt.ir_free);
97 30655298111 : }
98 :
99 : /* Compute the freecount of an incore inode record. */
100 : uint8_t
101 30729036409 : xfs_inobt_rec_freecount(
102 : const struct xfs_inobt_rec_incore *irec)
103 : {
104 30729036409 : uint64_t realfree;
105 :
106 30729036409 : if (!xfs_inobt_issparse(irec->ir_holemask))
107 20346261918 : realfree = irec->ir_free;
108 : else
109 10382774491 : realfree = irec->ir_free & xfs_inobt_irec_to_allocmask(irec);
110 30735266990 : return hweight64(realfree);
111 : }
112 :
113 : inline xfs_failaddr_t
114 30650845590 : xfs_inobt_check_perag_irec(
115 : struct xfs_perag *pag,
116 : const struct xfs_inobt_rec_incore *irec)
117 : {
118 : /* Record has to be properly aligned within the AG. */
119 30650845590 : if (!xfs_verify_agino(pag, irec->ir_startino))
120 0 : return __this_address;
121 30650845590 : if (!xfs_verify_agino(pag,
122 : irec->ir_startino + XFS_INODES_PER_CHUNK - 1))
123 0 : return __this_address;
124 30650845590 : if (irec->ir_count < XFS_INODES_PER_HOLEMASK_BIT ||
125 : irec->ir_count > XFS_INODES_PER_CHUNK)
126 0 : return __this_address;
127 30650845590 : if (irec->ir_freecount > XFS_INODES_PER_CHUNK)
128 0 : return __this_address;
129 :
130 30650845590 : if (xfs_inobt_rec_freecount(irec) != irec->ir_freecount)
131 0 : return __this_address;
132 :
133 : return NULL;
134 : }
135 :
136 : /* Simple checks for inode records. */
137 : xfs_failaddr_t
138 58401888 : xfs_inobt_check_irec(
139 : struct xfs_btree_cur *cur,
140 : const struct xfs_inobt_rec_incore *irec)
141 : {
142 58401888 : return xfs_inobt_check_perag_irec(cur->bc_ag.pag, irec);
143 : }
144 :
145 : static inline int
146 0 : xfs_inobt_complain_bad_rec(
147 : struct xfs_btree_cur *cur,
148 : xfs_failaddr_t fa,
149 : const struct xfs_inobt_rec_incore *irec)
150 : {
151 0 : struct xfs_mount *mp = cur->bc_mp;
152 :
153 0 : xfs_warn(mp,
154 : "%s Inode BTree record corruption in AG %d detected at %pS!",
155 : cur->bc_btnum == XFS_BTNUM_INO ? "Used" : "Free",
156 : cur->bc_ag.pag->pag_agno, fa);
157 0 : xfs_warn(mp,
158 : "start inode 0x%x, count 0x%x, free 0x%x freemask 0x%llx, holemask 0x%x",
159 : irec->ir_startino, irec->ir_count, irec->ir_freecount,
160 : irec->ir_free, irec->ir_holemask);
161 0 : xfs_btree_mark_sick(cur);
162 0 : return -EFSCORRUPTED;
163 : }
164 :
165 : /*
166 : * Get the data from the pointed-to record.
167 : */
168 : int
169 30400778121 : xfs_inobt_get_rec(
170 : struct xfs_btree_cur *cur,
171 : struct xfs_inobt_rec_incore *irec,
172 : int *stat)
173 : {
174 30400778121 : struct xfs_mount *mp = cur->bc_mp;
175 30400778121 : union xfs_btree_rec *rec;
176 30400778121 : xfs_failaddr_t fa;
177 30400778121 : int error;
178 :
179 30400778121 : error = xfs_btree_get_rec(cur, &rec, stat);
180 30325335844 : if (error || *stat == 0)
181 : return error;
182 :
183 30329934118 : xfs_inobt_btrec_to_irec(mp, rec, irec);
184 30340567556 : fa = xfs_inobt_check_irec(cur, irec);
185 30358068831 : if (fa)
186 0 : return xfs_inobt_complain_bad_rec(cur, fa, irec);
187 :
188 : return 0;
189 : }
190 :
191 : /*
192 : * Insert a single inobt record. Cursor must already point to desired location.
193 : */
194 : int
195 0 : xfs_inobt_insert_rec(
196 : struct xfs_btree_cur *cur,
197 : uint16_t holemask,
198 : uint8_t count,
199 : int32_t freecount,
200 : xfs_inofree_t free,
201 : int *stat)
202 : {
203 22955626 : cur->bc_rec.i.ir_holemask = holemask;
204 22955626 : cur->bc_rec.i.ir_count = count;
205 22955626 : cur->bc_rec.i.ir_freecount = freecount;
206 22955626 : cur->bc_rec.i.ir_free = free;
207 0 : return xfs_btree_insert(cur, stat);
208 : }
209 :
210 : /*
211 : * Insert records describing a newly allocated inode chunk into the inobt.
212 : */
213 : STATIC int
214 1459745 : xfs_inobt_insert(
215 : struct xfs_perag *pag,
216 : struct xfs_trans *tp,
217 : struct xfs_buf *agbp,
218 : xfs_agino_t newino,
219 : xfs_agino_t newlen,
220 : xfs_btnum_t btnum)
221 : {
222 1459745 : struct xfs_btree_cur *cur;
223 1459745 : xfs_agino_t thisino;
224 1459745 : int i;
225 1459745 : int error;
226 :
227 1459745 : cur = xfs_inobt_init_cursor(pag, tp, agbp, btnum);
228 :
229 1459745 : for (thisino = newino;
230 2919601 : thisino < newino + newlen;
231 1459717 : thisino += XFS_INODES_PER_CHUNK) {
232 1459875 : error = xfs_inobt_lookup(cur, thisino, XFS_LOOKUP_EQ, &i);
233 1459883 : if (error) {
234 6 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
235 6 : return error;
236 : }
237 1459877 : ASSERT(i == 0);
238 :
239 1459877 : error = xfs_inobt_insert_rec(cur, XFS_INOBT_HOLEMASK_FULL,
240 : XFS_INODES_PER_CHUNK,
241 : XFS_INODES_PER_CHUNK,
242 : XFS_INOBT_ALL_FREE, &i);
243 1459718 : if (error) {
244 1 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
245 1 : return error;
246 : }
247 1459717 : ASSERT(i == 1);
248 : }
249 :
250 1459726 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
251 :
252 1459726 : return 0;
253 : }
254 :
255 : /*
256 : * Verify that the number of free inodes in the AGI is correct.
257 : */
258 : #ifdef DEBUG
259 : static int
260 720441708 : xfs_check_agi_freecount(
261 : struct xfs_btree_cur *cur)
262 : {
263 720441708 : if (cur->bc_nlevels == 1) {
264 564522802 : xfs_inobt_rec_incore_t rec;
265 564522802 : int freecount = 0;
266 564522802 : int error;
267 564522802 : int i;
268 :
269 564522802 : error = xfs_inobt_lookup(cur, 0, XFS_LOOKUP_GE, &i);
270 565070860 : if (error)
271 239 : return error;
272 :
273 18805031116 : do {
274 18805031116 : error = xfs_inobt_get_rec(cur, &rec, &i);
275 18848101545 : if (error)
276 0 : return error;
277 :
278 18848101545 : if (i) {
279 18857972483 : freecount += rec.ir_freecount;
280 18857972483 : error = xfs_btree_increment(cur, 0, &i);
281 18815183693 : if (error)
282 0 : return error;
283 : }
284 18805312755 : } while (i == 1);
285 :
286 1130704520 : if (!xfs_is_shutdown(cur->bc_mp))
287 565437583 : ASSERT(freecount == cur->bc_ag.pag->pagi_freecount);
288 : }
289 : return 0;
290 : }
291 : #else
292 : #define xfs_check_agi_freecount(cur) 0
293 : #endif
294 :
295 : /*
296 : * Initialise a new set of inodes. When called without a transaction context
297 : * (e.g. from recovery) we initiate a delayed write of the inode buffers rather
298 : * than logging them (which in a transaction context puts them into the AIL
299 : * for writeback rather than the xfsbufd queue).
300 : */
301 : int
302 1499522 : xfs_ialloc_inode_init(
303 : struct xfs_mount *mp,
304 : struct xfs_trans *tp,
305 : struct list_head *buffer_list,
306 : int icount,
307 : xfs_agnumber_t agno,
308 : xfs_agblock_t agbno,
309 : xfs_agblock_t length,
310 : unsigned int gen)
311 : {
312 1499522 : struct xfs_buf *fbuf;
313 1499522 : struct xfs_dinode *free;
314 1499522 : int nbufs;
315 1499522 : int version;
316 1499522 : int i, j;
317 1499522 : xfs_daddr_t d;
318 1499522 : xfs_ino_t ino = 0;
319 1499522 : int error;
320 :
321 : /*
322 : * Loop over the new block(s), filling in the inodes. For small block
323 : * sizes, manipulate the inodes in buffers which are multiples of the
324 : * blocks size.
325 : */
326 1499522 : nbufs = length / M_IGEO(mp)->blocks_per_cluster;
327 :
328 : /*
329 : * Figure out what version number to use in the inodes we create. If
330 : * the superblock version has caught up to the one that supports the new
331 : * inode format, then use the new inode version. Otherwise use the old
332 : * version so that old kernels will continue to be able to use the file
333 : * system.
334 : *
335 : * For v3 inodes, we also need to write the inode number into the inode,
336 : * so calculate the first inode number of the chunk here as
337 : * XFS_AGB_TO_AGINO() only works within a filesystem block, not
338 : * across multiple filesystem blocks (such as a cluster) and so cannot
339 : * be used in the cluster buffer loop below.
340 : *
341 : * Further, because we are writing the inode directly into the buffer
342 : * and calculating a CRC on the entire inode, we have ot log the entire
343 : * inode so that the entire range the CRC covers is present in the log.
344 : * That means for v3 inode we log the entire buffer rather than just the
345 : * inode cores.
346 : */
347 1499522 : if (xfs_has_v3inodes(mp)) {
348 1499445 : version = 3;
349 1499445 : ino = XFS_AGINO_TO_INO(mp, agno, XFS_AGB_TO_AGINO(mp, agbno));
350 :
351 : /*
352 : * log the initialisation that is about to take place as an
353 : * logical operation. This means the transaction does not
354 : * need to log the physical changes to the inode buffers as log
355 : * recovery will know what initialisation is actually needed.
356 : * Hence we only need to log the buffers as "ordered" buffers so
357 : * they track in the AIL as if they were physically logged.
358 : */
359 1499445 : if (tp)
360 1469247 : xfs_icreate_log(tp, agno, agbno, icount,
361 1469247 : mp->m_sb.sb_inodesize, length, gen);
362 : } else
363 : version = 2;
364 :
365 3745288 : for (j = 0; j < nbufs; j++) {
366 : /*
367 : * Get the block.
368 : */
369 2245092 : d = XFS_AGB_TO_DADDR(mp, agno, agbno +
370 : (j * M_IGEO(mp)->blocks_per_cluster));
371 2245092 : error = xfs_trans_get_buf(tp, mp->m_ddev_targp, d,
372 2245092 : mp->m_bsize * M_IGEO(mp)->blocks_per_cluster,
373 : XBF_UNMAPPED, &fbuf);
374 2244938 : if (error)
375 0 : return error;
376 :
377 : /* Initialize the inode buffers and log them appropriately. */
378 2244938 : fbuf->b_ops = &xfs_inode_buf_ops;
379 2244938 : xfs_buf_zero(fbuf, 0, BBTOB(fbuf->b_length));
380 76338290 : for (i = 0; i < M_IGEO(mp)->inodes_per_cluster; i++) {
381 71847954 : int ioffset = i << mp->m_sb.sb_inodelog;
382 :
383 71847954 : free = xfs_make_iptr(mp, fbuf, i);
384 71844574 : free->di_magic = cpu_to_be16(XFS_DINODE_MAGIC);
385 71844574 : free->di_version = version;
386 71844574 : free->di_gen = cpu_to_be32(gen);
387 71844574 : free->di_next_unlinked = cpu_to_be32(NULLAGINO);
388 :
389 71844574 : if (version == 3) {
390 71843870 : free->di_ino = cpu_to_be64(ino);
391 71843870 : ino++;
392 71843870 : uuid_copy(&free->di_uuid,
393 71843870 : &mp->m_sb.sb_meta_uuid);
394 71843546 : xfs_dinode_calc_crc(mp, free);
395 704 : } else if (tp) {
396 : /* just log the inode core */
397 704 : xfs_trans_log_buf(tp, fbuf, ioffset,
398 1408 : ioffset + XFS_DINODE_SIZE(mp) - 1);
399 : }
400 : }
401 :
402 2245398 : if (tp) {
403 : /*
404 : * Mark the buffer as an inode allocation buffer so it
405 : * sticks in AIL at the point of this allocation
406 : * transaction. This ensures the they are on disk before
407 : * the tail of the log can be moved past this
408 : * transaction (i.e. by preventing relogging from moving
409 : * it forward in the log).
410 : */
411 2200110 : xfs_trans_inode_alloc_buf(tp, fbuf);
412 2199859 : if (version == 3) {
413 : /*
414 : * Mark the buffer as ordered so that they are
415 : * not physically logged in the transaction but
416 : * still tracked in the AIL as part of the
417 : * transaction and pin the log appropriately.
418 : */
419 2199837 : xfs_trans_ordered_buf(tp, fbuf);
420 : }
421 : } else {
422 45288 : fbuf->b_flags |= XBF_DONE;
423 45288 : xfs_buf_delwri_queue(fbuf, buffer_list);
424 45288 : xfs_buf_relse(fbuf);
425 : }
426 : }
427 : return 0;
428 : }
429 :
430 : /*
431 : * Align startino and allocmask for a recently allocated sparse chunk such that
432 : * they are fit for insertion (or merge) into the on-disk inode btrees.
433 : *
434 : * Background:
435 : *
436 : * When enabled, sparse inode support increases the inode alignment from cluster
437 : * size to inode chunk size. This means that the minimum range between two
438 : * non-adjacent inode records in the inobt is large enough for a full inode
439 : * record. This allows for cluster sized, cluster aligned block allocation
440 : * without need to worry about whether the resulting inode record overlaps with
441 : * another record in the tree. Without this basic rule, we would have to deal
442 : * with the consequences of overlap by potentially undoing recent allocations in
443 : * the inode allocation codepath.
444 : *
445 : * Because of this alignment rule (which is enforced on mount), there are two
446 : * inobt possibilities for newly allocated sparse chunks. One is that the
447 : * aligned inode record for the chunk covers a range of inodes not already
448 : * covered in the inobt (i.e., it is safe to insert a new sparse record). The
449 : * other is that a record already exists at the aligned startino that considers
450 : * the newly allocated range as sparse. In the latter case, record content is
451 : * merged in hope that sparse inode chunks fill to full chunks over time.
452 : */
453 : STATIC void
454 740021 : xfs_align_sparse_ino(
455 : struct xfs_mount *mp,
456 : xfs_agino_t *startino,
457 : uint16_t *allocmask)
458 : {
459 740021 : xfs_agblock_t agbno;
460 740021 : xfs_agblock_t mod;
461 740021 : int offset;
462 :
463 740021 : agbno = XFS_AGINO_TO_AGBNO(mp, *startino);
464 740021 : mod = agbno % mp->m_sb.sb_inoalignmt;
465 740021 : if (!mod)
466 : return;
467 :
468 : /* calculate the inode offset and align startino */
469 413743 : offset = XFS_AGB_TO_AGINO(mp, mod);
470 413743 : *startino -= offset;
471 :
472 : /*
473 : * Since startino has been aligned down, left shift allocmask such that
474 : * it continues to represent the same physical inodes relative to the
475 : * new startino.
476 : */
477 413743 : *allocmask <<= offset / XFS_INODES_PER_HOLEMASK_BIT;
478 : }
479 :
480 : /*
481 : * Determine whether the source inode record can merge into the target. Both
482 : * records must be sparse, the inode ranges must match and there must be no
483 : * allocation overlap between the records.
484 : */
485 : STATIC bool
486 129712 : __xfs_inobt_can_merge(
487 : struct xfs_inobt_rec_incore *trec, /* tgt record */
488 : struct xfs_inobt_rec_incore *srec) /* src record */
489 : {
490 129712 : uint64_t talloc;
491 129712 : uint64_t salloc;
492 :
493 : /* records must cover the same inode range */
494 129712 : if (trec->ir_startino != srec->ir_startino)
495 : return false;
496 :
497 : /* both records must be sparse */
498 129708 : if (!xfs_inobt_issparse(trec->ir_holemask) ||
499 129708 : !xfs_inobt_issparse(srec->ir_holemask))
500 : return false;
501 :
502 : /* both records must track some inodes */
503 129708 : if (!trec->ir_count || !srec->ir_count)
504 : return false;
505 :
506 : /* can't exceed capacity of a full record */
507 129708 : if (trec->ir_count + srec->ir_count > XFS_INODES_PER_CHUNK)
508 : return false;
509 :
510 : /* verify there is no allocation overlap */
511 129708 : talloc = xfs_inobt_irec_to_allocmask(trec);
512 129726 : salloc = xfs_inobt_irec_to_allocmask(srec);
513 129726 : if (talloc & salloc)
514 0 : return false;
515 :
516 : return true;
517 : }
518 :
519 : /*
520 : * Merge the source inode record into the target. The caller must call
521 : * __xfs_inobt_can_merge() to ensure the merge is valid.
522 : */
523 : STATIC void
524 129704 : __xfs_inobt_rec_merge(
525 : struct xfs_inobt_rec_incore *trec, /* target */
526 : struct xfs_inobt_rec_incore *srec) /* src */
527 : {
528 129704 : ASSERT(trec->ir_startino == srec->ir_startino);
529 :
530 : /* combine the counts */
531 129704 : trec->ir_count += srec->ir_count;
532 129704 : trec->ir_freecount += srec->ir_freecount;
533 :
534 : /*
535 : * Merge the holemask and free mask. For both fields, 0 bits refer to
536 : * allocated inodes. We combine the allocated ranges with bitwise AND.
537 : */
538 129704 : trec->ir_holemask &= srec->ir_holemask;
539 129704 : trec->ir_free &= srec->ir_free;
540 129704 : }
541 :
542 : /*
543 : * Insert a new sparse inode chunk into the associated inode btree. The inode
544 : * record for the sparse chunk is pre-aligned to a startino that should match
545 : * any pre-existing sparse inode record in the tree. This allows sparse chunks
546 : * to fill over time.
547 : *
548 : * This function supports two modes of handling preexisting records depending on
549 : * the merge flag. If merge is true, the provided record is merged with the
550 : * existing record and updated in place. The merged record is returned in nrec.
551 : * If merge is false, an existing record is replaced with the provided record.
552 : * If no preexisting record exists, the provided record is always inserted.
553 : *
554 : * It is considered corruption if a merge is requested and not possible. Given
555 : * the sparse inode alignment constraints, this should never happen.
556 : */
557 : STATIC int
558 1479928 : xfs_inobt_insert_sprec(
559 : struct xfs_perag *pag,
560 : struct xfs_trans *tp,
561 : struct xfs_buf *agbp,
562 : int btnum,
563 : struct xfs_inobt_rec_incore *nrec, /* in/out: new/merged rec. */
564 : bool merge) /* merge or replace */
565 : {
566 1479928 : struct xfs_mount *mp = pag->pag_mount;
567 1479928 : struct xfs_btree_cur *cur;
568 1479928 : int error;
569 1479928 : int i;
570 1479928 : struct xfs_inobt_rec_incore rec;
571 :
572 1479928 : cur = xfs_inobt_init_cursor(pag, tp, agbp, btnum);
573 :
574 : /* the new record is pre-aligned so we know where to look */
575 1480070 : error = xfs_inobt_lookup(cur, nrec->ir_startino, XFS_LOOKUP_EQ, &i);
576 1480116 : if (error)
577 4 : goto error;
578 : /* if nothing there, insert a new record and return */
579 1480112 : if (i == 0) {
580 1350376 : error = xfs_inobt_insert_rec(cur, nrec->ir_holemask,
581 1350376 : nrec->ir_count, nrec->ir_freecount,
582 : nrec->ir_free, &i);
583 1350311 : if (error)
584 0 : goto error;
585 1350311 : if (XFS_IS_CORRUPT(mp, i != 1)) {
586 0 : xfs_btree_mark_sick(cur);
587 0 : error = -EFSCORRUPTED;
588 0 : goto error;
589 : }
590 :
591 1350311 : goto out;
592 : }
593 :
594 : /*
595 : * A record exists at this startino. Merge or replace the record
596 : * depending on what we've been asked to do.
597 : */
598 129736 : if (merge) {
599 129735 : error = xfs_inobt_get_rec(cur, &rec, &i);
600 129721 : if (error)
601 0 : goto error;
602 129721 : if (XFS_IS_CORRUPT(mp, i != 1)) {
603 0 : xfs_btree_mark_sick(cur);
604 0 : error = -EFSCORRUPTED;
605 0 : goto error;
606 : }
607 129721 : if (XFS_IS_CORRUPT(mp, rec.ir_startino != nrec->ir_startino)) {
608 0 : xfs_btree_mark_sick(cur);
609 0 : error = -EFSCORRUPTED;
610 0 : goto error;
611 : }
612 :
613 : /*
614 : * This should never fail. If we have coexisting records that
615 : * cannot merge, something is seriously wrong.
616 : */
617 129721 : if (XFS_IS_CORRUPT(mp, !__xfs_inobt_can_merge(nrec, &rec))) {
618 0 : xfs_btree_mark_sick(cur);
619 0 : error = -EFSCORRUPTED;
620 0 : goto error;
621 : }
622 :
623 129721 : trace_xfs_irec_merge_pre(mp, pag->pag_agno, rec.ir_startino,
624 129721 : rec.ir_holemask, nrec->ir_startino,
625 129721 : nrec->ir_holemask);
626 :
627 : /* merge to nrec to output the updated record */
628 129704 : __xfs_inobt_rec_merge(nrec, &rec);
629 :
630 129707 : trace_xfs_irec_merge_post(mp, pag->pag_agno, nrec->ir_startino,
631 129707 : nrec->ir_holemask);
632 :
633 129707 : error = xfs_inobt_rec_check_count(mp, nrec);
634 129727 : if (error)
635 0 : goto error;
636 : }
637 :
638 129728 : error = xfs_inobt_update(cur, nrec);
639 129726 : if (error)
640 0 : goto error;
641 :
642 129726 : out:
643 1480037 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
644 1480037 : return 0;
645 4 : error:
646 4 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
647 4 : return error;
648 : }
649 :
650 : /*
651 : * Allocate new inodes in the allocation group specified by agbp. Returns 0 if
652 : * inodes were allocated in this AG; -EAGAIN if there was no space in this AG so
653 : * the caller knows it can try another AG, a hard -ENOSPC when over the maximum
654 : * inode count threshold, or the usual negative error code for other errors.
655 : */
656 : STATIC int
657 3121689 : xfs_ialloc_ag_alloc(
658 : struct xfs_perag *pag,
659 : struct xfs_trans *tp,
660 : struct xfs_buf *agbp)
661 : {
662 3121689 : struct xfs_agi *agi;
663 3121689 : struct xfs_alloc_arg args;
664 3121689 : int error;
665 3121689 : xfs_agino_t newino; /* new first inode's number */
666 3121689 : xfs_agino_t newlen; /* new number of inodes */
667 3121689 : int isaligned = 0; /* inode allocation at stripe */
668 : /* unit boundary */
669 : /* init. to full chunk */
670 3121689 : struct xfs_inobt_rec_incore rec;
671 3121689 : struct xfs_ino_geometry *igeo = M_IGEO(tp->t_mountp);
672 3121689 : uint16_t allocmask = (uint16_t) -1;
673 3121689 : int do_sparse = 0;
674 :
675 3121689 : memset(&args, 0, sizeof(args));
676 3121689 : args.tp = tp;
677 3121689 : args.mp = tp->t_mountp;
678 3121689 : args.fsbno = NULLFSBLOCK;
679 3121689 : args.oinfo = XFS_RMAP_OINFO_INODES;
680 3121689 : args.pag = pag;
681 :
682 : #ifdef DEBUG
683 : /* randomly do sparse inode allocations */
684 3121689 : if (xfs_has_sparseinodes(tp->t_mountp) &&
685 3121492 : igeo->ialloc_min_blks < igeo->ialloc_blks)
686 3121383 : do_sparse = get_random_u32_below(2);
687 : #endif
688 :
689 : /*
690 : * Locking will ensure that we don't have two callers in here
691 : * at one time.
692 : */
693 3121717 : newlen = igeo->ialloc_inos;
694 3121717 : if (igeo->maxicount &&
695 3121623 : percpu_counter_read_positive(&args.mp->m_icount) + newlen >
696 : igeo->maxicount)
697 : return -ENOSPC;
698 3121690 : args.minlen = args.maxlen = igeo->ialloc_blks;
699 : /*
700 : * First try to allocate inodes contiguous with the last-allocated
701 : * chunk of inodes. If the filesystem is striped, this will fill
702 : * an entire stripe unit with inodes.
703 : */
704 3121690 : agi = agbp->b_addr;
705 3121690 : newino = be32_to_cpu(agi->agi_newino);
706 3121690 : args.agbno = XFS_AGINO_TO_AGBNO(args.mp, newino) +
707 3121690 : igeo->ialloc_blks;
708 3121690 : if (do_sparse)
709 1559634 : goto sparse_alloc;
710 1562056 : if (likely(newino != NULLAGINO &&
711 : (args.agbno < be32_to_cpu(agi->agi_length)))) {
712 1501019 : args.prod = 1;
713 :
714 : /*
715 : * We need to take into account alignment here to ensure that
716 : * we don't modify the free list if we fail to have an exact
717 : * block. If we don't have an exact match, and every oher
718 : * attempt allocation attempt fails, we'll end up cancelling
719 : * a dirty transaction and shutting down.
720 : *
721 : * For an exact allocation, alignment must be 1,
722 : * however we need to take cluster alignment into account when
723 : * fixing up the freelist. Use the minalignslop field to
724 : * indicate that extra blocks might be required for alignment,
725 : * but not to use them in the actual exact allocation.
726 : */
727 1501019 : args.alignment = 1;
728 1501019 : args.minalignslop = igeo->cluster_align - 1;
729 :
730 : /* Allow space for the inode btree to split. */
731 1501019 : args.minleft = igeo->inobt_maxlevels;
732 3002038 : error = xfs_alloc_vextent_exact_bno(&args,
733 1501019 : XFS_AGB_TO_FSB(args.mp, pag->pag_agno,
734 : args.agbno));
735 1501059 : if (error)
736 : return error;
737 :
738 : /*
739 : * This request might have dirtied the transaction if the AG can
740 : * satisfy the request, but the exact block was not available.
741 : * If the allocation did fail, subsequent requests will relax
742 : * the exact agbno requirement and increase the alignment
743 : * instead. It is critical that the total size of the request
744 : * (len + alignment + slop) does not increase from this point
745 : * on, so reset minalignslop to ensure it is not included in
746 : * subsequent requests.
747 : */
748 1501058 : args.minalignslop = 0;
749 : }
750 :
751 1562095 : if (unlikely(args.fsbno == NULLFSBLOCK)) {
752 : /*
753 : * Set the alignment for the allocation.
754 : * If stripe alignment is turned on then align at stripe unit
755 : * boundary.
756 : * If the cluster size is smaller than a filesystem block
757 : * then we're doing I/O for inodes in filesystem block size
758 : * pieces, so don't need alignment anyway.
759 : */
760 1266794 : isaligned = 0;
761 1266794 : if (igeo->ialloc_align) {
762 0 : ASSERT(!xfs_has_noalign(args.mp));
763 0 : args.alignment = args.mp->m_dalign;
764 0 : isaligned = 1;
765 : } else
766 1266794 : args.alignment = igeo->cluster_align;
767 : /*
768 : * Allocate a fixed-size extent of inodes.
769 : */
770 1266794 : args.prod = 1;
771 : /*
772 : * Allow space for the inode btree to split.
773 : */
774 1266794 : args.minleft = igeo->inobt_maxlevels;
775 2533588 : error = xfs_alloc_vextent_near_bno(&args,
776 1266794 : XFS_AGB_TO_FSB(args.mp, pag->pag_agno,
777 : be32_to_cpu(agi->agi_root)));
778 1266794 : if (error)
779 : return error;
780 : }
781 :
782 : /*
783 : * If stripe alignment is turned on, then try again with cluster
784 : * alignment.
785 : */
786 1266794 : if (isaligned && args.fsbno == NULLFSBLOCK) {
787 0 : args.alignment = igeo->cluster_align;
788 0 : error = xfs_alloc_vextent_near_bno(&args,
789 0 : XFS_AGB_TO_FSB(args.mp, pag->pag_agno,
790 : be32_to_cpu(agi->agi_root)));
791 0 : if (error)
792 : return error;
793 : }
794 :
795 : /*
796 : * Finally, try a sparse allocation if the filesystem supports it and
797 : * the sparse allocation length is smaller than a full chunk.
798 : */
799 1562095 : if (xfs_has_sparseinodes(args.mp) &&
800 1562048 : igeo->ialloc_min_blks < igeo->ialloc_blks &&
801 1562029 : args.fsbno == NULLFSBLOCK) {
802 832151 : sparse_alloc:
803 2391785 : args.alignment = args.mp->m_sb.sb_spino_align;
804 2391785 : args.prod = 1;
805 :
806 2391785 : args.minlen = igeo->ialloc_min_blks;
807 2391785 : args.maxlen = args.minlen;
808 :
809 : /*
810 : * The inode record will be aligned to full chunk size. We must
811 : * prevent sparse allocation from AG boundaries that result in
812 : * invalid inode records, such as records that start at agbno 0
813 : * or extend beyond the AG.
814 : *
815 : * Set min agbno to the first aligned, non-zero agbno and max to
816 : * the last aligned agbno that is at least one full chunk from
817 : * the end of the AG.
818 : */
819 2391785 : args.min_agbno = args.mp->m_sb.sb_inoalignmt;
820 2391785 : args.max_agbno = round_down(args.mp->m_sb.sb_agblocks,
821 2391785 : args.mp->m_sb.sb_inoalignmt) -
822 2391785 : igeo->ialloc_blks;
823 :
824 4783570 : error = xfs_alloc_vextent_near_bno(&args,
825 2391785 : XFS_AGB_TO_FSB(args.mp, pag->pag_agno,
826 : be32_to_cpu(agi->agi_root)));
827 2391828 : if (error)
828 : return error;
829 :
830 2391826 : newlen = XFS_AGB_TO_AGINO(args.mp, args.len);
831 2391826 : ASSERT(newlen <= XFS_INODES_PER_CHUNK);
832 2391826 : allocmask = (1 << (newlen / XFS_INODES_PER_HOLEMASK_BIT)) - 1;
833 : }
834 :
835 3121770 : if (args.fsbno == NULLFSBLOCK)
836 : return -EAGAIN;
837 :
838 1469970 : ASSERT(args.len == args.minlen);
839 :
840 : /*
841 : * Stamp and write the inode buffers.
842 : *
843 : * Seed the new inode cluster with a random generation number. This
844 : * prevents short-term reuse of generation numbers if a chunk is
845 : * freed and then immediately reallocated. We use random numbers
846 : * rather than a linear progression to prevent the next generation
847 : * number from being easily guessable.
848 : */
849 1469970 : error = xfs_ialloc_inode_init(args.mp, tp, NULL, newlen, pag->pag_agno,
850 : args.agbno, args.len, get_random_u32());
851 :
852 1470004 : if (error)
853 : return error;
854 : /*
855 : * Convert the results.
856 : */
857 1470004 : newino = XFS_AGB_TO_AGINO(args.mp, args.agbno);
858 :
859 1470004 : if (xfs_inobt_issparse(~allocmask)) {
860 : /*
861 : * We've allocated a sparse chunk. Align the startino and mask.
862 : */
863 740034 : xfs_align_sparse_ino(args.mp, &newino, &allocmask);
864 :
865 739983 : rec.ir_startino = newino;
866 739983 : rec.ir_holemask = ~allocmask;
867 739983 : rec.ir_count = newlen;
868 739983 : rec.ir_freecount = newlen;
869 739983 : rec.ir_free = XFS_INOBT_ALL_FREE;
870 :
871 : /*
872 : * Insert the sparse record into the inobt and allow for a merge
873 : * if necessary. If a merge does occur, rec is updated to the
874 : * merged record.
875 : */
876 739983 : error = xfs_inobt_insert_sprec(pag, tp, agbp,
877 : XFS_BTNUM_INO, &rec, true);
878 740083 : if (error == -EFSCORRUPTED) {
879 0 : xfs_alert(args.mp,
880 : "invalid sparse inode record: ino 0x%llx holemask 0x%x count %u",
881 : XFS_AGINO_TO_INO(args.mp, pag->pag_agno,
882 : rec.ir_startino),
883 : rec.ir_holemask, rec.ir_count);
884 0 : xfs_force_shutdown(args.mp, SHUTDOWN_CORRUPT_INCORE);
885 : }
886 740083 : if (error)
887 : return error;
888 :
889 : /*
890 : * We can't merge the part we've just allocated as for the inobt
891 : * due to finobt semantics. The original record may or may not
892 : * exist independent of whether physical inodes exist in this
893 : * sparse chunk.
894 : *
895 : * We must update the finobt record based on the inobt record.
896 : * rec contains the fully merged and up to date inobt record
897 : * from the previous call. Set merge false to replace any
898 : * existing record with this one.
899 : */
900 740083 : if (xfs_has_finobt(args.mp)) {
901 739996 : error = xfs_inobt_insert_sprec(pag, tp, agbp,
902 : XFS_BTNUM_FINO, &rec, false);
903 740033 : if (error)
904 : return error;
905 : }
906 : } else {
907 : /* full chunk - insert new records to both btrees */
908 729970 : error = xfs_inobt_insert(pag, tp, agbp, newino, newlen,
909 : XFS_BTNUM_INO);
910 729980 : if (error)
911 : return error;
912 :
913 729979 : if (xfs_has_finobt(args.mp)) {
914 729901 : error = xfs_inobt_insert(pag, tp, agbp, newino,
915 : newlen, XFS_BTNUM_FINO);
916 729909 : if (error)
917 : return error;
918 : }
919 : }
920 :
921 : /*
922 : * Update AGI counts and newino.
923 : */
924 1470097 : be32_add_cpu(&agi->agi_count, newlen);
925 1470097 : be32_add_cpu(&agi->agi_freecount, newlen);
926 1470097 : pag->pagi_freecount += newlen;
927 1470097 : pag->pagi_count += newlen;
928 1470097 : agi->agi_newino = cpu_to_be32(newino);
929 :
930 : /*
931 : * Log allocation group header fields
932 : */
933 1470097 : xfs_ialloc_log_agi(tp, agbp,
934 : XFS_AGI_COUNT | XFS_AGI_FREECOUNT | XFS_AGI_NEWINO);
935 : /*
936 : * Modify/log superblock values for inode count and inode free count.
937 : */
938 1470051 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_ICOUNT, (long)newlen);
939 1469746 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, (long)newlen);
940 1469746 : return 0;
941 : }
942 :
943 : /*
944 : * Try to retrieve the next record to the left/right from the current one.
945 : */
946 : STATIC int
947 2877 : xfs_ialloc_next_rec(
948 : struct xfs_btree_cur *cur,
949 : xfs_inobt_rec_incore_t *rec,
950 : int *done,
951 : int left)
952 : {
953 2877 : int error;
954 2877 : int i;
955 :
956 2877 : if (left)
957 1395 : error = xfs_btree_decrement(cur, 0, &i);
958 : else
959 1482 : error = xfs_btree_increment(cur, 0, &i);
960 :
961 2878 : if (error)
962 : return error;
963 2878 : *done = !i;
964 2878 : if (i) {
965 1627 : error = xfs_inobt_get_rec(cur, rec, &i);
966 1627 : if (error)
967 : return error;
968 1627 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
969 0 : xfs_btree_mark_sick(cur);
970 0 : return -EFSCORRUPTED;
971 : }
972 : }
973 :
974 : return 0;
975 : }
976 :
977 : STATIC int
978 1370 : xfs_ialloc_get_rec(
979 : struct xfs_btree_cur *cur,
980 : xfs_agino_t agino,
981 : xfs_inobt_rec_incore_t *rec,
982 : int *done)
983 : {
984 1370 : int error;
985 1370 : int i;
986 :
987 1370 : error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_EQ, &i);
988 1370 : if (error)
989 : return error;
990 1370 : *done = !i;
991 1370 : if (i) {
992 709 : error = xfs_inobt_get_rec(cur, rec, &i);
993 709 : if (error)
994 : return error;
995 709 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
996 0 : xfs_btree_mark_sick(cur);
997 0 : return -EFSCORRUPTED;
998 : }
999 : }
1000 :
1001 : return 0;
1002 : }
1003 :
1004 : /*
1005 : * Return the offset of the first free inode in the record. If the inode chunk
1006 : * is sparsely allocated, we convert the record holemask to inode granularity
1007 : * and mask off the unallocated regions from the inode free mask.
1008 : */
1009 : STATIC int
1010 123654072 : xfs_inobt_first_free_inode(
1011 : struct xfs_inobt_rec_incore *rec)
1012 : {
1013 123654072 : xfs_inofree_t realfree;
1014 :
1015 : /* if there are no holes, return the first available offset */
1016 123654072 : if (!xfs_inobt_issparse(rec->ir_holemask))
1017 87530559 : return xfs_lowbit64(rec->ir_free);
1018 :
1019 36123513 : realfree = xfs_inobt_irec_to_allocmask(rec);
1020 36119267 : realfree &= rec->ir_free;
1021 :
1022 36119267 : return xfs_lowbit64(realfree);
1023 : }
1024 :
1025 : /*
1026 : * If this AG has corrupt inodes, check if allocating this inode would fail
1027 : * with corruption errors. Returns 0 if we're clear, or EAGAIN to try again
1028 : * somewhere else.
1029 : */
1030 : static int
1031 0 : xfs_dialloc_check_ino(
1032 : struct xfs_perag *pag,
1033 : struct xfs_trans *tp,
1034 : xfs_ino_t ino)
1035 : {
1036 0 : struct xfs_imap imap;
1037 0 : struct xfs_buf *bp;
1038 0 : int error;
1039 :
1040 0 : error = xfs_imap(pag, tp, ino, &imap, 0);
1041 0 : if (error)
1042 : return -EAGAIN;
1043 :
1044 0 : error = xfs_imap_to_bp(pag->pag_mount, tp, &imap, &bp);
1045 0 : if (error)
1046 : return -EAGAIN;
1047 :
1048 0 : xfs_trans_brelse(tp, bp);
1049 0 : return 0;
1050 : }
1051 :
1052 : /*
1053 : * Allocate an inode using the inobt-only algorithm.
1054 : */
1055 : STATIC int
1056 7337 : xfs_dialloc_ag_inobt(
1057 : struct xfs_perag *pag,
1058 : struct xfs_trans *tp,
1059 : struct xfs_buf *agbp,
1060 : xfs_ino_t parent,
1061 : xfs_ino_t *inop)
1062 : {
1063 7337 : struct xfs_mount *mp = tp->t_mountp;
1064 7337 : struct xfs_agi *agi = agbp->b_addr;
1065 7337 : xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent);
1066 7337 : xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent);
1067 7337 : struct xfs_btree_cur *cur, *tcur;
1068 7337 : struct xfs_inobt_rec_incore rec, trec;
1069 7337 : xfs_ino_t ino;
1070 7337 : int error;
1071 7337 : int offset;
1072 7337 : int i, j;
1073 7337 : int searchdistance = 10;
1074 :
1075 14674 : ASSERT(xfs_perag_initialised_agi(pag));
1076 14674 : ASSERT(xfs_perag_allows_inodes(pag));
1077 14674 : ASSERT(!xfs_perag_prohibits_alloc(pag));
1078 7337 : ASSERT(pag->pagi_freecount > 0);
1079 :
1080 7337 : restart_pagno:
1081 7346 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_INO);
1082 : /*
1083 : * If pagino is 0 (this is the root inode allocation) use newino.
1084 : * This must work because we've just allocated some.
1085 : */
1086 7347 : if (!pagino)
1087 115 : pagino = be32_to_cpu(agi->agi_newino);
1088 :
1089 7347 : error = xfs_check_agi_freecount(cur);
1090 7347 : if (error)
1091 0 : goto error0;
1092 :
1093 : /*
1094 : * If in the same AG as the parent, try to get near the parent.
1095 : */
1096 7347 : if (pagno == pag->pag_agno) {
1097 6529 : int doneleft; /* done, to the left */
1098 6529 : int doneright; /* done, to the right */
1099 :
1100 6529 : error = xfs_inobt_lookup(cur, pagino, XFS_LOOKUP_LE, &i);
1101 6530 : if (error)
1102 0 : goto error0;
1103 6530 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1104 0 : xfs_btree_mark_sick(cur);
1105 0 : error = -EFSCORRUPTED;
1106 0 : goto error0;
1107 : }
1108 :
1109 6530 : error = xfs_inobt_get_rec(cur, &rec, &j);
1110 6530 : if (error)
1111 0 : goto error0;
1112 6530 : if (XFS_IS_CORRUPT(mp, j != 1)) {
1113 0 : xfs_btree_mark_sick(cur);
1114 0 : error = -EFSCORRUPTED;
1115 0 : goto error0;
1116 : }
1117 :
1118 6530 : if (rec.ir_freecount > 0) {
1119 : /*
1120 : * Found a free inode in the same chunk
1121 : * as the parent, done.
1122 : */
1123 6519 : goto alloc_inode;
1124 : }
1125 :
1126 :
1127 : /*
1128 : * In the same AG as parent, but parent's chunk is full.
1129 : */
1130 :
1131 : /* duplicate the cursor, search left & right simultaneously */
1132 1975 : error = xfs_btree_dup_cursor(cur, &tcur);
1133 1974 : if (error)
1134 0 : goto error0;
1135 :
1136 : /*
1137 : * Skip to last blocks looked up if same parent inode.
1138 : */
1139 1974 : if (pagino != NULLAGINO &&
1140 1974 : pag->pagl_pagino == pagino &&
1141 685 : pag->pagl_leftrec != NULLAGINO &&
1142 685 : pag->pagl_rightrec != NULLAGINO) {
1143 685 : error = xfs_ialloc_get_rec(tcur, pag->pagl_leftrec,
1144 : &trec, &doneleft);
1145 685 : if (error)
1146 0 : goto error1;
1147 :
1148 685 : error = xfs_ialloc_get_rec(cur, pag->pagl_rightrec,
1149 : &rec, &doneright);
1150 685 : if (error)
1151 0 : goto error1;
1152 : } else {
1153 : /* search left with tcur, back up 1 record */
1154 1289 : error = xfs_ialloc_next_rec(tcur, &trec, &doneleft, 1);
1155 1288 : if (error)
1156 0 : goto error1;
1157 :
1158 : /* search right with cur, go forward 1 record. */
1159 1288 : error = xfs_ialloc_next_rec(cur, &rec, &doneright, 0);
1160 1290 : if (error)
1161 0 : goto error1;
1162 : }
1163 :
1164 : /*
1165 : * Loop until we find an inode chunk with a free inode.
1166 : */
1167 2274 : while (--searchdistance > 0 && (!doneleft || !doneright)) {
1168 2264 : int useleft; /* using left inode chunk this time */
1169 :
1170 : /* figure out the closer block if both are valid. */
1171 2264 : if (!doneleft && !doneright) {
1172 183 : useleft = pagino -
1173 183 : (trec.ir_startino + XFS_INODES_PER_CHUNK - 1) <
1174 183 : rec.ir_startino - pagino;
1175 : } else {
1176 2081 : useleft = !doneleft;
1177 : }
1178 :
1179 : /* free inodes to the left? */
1180 2264 : if (useleft && trec.ir_freecount) {
1181 59 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
1182 59 : cur = tcur;
1183 :
1184 59 : pag->pagl_leftrec = trec.ir_startino;
1185 59 : pag->pagl_rightrec = rec.ir_startino;
1186 59 : pag->pagl_pagino = pagino;
1187 59 : rec = trec;
1188 59 : goto alloc_inode;
1189 : }
1190 :
1191 : /* free inodes to the right? */
1192 2205 : if (!useleft && rec.ir_freecount) {
1193 1906 : xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR);
1194 :
1195 1905 : pag->pagl_leftrec = trec.ir_startino;
1196 1905 : pag->pagl_rightrec = rec.ir_startino;
1197 1905 : pag->pagl_pagino = pagino;
1198 1905 : goto alloc_inode;
1199 : }
1200 :
1201 : /* get next record to check */
1202 299 : if (useleft) {
1203 105 : error = xfs_ialloc_next_rec(tcur, &trec,
1204 : &doneleft, 1);
1205 : } else {
1206 194 : error = xfs_ialloc_next_rec(cur, &rec,
1207 : &doneright, 0);
1208 : }
1209 299 : if (error)
1210 0 : goto error1;
1211 : }
1212 :
1213 10 : if (searchdistance <= 0) {
1214 : /*
1215 : * Not in range - save last search
1216 : * location and allocate a new inode
1217 : */
1218 1 : xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR);
1219 1 : pag->pagl_leftrec = trec.ir_startino;
1220 1 : pag->pagl_rightrec = rec.ir_startino;
1221 1 : pag->pagl_pagino = pagino;
1222 :
1223 : } else {
1224 : /*
1225 : * We've reached the end of the btree. because
1226 : * we are only searching a small chunk of the
1227 : * btree each search, there is obviously free
1228 : * inodes closer to the parent inode than we
1229 : * are now. restart the search again.
1230 : */
1231 9 : pag->pagl_pagino = NULLAGINO;
1232 9 : pag->pagl_leftrec = NULLAGINO;
1233 9 : pag->pagl_rightrec = NULLAGINO;
1234 9 : xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR);
1235 9 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
1236 9 : goto restart_pagno;
1237 : }
1238 : }
1239 :
1240 : /*
1241 : * In a different AG from the parent.
1242 : * See if the most recently allocated block has any free.
1243 : */
1244 819 : if (agi->agi_newino != cpu_to_be32(NULLAGINO)) {
1245 819 : error = xfs_inobt_lookup(cur, be32_to_cpu(agi->agi_newino),
1246 : XFS_LOOKUP_EQ, &i);
1247 819 : if (error)
1248 0 : goto error0;
1249 :
1250 819 : if (i == 1) {
1251 819 : error = xfs_inobt_get_rec(cur, &rec, &j);
1252 819 : if (error)
1253 0 : goto error0;
1254 :
1255 819 : if (j == 1 && rec.ir_freecount > 0) {
1256 : /*
1257 : * The last chunk allocated in the group
1258 : * still has a free inode.
1259 : */
1260 818 : goto alloc_inode;
1261 : }
1262 : }
1263 : }
1264 :
1265 : /*
1266 : * None left in the last group, search the whole AG
1267 : */
1268 1 : error = xfs_inobt_lookup(cur, 0, XFS_LOOKUP_GE, &i);
1269 1 : if (error)
1270 0 : goto error0;
1271 1 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1272 0 : xfs_btree_mark_sick(cur);
1273 0 : error = -EFSCORRUPTED;
1274 0 : goto error0;
1275 : }
1276 :
1277 1 : for (;;) {
1278 1 : error = xfs_inobt_get_rec(cur, &rec, &i);
1279 1 : if (error)
1280 0 : goto error0;
1281 1 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1282 0 : xfs_btree_mark_sick(cur);
1283 0 : error = -EFSCORRUPTED;
1284 0 : goto error0;
1285 : }
1286 1 : if (rec.ir_freecount > 0)
1287 : break;
1288 1 : error = xfs_btree_increment(cur, 0, &i);
1289 1 : if (error)
1290 0 : goto error0;
1291 1 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1292 1 : xfs_btree_mark_sick(cur);
1293 1 : error = -EFSCORRUPTED;
1294 1 : goto error0;
1295 : }
1296 : }
1297 :
1298 0 : alloc_inode:
1299 7337 : offset = xfs_inobt_first_free_inode(&rec);
1300 7335 : ASSERT(offset >= 0);
1301 7335 : ASSERT(offset < XFS_INODES_PER_CHUNK);
1302 7335 : ASSERT((XFS_AGINO_TO_OFFSET(mp, rec.ir_startino) %
1303 : XFS_INODES_PER_CHUNK) == 0);
1304 7335 : ino = XFS_AGINO_TO_INO(mp, pag->pag_agno, rec.ir_startino + offset);
1305 :
1306 7335 : if (xfs_ag_has_sickness(pag, XFS_SICK_AG_INODES)) {
1307 0 : error = xfs_dialloc_check_ino(pag, tp, ino);
1308 0 : if (error)
1309 0 : goto error0;
1310 : }
1311 :
1312 7337 : rec.ir_free &= ~XFS_INOBT_MASK(offset);
1313 7337 : rec.ir_freecount--;
1314 7337 : error = xfs_inobt_update(cur, &rec);
1315 7336 : if (error)
1316 0 : goto error0;
1317 7336 : be32_add_cpu(&agi->agi_freecount, -1);
1318 7336 : xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT);
1319 7338 : pag->pagi_freecount--;
1320 :
1321 7338 : error = xfs_check_agi_freecount(cur);
1322 7338 : if (error)
1323 0 : goto error0;
1324 :
1325 7338 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
1326 7337 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -1);
1327 7333 : *inop = ino;
1328 7333 : return 0;
1329 : error1:
1330 0 : xfs_btree_del_cursor(tcur, XFS_BTREE_ERROR);
1331 1 : error0:
1332 1 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
1333 1 : return error;
1334 : }
1335 :
1336 : /*
1337 : * Use the free inode btree to allocate an inode based on distance from the
1338 : * parent. Note that the provided cursor may be deleted and replaced.
1339 : */
1340 : STATIC int
1341 112098399 : xfs_dialloc_ag_finobt_near(
1342 : xfs_agino_t pagino,
1343 : struct xfs_btree_cur **ocur,
1344 : struct xfs_inobt_rec_incore *rec)
1345 : {
1346 112098399 : struct xfs_btree_cur *lcur = *ocur; /* left search cursor */
1347 112098399 : struct xfs_btree_cur *rcur; /* right search cursor */
1348 112098399 : struct xfs_inobt_rec_incore rrec;
1349 112098399 : int error;
1350 112098399 : int i, j;
1351 :
1352 112098399 : error = xfs_inobt_lookup(lcur, pagino, XFS_LOOKUP_LE, &i);
1353 112213805 : if (error)
1354 : return error;
1355 :
1356 112213805 : if (i == 1) {
1357 17188291 : error = xfs_inobt_get_rec(lcur, rec, &i);
1358 17190003 : if (error)
1359 : return error;
1360 17190003 : if (XFS_IS_CORRUPT(lcur->bc_mp, i != 1)) {
1361 0 : xfs_btree_mark_sick(lcur);
1362 0 : return -EFSCORRUPTED;
1363 : }
1364 :
1365 : /*
1366 : * See if we've landed in the parent inode record. The finobt
1367 : * only tracks chunks with at least one free inode, so record
1368 : * existence is enough.
1369 : */
1370 17190003 : if (pagino >= rec->ir_startino &&
1371 17189930 : pagino < (rec->ir_startino + XFS_INODES_PER_CHUNK))
1372 : return 0;
1373 : }
1374 :
1375 105286686 : error = xfs_btree_dup_cursor(lcur, &rcur);
1376 104953166 : if (error)
1377 : return error;
1378 :
1379 104913877 : error = xfs_inobt_lookup(rcur, pagino, XFS_LOOKUP_GE, &j);
1380 105398049 : if (error)
1381 0 : goto error_rcur;
1382 105398049 : if (j == 1) {
1383 101566779 : error = xfs_inobt_get_rec(rcur, &rrec, &j);
1384 101676819 : if (error)
1385 0 : goto error_rcur;
1386 101676819 : if (XFS_IS_CORRUPT(lcur->bc_mp, j != 1)) {
1387 0 : xfs_btree_mark_sick(lcur);
1388 0 : error = -EFSCORRUPTED;
1389 0 : goto error_rcur;
1390 : }
1391 : }
1392 :
1393 105508089 : if (XFS_IS_CORRUPT(lcur->bc_mp, i != 1 && j != 1)) {
1394 0 : xfs_btree_mark_sick(lcur);
1395 0 : error = -EFSCORRUPTED;
1396 0 : goto error_rcur;
1397 : }
1398 105508089 : if (i == 1 && j == 1) {
1399 : /*
1400 : * Both the left and right records are valid. Choose the closer
1401 : * inode chunk to the target.
1402 : */
1403 6382784 : if ((pagino - rec->ir_startino + XFS_INODES_PER_CHUNK - 1) >
1404 6382784 : (rrec.ir_startino - pagino)) {
1405 3303831 : *rec = rrec;
1406 3303831 : xfs_btree_del_cursor(lcur, XFS_BTREE_NOERROR);
1407 3303780 : *ocur = rcur;
1408 : } else {
1409 3078953 : xfs_btree_del_cursor(rcur, XFS_BTREE_NOERROR);
1410 : }
1411 99125305 : } else if (j == 1) {
1412 : /* only the right record is valid */
1413 95248567 : *rec = rrec;
1414 95248567 : xfs_btree_del_cursor(lcur, XFS_BTREE_NOERROR);
1415 95247658 : *ocur = rcur;
1416 3876738 : } else if (i == 1) {
1417 : /* only the left record is valid */
1418 3876912 : xfs_btree_del_cursor(rcur, XFS_BTREE_NOERROR);
1419 : }
1420 :
1421 : return 0;
1422 :
1423 0 : error_rcur:
1424 0 : xfs_btree_del_cursor(rcur, XFS_BTREE_ERROR);
1425 0 : return error;
1426 : }
1427 :
1428 : /*
1429 : * Use the free inode btree to find a free inode based on a newino hint. If
1430 : * the hint is NULL, find the first free inode in the AG.
1431 : */
1432 : STATIC int
1433 11501731 : xfs_dialloc_ag_finobt_newino(
1434 : struct xfs_agi *agi,
1435 : struct xfs_btree_cur *cur,
1436 : struct xfs_inobt_rec_incore *rec)
1437 : {
1438 11501731 : int error;
1439 11501731 : int i;
1440 :
1441 11501731 : if (agi->agi_newino != cpu_to_be32(NULLAGINO)) {
1442 11308771 : error = xfs_inobt_lookup(cur, be32_to_cpu(agi->agi_newino),
1443 : XFS_LOOKUP_EQ, &i);
1444 11310257 : if (error)
1445 : return error;
1446 11310257 : if (i == 1) {
1447 11051419 : error = xfs_inobt_get_rec(cur, rec, &i);
1448 11051921 : if (error)
1449 : return error;
1450 11051921 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1451 0 : xfs_btree_mark_sick(cur);
1452 0 : return -EFSCORRUPTED;
1453 : }
1454 : return 0;
1455 : }
1456 : }
1457 :
1458 : /*
1459 : * Find the first inode available in the AG.
1460 : */
1461 451798 : error = xfs_inobt_lookup(cur, 0, XFS_LOOKUP_GE, &i);
1462 452139 : if (error)
1463 : return error;
1464 452139 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1465 0 : xfs_btree_mark_sick(cur);
1466 0 : return -EFSCORRUPTED;
1467 : }
1468 :
1469 452139 : error = xfs_inobt_get_rec(cur, rec, &i);
1470 452147 : if (error)
1471 : return error;
1472 452147 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1473 0 : xfs_btree_mark_sick(cur);
1474 0 : return -EFSCORRUPTED;
1475 : }
1476 :
1477 : return 0;
1478 : }
1479 :
1480 : /*
1481 : * Update the inobt based on a modification made to the finobt. Also ensure that
1482 : * the records from both trees are equivalent post-modification.
1483 : */
1484 : STATIC int
1485 123963980 : xfs_dialloc_ag_update_inobt(
1486 : struct xfs_btree_cur *cur, /* inobt cursor */
1487 : struct xfs_inobt_rec_incore *frec, /* finobt record */
1488 : int offset) /* inode offset */
1489 : {
1490 123963980 : struct xfs_inobt_rec_incore rec;
1491 123963980 : int error;
1492 123963980 : int i;
1493 :
1494 123963980 : error = xfs_inobt_lookup(cur, frec->ir_startino, XFS_LOOKUP_EQ, &i);
1495 124142646 : if (error)
1496 : return error;
1497 124142590 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1498 0 : xfs_btree_mark_sick(cur);
1499 0 : return -EFSCORRUPTED;
1500 : }
1501 :
1502 124142590 : error = xfs_inobt_get_rec(cur, &rec, &i);
1503 124155695 : if (error)
1504 : return error;
1505 124155695 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1506 0 : xfs_btree_mark_sick(cur);
1507 0 : return -EFSCORRUPTED;
1508 : }
1509 124155695 : ASSERT((XFS_AGINO_TO_OFFSET(cur->bc_mp, rec.ir_startino) %
1510 : XFS_INODES_PER_CHUNK) == 0);
1511 :
1512 124155695 : rec.ir_free &= ~XFS_INOBT_MASK(offset);
1513 124155695 : rec.ir_freecount--;
1514 :
1515 124155695 : if (XFS_IS_CORRUPT(cur->bc_mp,
1516 : rec.ir_free != frec->ir_free ||
1517 : rec.ir_freecount != frec->ir_freecount)) {
1518 0 : xfs_btree_mark_sick(cur);
1519 0 : return -EFSCORRUPTED;
1520 : }
1521 :
1522 124155695 : return xfs_inobt_update(cur, &rec);
1523 : }
1524 :
1525 : /*
1526 : * Allocate an inode using the free inode btree, if available. Otherwise, fall
1527 : * back to the inobt search algorithm.
1528 : *
1529 : * The caller selected an AG for us, and made sure that free inodes are
1530 : * available.
1531 : */
1532 : static int
1533 123763452 : xfs_dialloc_ag(
1534 : struct xfs_perag *pag,
1535 : struct xfs_trans *tp,
1536 : struct xfs_buf *agbp,
1537 : xfs_ino_t parent,
1538 : xfs_ino_t *inop)
1539 : {
1540 123763452 : struct xfs_mount *mp = tp->t_mountp;
1541 123763452 : struct xfs_agi *agi = agbp->b_addr;
1542 123763452 : xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent);
1543 123763452 : xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent);
1544 123763452 : struct xfs_btree_cur *cur; /* finobt cursor */
1545 123763452 : struct xfs_btree_cur *icur; /* inobt cursor */
1546 123763452 : struct xfs_inobt_rec_incore rec;
1547 123763452 : xfs_ino_t ino;
1548 123763452 : int error;
1549 123763452 : int offset;
1550 123763452 : int i;
1551 :
1552 123763452 : if (!xfs_has_finobt(mp))
1553 7336 : return xfs_dialloc_ag_inobt(pag, tp, agbp, parent, inop);
1554 :
1555 : /*
1556 : * If pagino is 0 (this is the root inode allocation) use newino.
1557 : * This must work because we've just allocated some.
1558 : */
1559 123756116 : if (!pagino)
1560 3982 : pagino = be32_to_cpu(agi->agi_newino);
1561 :
1562 123756116 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_FINO);
1563 :
1564 123963677 : error = xfs_check_agi_freecount(cur);
1565 123741316 : if (error)
1566 215 : goto error_cur;
1567 :
1568 : /*
1569 : * The search algorithm depends on whether we're in the same AG as the
1570 : * parent. If so, find the closest available inode to the parent. If
1571 : * not, consider the agi hint or find the first free inode in the AG.
1572 : */
1573 123741101 : if (pag->pag_agno == pagno)
1574 112237543 : error = xfs_dialloc_ag_finobt_near(pagino, &cur, &rec);
1575 : else
1576 11503558 : error = xfs_dialloc_ag_finobt_newino(agi, cur, &rec);
1577 123726686 : if (error)
1578 1 : goto error_cur;
1579 :
1580 123726685 : offset = xfs_inobt_first_free_inode(&rec);
1581 123492899 : ASSERT(offset >= 0);
1582 123492899 : ASSERT(offset < XFS_INODES_PER_CHUNK);
1583 123492899 : ASSERT((XFS_AGINO_TO_OFFSET(mp, rec.ir_startino) %
1584 : XFS_INODES_PER_CHUNK) == 0);
1585 123492899 : ino = XFS_AGINO_TO_INO(mp, pag->pag_agno, rec.ir_startino + offset);
1586 :
1587 123492899 : if (xfs_ag_has_sickness(pag, XFS_SICK_AG_INODES)) {
1588 0 : error = xfs_dialloc_check_ino(pag, tp, ino);
1589 0 : if (error)
1590 0 : goto error_cur;
1591 : }
1592 :
1593 : /*
1594 : * Modify or remove the finobt record.
1595 : */
1596 124034424 : rec.ir_free &= ~XFS_INOBT_MASK(offset);
1597 124034424 : rec.ir_freecount--;
1598 124034424 : if (rec.ir_freecount)
1599 102800804 : error = xfs_inobt_update(cur, &rec);
1600 : else
1601 21233620 : error = xfs_btree_delete(cur, &i);
1602 122994700 : if (error)
1603 0 : goto error_cur;
1604 :
1605 : /*
1606 : * The finobt has now been updated appropriately. We haven't updated the
1607 : * agi and superblock yet, so we can create an inobt cursor and validate
1608 : * the original freecount. If all is well, make the equivalent update to
1609 : * the inobt using the finobt record and offset information.
1610 : */
1611 122994700 : icur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_INO);
1612 :
1613 124009142 : error = xfs_check_agi_freecount(icur);
1614 124016467 : if (error)
1615 21 : goto error_icur;
1616 :
1617 124016446 : error = xfs_dialloc_ag_update_inobt(icur, &rec, offset);
1618 123921631 : if (error)
1619 56 : goto error_icur;
1620 :
1621 : /*
1622 : * Both trees have now been updated. We must update the perag and
1623 : * superblock before we can check the freecount for each btree.
1624 : */
1625 123921575 : be32_add_cpu(&agi->agi_freecount, -1);
1626 123921575 : xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT);
1627 124025651 : pag->pagi_freecount--;
1628 :
1629 124025651 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -1);
1630 :
1631 123206103 : error = xfs_check_agi_freecount(icur);
1632 123435695 : if (error)
1633 0 : goto error_icur;
1634 123435695 : error = xfs_check_agi_freecount(cur);
1635 124110426 : if (error)
1636 0 : goto error_icur;
1637 :
1638 124110426 : xfs_btree_del_cursor(icur, XFS_BTREE_NOERROR);
1639 124169609 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
1640 124053146 : *inop = ino;
1641 124053146 : return 0;
1642 :
1643 77 : error_icur:
1644 77 : xfs_btree_del_cursor(icur, XFS_BTREE_ERROR);
1645 293 : error_cur:
1646 293 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
1647 293 : return error;
1648 : }
1649 :
1650 : static int
1651 1469729 : xfs_dialloc_roll(
1652 : struct xfs_trans **tpp,
1653 : struct xfs_buf *agibp)
1654 : {
1655 1469729 : struct xfs_trans *tp = *tpp;
1656 1469729 : struct xfs_dquot_acct *dqinfo;
1657 1469729 : int error;
1658 :
1659 : /*
1660 : * Hold to on to the agibp across the commit so no other allocation can
1661 : * come in and take the free inodes we just allocated for our caller.
1662 : */
1663 1469729 : xfs_trans_bhold(tp, agibp);
1664 :
1665 : /*
1666 : * We want the quota changes to be associated with the next transaction,
1667 : * NOT this one. So, detach the dqinfo from this and attach it to the
1668 : * next transaction.
1669 : */
1670 1469703 : dqinfo = tp->t_dqinfo;
1671 1469703 : tp->t_dqinfo = NULL;
1672 :
1673 1469703 : error = xfs_trans_roll(&tp);
1674 :
1675 : /* Re-attach the quota info that we detached from prev trx. */
1676 1469806 : tp->t_dqinfo = dqinfo;
1677 :
1678 : /*
1679 : * Join the buffer even on commit error so that the buffer is released
1680 : * when the caller cancels the transaction and doesn't have to handle
1681 : * this error case specially.
1682 : */
1683 1469806 : xfs_trans_bjoin(tp, agibp);
1684 1469865 : *tpp = tp;
1685 1469865 : return error;
1686 : }
1687 :
1688 : static bool
1689 229557817 : xfs_dialloc_good_ag(
1690 : struct xfs_perag *pag,
1691 : struct xfs_trans *tp,
1692 : umode_t mode,
1693 : int flags,
1694 : bool ok_alloc)
1695 : {
1696 229557817 : struct xfs_mount *mp = tp->t_mountp;
1697 229557817 : xfs_extlen_t ineed;
1698 229557817 : xfs_extlen_t longest = 0;
1699 229557817 : int needspace;
1700 229557817 : int error;
1701 :
1702 229557817 : if (!pag)
1703 : return false;
1704 459115634 : if (!xfs_perag_allows_inodes(pag))
1705 : return false;
1706 459115634 : if (xfs_perag_prohibits_alloc(pag))
1707 : return false;
1708 :
1709 459115634 : if (!xfs_perag_initialised_agi(pag)) {
1710 382 : error = xfs_ialloc_read_agi(pag, tp, NULL);
1711 382 : if (error)
1712 : return false;
1713 : }
1714 :
1715 229557817 : if (pag->pagi_freecount)
1716 : return true;
1717 106960525 : if (!ok_alloc)
1718 : return false;
1719 :
1720 6808662 : if (!xfs_perag_initialised_agf(pag)) {
1721 75 : error = xfs_alloc_read_agf(pag, tp, flags, NULL);
1722 75 : if (error)
1723 : return false;
1724 : }
1725 :
1726 : /*
1727 : * Check that there is enough free space for the file plus a chunk of
1728 : * inodes if we need to allocate some. If this is the first pass across
1729 : * the AGs, take into account the potential space needed for alignment
1730 : * of inode chunks when checking the longest contiguous free space in
1731 : * the AG - this prevents us from getting ENOSPC because we have free
1732 : * space larger than ialloc_blks but alignment constraints prevent us
1733 : * from using it.
1734 : *
1735 : * If we can't find an AG with space for full alignment slack to be
1736 : * taken into account, we must be near ENOSPC in all AGs. Hence we
1737 : * don't include alignment for the second pass and so if we fail
1738 : * allocation due to alignment issues then it is most likely a real
1739 : * ENOSPC condition.
1740 : *
1741 : * XXX(dgc): this calculation is now bogus thanks to the per-ag
1742 : * reservations that xfs_alloc_fix_freelist() now does via
1743 : * xfs_alloc_space_available(). When the AG fills up, pagf_freeblks will
1744 : * be more than large enough for the check below to succeed, but
1745 : * xfs_alloc_space_available() will fail because of the non-zero
1746 : * metadata reservation and hence we won't actually be able to allocate
1747 : * more inodes in this AG. We do soooo much unnecessary work near ENOSPC
1748 : * because of this.
1749 : */
1750 3404301 : ineed = M_IGEO(mp)->ialloc_min_blks;
1751 3404301 : if (flags && ineed > 1)
1752 3346275 : ineed += M_IGEO(mp)->cluster_align;
1753 3404301 : longest = pag->pagf_longest;
1754 3404301 : if (!longest)
1755 200 : longest = pag->pagf_flcount > 0;
1756 3404301 : needspace = S_ISDIR(mode) || S_ISREG(mode) || S_ISLNK(mode);
1757 :
1758 3404301 : if (pag->pagf_freeblks < needspace + ineed || longest < ineed)
1759 244368 : return false;
1760 : return true;
1761 : }
1762 :
1763 : static int
1764 125734157 : xfs_dialloc_try_ag(
1765 : struct xfs_perag *pag,
1766 : struct xfs_trans **tpp,
1767 : xfs_ino_t parent,
1768 : xfs_ino_t *new_ino,
1769 : bool ok_alloc)
1770 : {
1771 125734157 : struct xfs_buf *agbp;
1772 125734157 : xfs_ino_t ino;
1773 125734157 : int error;
1774 :
1775 : /*
1776 : * Then read in the AGI buffer and recheck with the AGI buffer
1777 : * lock held.
1778 : */
1779 125734157 : error = xfs_ialloc_read_agi(pag, *tpp, &agbp);
1780 125548771 : if (error)
1781 : return error;
1782 :
1783 125548748 : if (!pag->pagi_freecount) {
1784 3153436 : if (!ok_alloc) {
1785 31839 : error = -EAGAIN;
1786 31839 : goto out_release;
1787 : }
1788 :
1789 3121597 : error = xfs_ialloc_ag_alloc(pag, *tpp, agbp);
1790 3121683 : if (error < 0)
1791 1651842 : goto out_release;
1792 :
1793 : /*
1794 : * We successfully allocated space for an inode cluster in this
1795 : * AG. Roll the transaction so that we can allocate one of the
1796 : * new inodes.
1797 : */
1798 1469841 : ASSERT(pag->pagi_freecount > 0);
1799 1469841 : error = xfs_dialloc_roll(tpp, agbp);
1800 1469842 : if (error)
1801 0 : goto out_release;
1802 : }
1803 :
1804 : /* Allocate an inode in the found AG */
1805 123865154 : error = xfs_dialloc_ag(pag, *tpp, agbp, parent, &ino);
1806 123924423 : if (!error)
1807 123847863 : *new_ino = ino;
1808 : return error;
1809 :
1810 1683681 : out_release:
1811 1683681 : xfs_trans_brelse(*tpp, agbp);
1812 1683681 : return error;
1813 : }
1814 :
1815 : /*
1816 : * Pick an AG for the new inode.
1817 : *
1818 : * Directories, symlinks, and regular files frequently allocate at least one
1819 : * block, so factor that potential expansion when we examine whether an AG has
1820 : * enough space for file creation. Try to keep metadata files all in the same
1821 : * AG.
1822 : */
1823 : static inline xfs_agnumber_t
1824 131334144 : xfs_dialloc_pick_ag(
1825 : struct xfs_mount *mp,
1826 : struct xfs_inode *dp,
1827 : umode_t mode)
1828 : {
1829 131334144 : xfs_agnumber_t start_agno;
1830 :
1831 131334144 : if (!dp)
1832 : return 0;
1833 131330047 : if (xfs_is_metadir_inode(dp))
1834 : return 0;
1835 :
1836 131276721 : if (S_ISDIR(mode))
1837 13254253 : return (atomic_inc_return(&mp->m_agirotor) - 1) % mp->m_maxagi;
1838 :
1839 118022468 : start_agno = XFS_INO_TO_AGNO(mp, dp->i_ino);
1840 118022468 : if (start_agno >= mp->m_maxagi)
1841 0 : start_agno = 0;
1842 :
1843 : return start_agno;
1844 : }
1845 :
1846 : /*
1847 : * Allocate an on-disk inode.
1848 : *
1849 : * Mode is used to tell whether the new inode is a directory and hence where to
1850 : * locate it. The on-disk inode that is allocated will be returned in @new_ino
1851 : * on success, otherwise an error will be set to indicate the failure (e.g.
1852 : * -ENOSPC).
1853 : */
1854 : int
1855 131411613 : xfs_dialloc(
1856 : struct xfs_trans **tpp,
1857 : struct xfs_inode *dp,
1858 : umode_t mode,
1859 : xfs_ino_t *new_ino)
1860 : {
1861 131411613 : struct xfs_mount *mp = (*tpp)->t_mountp;
1862 131411613 : struct xfs_perag *pag;
1863 131411613 : struct xfs_ino_geometry *igeo = M_IGEO(mp);
1864 131411613 : xfs_ino_t ino = NULLFSINO;
1865 131411613 : xfs_ino_t parent = dp ? dp->i_ino : 0;
1866 131411613 : xfs_agnumber_t agno;
1867 131411613 : xfs_agnumber_t start_agno;
1868 131411613 : bool ok_alloc = true;
1869 131411613 : bool low_space = false;
1870 131411613 : int flags;
1871 131411613 : int error = 0;
1872 :
1873 131411613 : start_agno = xfs_dialloc_pick_ag(mp, dp, mode);
1874 :
1875 : /*
1876 : * If we have already hit the ceiling of inode blocks then clear
1877 : * ok_alloc so we scan all available agi structures for a free
1878 : * inode.
1879 : *
1880 : * Read rough value of mp->m_icount by percpu_counter_read_positive,
1881 : * which will sacrifice the preciseness but improve the performance.
1882 : */
1883 131297938 : if (igeo->maxicount &&
1884 131307393 : percpu_counter_read_positive(&mp->m_icount) + igeo->ialloc_inos
1885 : > igeo->maxicount) {
1886 7367695 : ok_alloc = false;
1887 : }
1888 :
1889 : /*
1890 : * If we are near to ENOSPC, we want to prefer allocation from AGs that
1891 : * have free inodes in them rather than use up free space allocating new
1892 : * inode chunks. Hence we turn off allocation for the first non-blocking
1893 : * pass through the AGs if we are near ENOSPC to consume free inodes
1894 : * that we can immediately allocate, but then we allow allocation on the
1895 : * second pass if we fail to find an AG with free inodes in it.
1896 : */
1897 131297938 : if (percpu_counter_read_positive(&mp->m_fdblocks) <
1898 131297938 : mp->m_low_space[XFS_LOWSP_1_PCNT]) {
1899 1042028 : ok_alloc = false;
1900 1042028 : low_space = true;
1901 : }
1902 :
1903 : /*
1904 : * Loop until we find an allocation group that either has free inodes
1905 : * or in which we can allocate some inodes. Iterate through the
1906 : * allocation groups upward, wrapping at the end.
1907 : */
1908 131297938 : flags = XFS_ALLOC_FLAG_TRYLOCK;
1909 138690502 : retry:
1910 244120690 : for_each_perag_wrap_at(mp, start_agno, mp->m_maxagi, agno, pag) {
1911 229498654 : if (xfs_dialloc_good_ag(pag, *tpp, mode, flags, ok_alloc)) {
1912 125737153 : error = xfs_dialloc_try_ag(pag, tpp, parent,
1913 : &ino, ok_alloc);
1914 125471268 : if (error != -EAGAIN)
1915 : break;
1916 : error = 0;
1917 : }
1918 :
1919 210860376 : if (xfs_is_shutdown(mp)) {
1920 : error = -EFSCORRUPTED;
1921 : break;
1922 : }
1923 : }
1924 138522520 : if (pag)
1925 123766612 : xfs_perag_rele(pag);
1926 138816727 : if (error)
1927 358 : return error;
1928 138816369 : if (ino == NULLFSINO) {
1929 14741727 : if (flags) {
1930 7392564 : flags = 0;
1931 7392564 : if (low_space)
1932 24666 : ok_alloc = true;
1933 7392564 : goto retry;
1934 : }
1935 : return -ENOSPC;
1936 : }
1937 :
1938 : /*
1939 : * Protect against obviously corrupt allocation btree records. Later
1940 : * xfs_iget checks will catch re-allocation of other active in-memory
1941 : * and on-disk inodes. If we don't catch reallocating the parent inode
1942 : * here we will deadlock in xfs_iget() so we have to do these checks
1943 : * first.
1944 : */
1945 124074642 : if (ino == parent || !xfs_verify_dir_ino(mp, ino)) {
1946 0 : xfs_alert(mp, "Allocated a known in-use inode 0x%llx!", ino);
1947 0 : xfs_agno_mark_sick(mp, XFS_INO_TO_AGNO(mp, ino),
1948 : XFS_SICK_AG_INOBT);
1949 0 : return -EFSCORRUPTED;
1950 : }
1951 :
1952 123527052 : *new_ino = ino;
1953 123527052 : return 0;
1954 : }
1955 :
1956 : /*
1957 : * Free the blocks of an inode chunk. We must consider that the inode chunk
1958 : * might be sparse and only free the regions that are allocated as part of the
1959 : * chunk.
1960 : */
1961 : static int
1962 355842 : xfs_difree_inode_chunk(
1963 : struct xfs_trans *tp,
1964 : xfs_agnumber_t agno,
1965 : struct xfs_inobt_rec_incore *rec)
1966 : {
1967 355842 : struct xfs_mount *mp = tp->t_mountp;
1968 355842 : xfs_agblock_t sagbno = XFS_AGINO_TO_AGBNO(mp,
1969 : rec->ir_startino);
1970 355842 : int startidx, endidx;
1971 355842 : int nextbit;
1972 355842 : xfs_agblock_t agbno;
1973 355842 : int contigblk;
1974 355842 : DECLARE_BITMAP(holemask, XFS_INOBT_HOLEMASK_BITS);
1975 :
1976 355842 : if (!xfs_inobt_issparse(rec->ir_holemask)) {
1977 : /* not sparse, calculate extent info directly */
1978 608816 : return xfs_free_extent_later(tp,
1979 304408 : XFS_AGB_TO_FSB(mp, agno, sagbno),
1980 304408 : M_IGEO(mp)->ialloc_blks, &XFS_RMAP_OINFO_INODES,
1981 : XFS_AG_RESV_NONE, 0);
1982 : }
1983 :
1984 : /* holemask is only 16-bits (fits in an unsigned long) */
1985 51434 : ASSERT(sizeof(rec->ir_holemask) <= sizeof(holemask[0]));
1986 51434 : holemask[0] = rec->ir_holemask;
1987 :
1988 : /*
1989 : * Find contiguous ranges of zeroes (i.e., allocated regions) in the
1990 : * holemask and convert the start/end index of each range to an extent.
1991 : * We start with the start and end index both pointing at the first 0 in
1992 : * the mask.
1993 : */
1994 51434 : startidx = endidx = find_first_zero_bit(holemask,
1995 : XFS_INOBT_HOLEMASK_BITS);
1996 51427 : nextbit = startidx + 1;
1997 462844 : while (startidx < XFS_INOBT_HOLEMASK_BITS) {
1998 411405 : int error;
1999 :
2000 411405 : nextbit = find_next_zero_bit(holemask, XFS_INOBT_HOLEMASK_BITS,
2001 : nextbit);
2002 : /*
2003 : * If the next zero bit is contiguous, update the end index of
2004 : * the current range and continue.
2005 : */
2006 411411 : if (nextbit != XFS_INOBT_HOLEMASK_BITS &&
2007 359978 : nextbit == endidx + 1) {
2008 359978 : endidx = nextbit;
2009 359978 : goto next;
2010 : }
2011 :
2012 : /*
2013 : * nextbit is not contiguous with the current end index. Convert
2014 : * the current start/end to an extent and add it to the free
2015 : * list.
2016 : */
2017 0 : agbno = sagbno + (startidx * XFS_INODES_PER_HOLEMASK_BIT) /
2018 51433 : mp->m_sb.sb_inopblock;
2019 102866 : contigblk = ((endidx - startidx + 1) *
2020 0 : XFS_INODES_PER_HOLEMASK_BIT) /
2021 51433 : mp->m_sb.sb_inopblock;
2022 :
2023 51433 : ASSERT(agbno % mp->m_sb.sb_spino_align == 0);
2024 51433 : ASSERT(contigblk % mp->m_sb.sb_spino_align == 0);
2025 102866 : error = xfs_free_extent_later(tp,
2026 51433 : XFS_AGB_TO_FSB(mp, agno, agbno), contigblk,
2027 : &XFS_RMAP_OINFO_INODES, XFS_AG_RESV_NONE, 0);
2028 51439 : if (error)
2029 0 : return error;
2030 :
2031 : /* reset range to current bit and carry on... */
2032 : startidx = endidx = nextbit;
2033 :
2034 411417 : next:
2035 411417 : nextbit++;
2036 : }
2037 : return 0;
2038 : }
2039 :
2040 : STATIC int
2041 75443212 : xfs_difree_inobt(
2042 : struct xfs_perag *pag,
2043 : struct xfs_trans *tp,
2044 : struct xfs_buf *agbp,
2045 : xfs_agino_t agino,
2046 : struct xfs_icluster *xic,
2047 : struct xfs_inobt_rec_incore *orec)
2048 : {
2049 75443212 : struct xfs_mount *mp = pag->pag_mount;
2050 75443212 : struct xfs_agi *agi = agbp->b_addr;
2051 75443212 : struct xfs_btree_cur *cur;
2052 75443212 : struct xfs_inobt_rec_incore rec;
2053 75443212 : int ilen;
2054 75443212 : int error;
2055 75443212 : int i;
2056 75443212 : int off;
2057 :
2058 75443212 : ASSERT(agi->agi_magicnum == cpu_to_be32(XFS_AGI_MAGIC));
2059 75443212 : ASSERT(XFS_AGINO_TO_AGBNO(mp, agino) < be32_to_cpu(agi->agi_length));
2060 :
2061 : /*
2062 : * Initialize the cursor.
2063 : */
2064 75443212 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_INO);
2065 :
2066 75524987 : error = xfs_check_agi_freecount(cur);
2067 75493331 : if (error)
2068 3 : goto error0;
2069 :
2070 : /*
2071 : * Look for the entry describing this inode.
2072 : */
2073 75493328 : if ((error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &i))) {
2074 13 : xfs_warn(mp, "%s: xfs_inobt_lookup() returned error %d.",
2075 : __func__, error);
2076 13 : goto error0;
2077 : }
2078 75530134 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2079 0 : xfs_btree_mark_sick(cur);
2080 0 : error = -EFSCORRUPTED;
2081 0 : goto error0;
2082 : }
2083 75530134 : error = xfs_inobt_get_rec(cur, &rec, &i);
2084 75494079 : if (error) {
2085 0 : xfs_warn(mp, "%s: xfs_inobt_get_rec() returned error %d.",
2086 : __func__, error);
2087 0 : goto error0;
2088 : }
2089 75494079 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2090 0 : xfs_btree_mark_sick(cur);
2091 0 : error = -EFSCORRUPTED;
2092 0 : goto error0;
2093 : }
2094 : /*
2095 : * Get the offset in the inode chunk.
2096 : */
2097 75494079 : off = agino - rec.ir_startino;
2098 75494079 : ASSERT(off >= 0 && off < XFS_INODES_PER_CHUNK);
2099 75494079 : ASSERT(!(rec.ir_free & XFS_INOBT_MASK(off)));
2100 : /*
2101 : * Mark the inode free & increment the count.
2102 : */
2103 75494079 : rec.ir_free |= XFS_INOBT_MASK(off);
2104 75494079 : rec.ir_freecount++;
2105 :
2106 : /*
2107 : * When an inode chunk is free, it becomes eligible for removal. Don't
2108 : * remove the chunk if the block size is large enough for multiple inode
2109 : * chunks (that might not be free).
2110 : */
2111 75494079 : if (!xfs_has_ikeep(mp) && rec.ir_free == XFS_INOBT_ALL_FREE &&
2112 355866 : mp->m_sb.sb_inopblock <= XFS_INODES_PER_CHUNK) {
2113 355864 : xic->deleted = true;
2114 355864 : xic->first_ino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
2115 : rec.ir_startino);
2116 355864 : xic->alloc = xfs_inobt_irec_to_allocmask(&rec);
2117 :
2118 : /*
2119 : * Remove the inode cluster from the AGI B+Tree, adjust the
2120 : * AGI and Superblock inode counts, and mark the disk space
2121 : * to be freed when the transaction is committed.
2122 : */
2123 355880 : ilen = rec.ir_freecount;
2124 355880 : be32_add_cpu(&agi->agi_count, -ilen);
2125 355880 : be32_add_cpu(&agi->agi_freecount, -(ilen - 1));
2126 355880 : xfs_ialloc_log_agi(tp, agbp, XFS_AGI_COUNT | XFS_AGI_FREECOUNT);
2127 355846 : pag->pagi_freecount -= ilen - 1;
2128 355846 : pag->pagi_count -= ilen;
2129 355846 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_ICOUNT, -ilen);
2130 355787 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -(ilen - 1));
2131 :
2132 355798 : if ((error = xfs_btree_delete(cur, &i))) {
2133 0 : xfs_warn(mp, "%s: xfs_btree_delete returned error %d.",
2134 : __func__, error);
2135 0 : goto error0;
2136 : }
2137 :
2138 355830 : error = xfs_difree_inode_chunk(tp, pag->pag_agno, &rec);
2139 355827 : if (error)
2140 0 : goto error0;
2141 : } else {
2142 75138215 : xic->deleted = false;
2143 :
2144 75138215 : error = xfs_inobt_update(cur, &rec);
2145 75076459 : if (error) {
2146 0 : xfs_warn(mp, "%s: xfs_inobt_update returned error %d.",
2147 : __func__, error);
2148 0 : goto error0;
2149 : }
2150 :
2151 : /*
2152 : * Change the inode free counts and log the ag/sb changes.
2153 : */
2154 75076459 : be32_add_cpu(&agi->agi_freecount, 1);
2155 75076459 : xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT);
2156 75140335 : pag->pagi_freecount++;
2157 75140335 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, 1);
2158 : }
2159 :
2160 75332220 : error = xfs_check_agi_freecount(cur);
2161 75389770 : if (error)
2162 0 : goto error0;
2163 :
2164 75389770 : *orec = rec;
2165 75389770 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
2166 75389770 : return 0;
2167 :
2168 16 : error0:
2169 16 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
2170 16 : return error;
2171 : }
2172 :
2173 : /*
2174 : * Free an inode in the free inode btree.
2175 : */
2176 : STATIC int
2177 75461010 : xfs_difree_finobt(
2178 : struct xfs_perag *pag,
2179 : struct xfs_trans *tp,
2180 : struct xfs_buf *agbp,
2181 : xfs_agino_t agino,
2182 : struct xfs_inobt_rec_incore *ibtrec) /* inobt record */
2183 : {
2184 75461010 : struct xfs_mount *mp = pag->pag_mount;
2185 75461010 : struct xfs_btree_cur *cur;
2186 75461010 : struct xfs_inobt_rec_incore rec;
2187 75461010 : int offset = agino - ibtrec->ir_startino;
2188 75461010 : int error;
2189 75461010 : int i;
2190 :
2191 75461010 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_FINO);
2192 :
2193 75546176 : error = xfs_inobt_lookup(cur, ibtrec->ir_startino, XFS_LOOKUP_EQ, &i);
2194 75541735 : if (error)
2195 6 : goto error;
2196 75541729 : if (i == 0) {
2197 : /*
2198 : * If the record does not exist in the finobt, we must have just
2199 : * freed an inode in a previously fully allocated chunk. If not,
2200 : * something is out of sync.
2201 : */
2202 20145373 : if (XFS_IS_CORRUPT(mp, ibtrec->ir_freecount != 1)) {
2203 0 : xfs_btree_mark_sick(cur);
2204 0 : error = -EFSCORRUPTED;
2205 0 : goto error;
2206 : }
2207 :
2208 20145373 : error = xfs_inobt_insert_rec(cur, ibtrec->ir_holemask,
2209 20145373 : ibtrec->ir_count,
2210 : ibtrec->ir_freecount,
2211 : ibtrec->ir_free, &i);
2212 20124816 : if (error)
2213 0 : goto error;
2214 20124816 : ASSERT(i == 1);
2215 :
2216 20124816 : goto out;
2217 : }
2218 :
2219 : /*
2220 : * Read and update the existing record. We could just copy the ibtrec
2221 : * across here, but that would defeat the purpose of having redundant
2222 : * metadata. By making the modifications independently, we can catch
2223 : * corruptions that we wouldn't see if we just copied from one record
2224 : * to another.
2225 : */
2226 55396356 : error = xfs_inobt_get_rec(cur, &rec, &i);
2227 55358130 : if (error)
2228 0 : goto error;
2229 55358130 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2230 0 : xfs_btree_mark_sick(cur);
2231 0 : error = -EFSCORRUPTED;
2232 0 : goto error;
2233 : }
2234 :
2235 55358130 : rec.ir_free |= XFS_INOBT_MASK(offset);
2236 55358130 : rec.ir_freecount++;
2237 :
2238 55358130 : if (XFS_IS_CORRUPT(mp,
2239 : rec.ir_free != ibtrec->ir_free ||
2240 : rec.ir_freecount != ibtrec->ir_freecount)) {
2241 0 : xfs_btree_mark_sick(cur);
2242 0 : error = -EFSCORRUPTED;
2243 0 : goto error;
2244 : }
2245 :
2246 : /*
2247 : * The content of inobt records should always match between the inobt
2248 : * and finobt. The lifecycle of records in the finobt is different from
2249 : * the inobt in that the finobt only tracks records with at least one
2250 : * free inode. Hence, if all of the inodes are free and we aren't
2251 : * keeping inode chunks permanently on disk, remove the record.
2252 : * Otherwise, update the record with the new information.
2253 : *
2254 : * Note that we currently can't free chunks when the block size is large
2255 : * enough for multiple chunks. Leave the finobt record to remain in sync
2256 : * with the inobt.
2257 : */
2258 55358130 : if (!xfs_has_ikeep(mp) && rec.ir_free == XFS_INOBT_ALL_FREE &&
2259 355848 : mp->m_sb.sb_inopblock <= XFS_INODES_PER_CHUNK) {
2260 355848 : error = xfs_btree_delete(cur, &i);
2261 355873 : if (error)
2262 0 : goto error;
2263 355873 : ASSERT(i == 1);
2264 : } else {
2265 55002282 : error = xfs_inobt_update(cur, &rec);
2266 55016734 : if (error)
2267 0 : goto error;
2268 : }
2269 :
2270 55016734 : out:
2271 75497423 : error = xfs_check_agi_freecount(cur);
2272 75549226 : if (error)
2273 0 : goto error;
2274 :
2275 75549226 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
2276 75549226 : return 0;
2277 :
2278 6 : error:
2279 6 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
2280 6 : return error;
2281 : }
2282 :
2283 : /*
2284 : * Free disk inode. Carefully avoids touching the incore inode, all
2285 : * manipulations incore are the caller's responsibility.
2286 : * The on-disk inode is not changed by this operation, only the
2287 : * btree (free inode mask) is changed.
2288 : */
2289 : int
2290 75383232 : xfs_difree(
2291 : struct xfs_trans *tp,
2292 : struct xfs_perag *pag,
2293 : xfs_ino_t inode,
2294 : struct xfs_icluster *xic)
2295 : {
2296 : /* REFERENCED */
2297 75383232 : xfs_agblock_t agbno; /* block number containing inode */
2298 75383232 : struct xfs_buf *agbp; /* buffer for allocation group header */
2299 75383232 : xfs_agino_t agino; /* allocation group inode number */
2300 75383232 : int error; /* error return value */
2301 75383232 : struct xfs_mount *mp = tp->t_mountp;
2302 75383232 : struct xfs_inobt_rec_incore rec;/* btree record */
2303 :
2304 : /*
2305 : * Break up inode number into its components.
2306 : */
2307 75383232 : if (pag->pag_agno != XFS_INO_TO_AGNO(mp, inode)) {
2308 0 : xfs_warn(mp, "%s: agno != pag->pag_agno (%d != %d).",
2309 : __func__, XFS_INO_TO_AGNO(mp, inode), pag->pag_agno);
2310 0 : ASSERT(0);
2311 0 : return -EINVAL;
2312 : }
2313 75383232 : agino = XFS_INO_TO_AGINO(mp, inode);
2314 75383232 : if (inode != XFS_AGINO_TO_INO(mp, pag->pag_agno, agino)) {
2315 0 : xfs_warn(mp, "%s: inode != XFS_AGINO_TO_INO() (%llu != %llu).",
2316 : __func__, (unsigned long long)inode,
2317 : (unsigned long long)XFS_AGINO_TO_INO(mp, pag->pag_agno, agino));
2318 0 : ASSERT(0);
2319 0 : return -EINVAL;
2320 : }
2321 75383232 : agbno = XFS_AGINO_TO_AGBNO(mp, agino);
2322 75383232 : if (agbno >= mp->m_sb.sb_agblocks) {
2323 0 : xfs_warn(mp, "%s: agbno >= mp->m_sb.sb_agblocks (%d >= %d).",
2324 : __func__, agbno, mp->m_sb.sb_agblocks);
2325 0 : ASSERT(0);
2326 0 : return -EINVAL;
2327 : }
2328 : /*
2329 : * Get the allocation group header.
2330 : */
2331 75383232 : error = xfs_ialloc_read_agi(pag, tp, &agbp);
2332 75470471 : if (error) {
2333 99 : xfs_warn(mp, "%s: xfs_ialloc_read_agi() returned error %d.",
2334 : __func__, error);
2335 99 : return error;
2336 : }
2337 :
2338 : /*
2339 : * Fix up the inode allocation btree.
2340 : */
2341 75470372 : error = xfs_difree_inobt(pag, tp, agbp, agino, xic, &rec);
2342 75538575 : if (error)
2343 16 : goto error0;
2344 :
2345 : /*
2346 : * Fix up the free inode btree.
2347 : */
2348 75538559 : if (xfs_has_finobt(mp)) {
2349 75525285 : error = xfs_difree_finobt(pag, tp, agbp, agino, &rec);
2350 75585989 : if (error)
2351 6 : goto error0;
2352 : }
2353 :
2354 : return 0;
2355 :
2356 : error0:
2357 : return error;
2358 : }
2359 :
2360 : STATIC int
2361 364219600 : xfs_imap_lookup(
2362 : struct xfs_perag *pag,
2363 : struct xfs_trans *tp,
2364 : xfs_agino_t agino,
2365 : xfs_agblock_t agbno,
2366 : xfs_agblock_t *chunk_agbno,
2367 : xfs_agblock_t *offset_agbno,
2368 : int flags)
2369 : {
2370 364219600 : struct xfs_mount *mp = pag->pag_mount;
2371 364219600 : struct xfs_inobt_rec_incore rec;
2372 364219600 : struct xfs_btree_cur *cur;
2373 364219600 : struct xfs_buf *agbp;
2374 364219600 : int error;
2375 364219600 : int i;
2376 :
2377 364219600 : error = xfs_ialloc_read_agi(pag, tp, &agbp);
2378 364254997 : if (error) {
2379 182 : xfs_alert(mp,
2380 : "%s: xfs_ialloc_read_agi() returned error %d, agno %d",
2381 : __func__, error, pag->pag_agno);
2382 182 : return error;
2383 : }
2384 :
2385 : /*
2386 : * Lookup the inode record for the given agino. If the record cannot be
2387 : * found, then it's an invalid inode number and we should abort. Once
2388 : * we have a record, we need to ensure it contains the inode number
2389 : * we are looking up.
2390 : */
2391 364254815 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_INO);
2392 364258519 : error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &i);
2393 364258910 : if (!error) {
2394 364258265 : if (i)
2395 364256293 : error = xfs_inobt_get_rec(cur, &rec, &i);
2396 364240558 : if (!error && i == 0)
2397 1221 : error = -EINVAL;
2398 : }
2399 :
2400 364241203 : xfs_trans_brelse(tp, agbp);
2401 364251239 : xfs_btree_del_cursor(cur, error);
2402 364264638 : if (error)
2403 : return error;
2404 :
2405 : /* check that the returned record contains the required inode */
2406 364263404 : if (rec.ir_startino > agino ||
2407 364263404 : rec.ir_startino + M_IGEO(mp)->ialloc_inos <= agino)
2408 : return -EINVAL;
2409 :
2410 : /* for untrusted inodes check it is allocated first */
2411 364174749 : if ((flags & XFS_IGET_UNTRUSTED) &&
2412 364174771 : (rec.ir_free & XFS_INOBT_MASK(agino - rec.ir_startino)))
2413 : return -EINVAL;
2414 :
2415 364172016 : *chunk_agbno = XFS_AGINO_TO_AGBNO(mp, rec.ir_startino);
2416 364172016 : *offset_agbno = agbno - *chunk_agbno;
2417 364172016 : return 0;
2418 : }
2419 :
2420 : /*
2421 : * Return the location of the inode in imap, for mapping it into a buffer.
2422 : */
2423 : int
2424 445301741 : xfs_imap(
2425 : struct xfs_perag *pag,
2426 : struct xfs_trans *tp,
2427 : xfs_ino_t ino, /* inode to locate */
2428 : struct xfs_imap *imap, /* location map structure */
2429 : uint flags) /* flags for inode btree lookup */
2430 : {
2431 445301741 : struct xfs_mount *mp = pag->pag_mount;
2432 445301741 : xfs_agblock_t agbno; /* block number of inode in the alloc group */
2433 445301741 : xfs_agino_t agino; /* inode number within alloc group */
2434 445301741 : xfs_agblock_t chunk_agbno; /* first block in inode chunk */
2435 445301741 : xfs_agblock_t cluster_agbno; /* first block in inode cluster */
2436 445301741 : int error; /* error code */
2437 445301741 : int offset; /* index of inode in its buffer */
2438 445301741 : xfs_agblock_t offset_agbno; /* blks from chunk start to inode */
2439 :
2440 445301741 : ASSERT(ino != NULLFSINO);
2441 :
2442 : /*
2443 : * Split up the inode number into its parts.
2444 : */
2445 445301741 : agino = XFS_INO_TO_AGINO(mp, ino);
2446 445301741 : agbno = XFS_AGINO_TO_AGBNO(mp, agino);
2447 445301741 : if (agbno >= mp->m_sb.sb_agblocks ||
2448 445620946 : ino != XFS_AGINO_TO_INO(mp, pag->pag_agno, agino)) {
2449 32654 : error = -EINVAL;
2450 : #ifdef DEBUG
2451 : /*
2452 : * Don't output diagnostic information for untrusted inodes
2453 : * as they can be invalid without implying corruption.
2454 : */
2455 32654 : if (flags & XFS_IGET_UNTRUSTED)
2456 : return error;
2457 0 : if (agbno >= mp->m_sb.sb_agblocks) {
2458 0 : xfs_alert(mp,
2459 : "%s: agbno (0x%llx) >= mp->m_sb.sb_agblocks (0x%lx)",
2460 : __func__, (unsigned long long)agbno,
2461 : (unsigned long)mp->m_sb.sb_agblocks);
2462 : }
2463 0 : if (ino != XFS_AGINO_TO_INO(mp, pag->pag_agno, agino)) {
2464 0 : xfs_alert(mp,
2465 : "%s: ino (0x%llx) != XFS_AGINO_TO_INO() (0x%llx)",
2466 : __func__, ino,
2467 : XFS_AGINO_TO_INO(mp, pag->pag_agno, agino));
2468 : }
2469 0 : xfs_stack_trace();
2470 : #endif /* DEBUG */
2471 0 : return error;
2472 : }
2473 :
2474 : /*
2475 : * For bulkstat and handle lookups, we have an untrusted inode number
2476 : * that we have to verify is valid. We cannot do this just by reading
2477 : * the inode buffer as it may have been unlinked and removed leaving
2478 : * inodes in stale state on disk. Hence we have to do a btree lookup
2479 : * in all cases where an untrusted inode number is passed.
2480 : */
2481 445269087 : if (flags & XFS_IGET_UNTRUSTED) {
2482 364214049 : error = xfs_imap_lookup(pag, tp, agino, agbno,
2483 : &chunk_agbno, &offset_agbno, flags);
2484 364263851 : if (error)
2485 : return error;
2486 364170682 : goto out_map;
2487 : }
2488 :
2489 : /*
2490 : * If the inode cluster size is the same as the blocksize or
2491 : * smaller we get to the buffer by simple arithmetics.
2492 : */
2493 81055038 : if (M_IGEO(mp)->blocks_per_cluster == 1) {
2494 0 : offset = XFS_INO_TO_OFFSET(mp, ino);
2495 0 : ASSERT(offset < mp->m_sb.sb_inopblock);
2496 :
2497 0 : imap->im_blkno = XFS_AGB_TO_DADDR(mp, pag->pag_agno, agbno);
2498 0 : imap->im_len = XFS_FSB_TO_BB(mp, 1);
2499 0 : imap->im_boffset = (unsigned short)(offset <<
2500 0 : mp->m_sb.sb_inodelog);
2501 0 : return 0;
2502 : }
2503 :
2504 : /*
2505 : * If the inode chunks are aligned then use simple maths to
2506 : * find the location. Otherwise we have to do a btree
2507 : * lookup to find the location.
2508 : */
2509 81055038 : if (M_IGEO(mp)->inoalign_mask) {
2510 81055038 : offset_agbno = agbno & M_IGEO(mp)->inoalign_mask;
2511 81055038 : chunk_agbno = agbno - offset_agbno;
2512 : } else {
2513 0 : error = xfs_imap_lookup(pag, tp, agino, agbno,
2514 : &chunk_agbno, &offset_agbno, flags);
2515 0 : if (error)
2516 : return error;
2517 : }
2518 :
2519 0 : out_map:
2520 445225720 : ASSERT(agbno >= chunk_agbno);
2521 890451440 : cluster_agbno = chunk_agbno +
2522 445225720 : ((offset_agbno / M_IGEO(mp)->blocks_per_cluster) *
2523 445225720 : M_IGEO(mp)->blocks_per_cluster);
2524 890451440 : offset = ((agbno - cluster_agbno) * mp->m_sb.sb_inopblock) +
2525 445225720 : XFS_INO_TO_OFFSET(mp, ino);
2526 :
2527 445225720 : imap->im_blkno = XFS_AGB_TO_DADDR(mp, pag->pag_agno, cluster_agbno);
2528 445225720 : imap->im_len = XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster);
2529 445225720 : imap->im_boffset = (unsigned short)(offset << mp->m_sb.sb_inodelog);
2530 :
2531 : /*
2532 : * If the inode number maps to a block outside the bounds
2533 : * of the file system then return NULL rather than calling
2534 : * read_buf and panicing when we get an error from the
2535 : * driver.
2536 : */
2537 890451440 : if ((imap->im_blkno + imap->im_len) >
2538 445225720 : XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks)) {
2539 0 : xfs_alert(mp,
2540 : "%s: (im_blkno (0x%llx) + im_len (0x%llx)) > sb_dblocks (0x%llx)",
2541 : __func__, (unsigned long long) imap->im_blkno,
2542 : (unsigned long long) imap->im_len,
2543 : XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks));
2544 0 : return -EINVAL;
2545 : }
2546 : return 0;
2547 : }
2548 :
2549 : /*
2550 : * Log specified fields for the ag hdr (inode section). The growth of the agi
2551 : * structure over time requires that we interpret the buffer as two logical
2552 : * regions delineated by the end of the unlinked list. This is due to the size
2553 : * of the hash table and its location in the middle of the agi.
2554 : *
2555 : * For example, a request to log a field before agi_unlinked and a field after
2556 : * agi_unlinked could cause us to log the entire hash table and use an excessive
2557 : * amount of log space. To avoid this behavior, log the region up through
2558 : * agi_unlinked in one call and the region after agi_unlinked through the end of
2559 : * the structure in another.
2560 : */
2561 : void
2562 202702152 : xfs_ialloc_log_agi(
2563 : struct xfs_trans *tp,
2564 : struct xfs_buf *bp,
2565 : uint32_t fields)
2566 : {
2567 202702152 : int first; /* first byte number */
2568 202702152 : int last; /* last byte number */
2569 202702152 : static const short offsets[] = { /* field starting offsets */
2570 : /* keep in sync with bit definitions */
2571 : offsetof(xfs_agi_t, agi_magicnum),
2572 : offsetof(xfs_agi_t, agi_versionnum),
2573 : offsetof(xfs_agi_t, agi_seqno),
2574 : offsetof(xfs_agi_t, agi_length),
2575 : offsetof(xfs_agi_t, agi_count),
2576 : offsetof(xfs_agi_t, agi_root),
2577 : offsetof(xfs_agi_t, agi_level),
2578 : offsetof(xfs_agi_t, agi_freecount),
2579 : offsetof(xfs_agi_t, agi_newino),
2580 : offsetof(xfs_agi_t, agi_dirino),
2581 : offsetof(xfs_agi_t, agi_unlinked),
2582 : offsetof(xfs_agi_t, agi_free_root),
2583 : offsetof(xfs_agi_t, agi_free_level),
2584 : offsetof(xfs_agi_t, agi_iblocks),
2585 : sizeof(xfs_agi_t)
2586 : };
2587 : #ifdef DEBUG
2588 202702152 : struct xfs_agi *agi = bp->b_addr;
2589 :
2590 202702152 : ASSERT(agi->agi_magicnum == cpu_to_be32(XFS_AGI_MAGIC));
2591 : #endif
2592 :
2593 : /*
2594 : * Compute byte offsets for the first and last fields in the first
2595 : * region and log the agi buffer. This only logs up through
2596 : * agi_unlinked.
2597 : */
2598 202702152 : if (fields & XFS_AGI_ALL_BITS_R1) {
2599 202488002 : xfs_btree_offsets(fields, offsets, XFS_AGI_NUM_BITS_R1,
2600 : &first, &last);
2601 202419035 : xfs_trans_log_buf(tp, bp, first, last);
2602 : }
2603 :
2604 : /*
2605 : * Mask off the bits in the first region and calculate the first and
2606 : * last field offsets for any bits in the second region.
2607 : */
2608 203123610 : fields &= ~XFS_AGI_ALL_BITS_R1;
2609 203123610 : if (fields) {
2610 299208 : xfs_btree_offsets(fields, offsets, XFS_AGI_NUM_BITS_R2,
2611 : &first, &last);
2612 299138 : xfs_trans_log_buf(tp, bp, first, last);
2613 : }
2614 203123630 : }
2615 :
2616 : static xfs_failaddr_t
2617 2686475 : xfs_agi_verify(
2618 : struct xfs_buf *bp)
2619 : {
2620 2686475 : struct xfs_mount *mp = bp->b_mount;
2621 2686475 : struct xfs_agi *agi = bp->b_addr;
2622 2686475 : xfs_failaddr_t fa;
2623 2686475 : uint32_t agi_seqno = be32_to_cpu(agi->agi_seqno);
2624 2686475 : uint32_t agi_length = be32_to_cpu(agi->agi_length);
2625 2686475 : int i;
2626 :
2627 2686475 : if (xfs_has_crc(mp)) {
2628 2671528 : if (!uuid_equal(&agi->agi_uuid, &mp->m_sb.sb_meta_uuid))
2629 0 : return __this_address;
2630 2672105 : if (!xfs_log_check_lsn(mp, be64_to_cpu(agi->agi_lsn)))
2631 0 : return __this_address;
2632 : }
2633 :
2634 : /*
2635 : * Validate the magic number of the agi block.
2636 : */
2637 2687178 : if (!xfs_verify_magic(bp, agi->agi_magicnum))
2638 0 : return __this_address;
2639 2686527 : if (!XFS_AGI_GOOD_VERSION(be32_to_cpu(agi->agi_versionnum)))
2640 0 : return __this_address;
2641 :
2642 2686527 : fa = xfs_validate_ag_length(bp, agi_seqno, agi_length);
2643 2686252 : if (fa)
2644 : return fa;
2645 :
2646 2686341 : if (be32_to_cpu(agi->agi_level) < 1 ||
2647 2686341 : be32_to_cpu(agi->agi_level) > M_IGEO(mp)->inobt_maxlevels)
2648 0 : return __this_address;
2649 :
2650 2686341 : if (xfs_has_finobt(mp) &&
2651 2671791 : (be32_to_cpu(agi->agi_free_level) < 1 ||
2652 : be32_to_cpu(agi->agi_free_level) > M_IGEO(mp)->inobt_maxlevels))
2653 0 : return __this_address;
2654 :
2655 174538924 : for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) {
2656 171852005 : if (agi->agi_unlinked[i] == cpu_to_be32(NULLAGINO))
2657 171642029 : continue;
2658 210554 : if (!xfs_verify_ino(mp, be32_to_cpu(agi->agi_unlinked[i])))
2659 0 : return __this_address;
2660 : }
2661 :
2662 : return NULL;
2663 : }
2664 :
2665 : static void
2666 968945 : xfs_agi_read_verify(
2667 : struct xfs_buf *bp)
2668 : {
2669 968945 : struct xfs_mount *mp = bp->b_mount;
2670 968945 : xfs_failaddr_t fa;
2671 :
2672 1937246 : if (xfs_has_crc(mp) &&
2673 : !xfs_buf_verify_cksum(bp, XFS_AGI_CRC_OFF))
2674 10 : xfs_verifier_error(bp, -EFSBADCRC, __this_address);
2675 : else {
2676 968935 : fa = xfs_agi_verify(bp);
2677 968935 : if (XFS_TEST_ERROR(fa, mp, XFS_ERRTAG_IALLOC_READ_AGI))
2678 0 : xfs_verifier_error(bp, -EFSCORRUPTED, fa);
2679 : }
2680 968945 : }
2681 :
2682 : static void
2683 806813 : xfs_agi_write_verify(
2684 : struct xfs_buf *bp)
2685 : {
2686 806813 : struct xfs_mount *mp = bp->b_mount;
2687 806813 : struct xfs_buf_log_item *bip = bp->b_log_item;
2688 806813 : struct xfs_agi *agi = bp->b_addr;
2689 806813 : xfs_failaddr_t fa;
2690 :
2691 806813 : fa = xfs_agi_verify(bp);
2692 806813 : if (fa) {
2693 0 : xfs_verifier_error(bp, -EFSCORRUPTED, fa);
2694 0 : return;
2695 : }
2696 :
2697 806813 : if (!xfs_has_crc(mp))
2698 : return;
2699 :
2700 793141 : if (bip)
2701 765617 : agi->agi_lsn = cpu_to_be64(bip->bli_item.li_lsn);
2702 793141 : xfs_buf_update_cksum(bp, XFS_AGI_CRC_OFF);
2703 : }
2704 :
2705 : const struct xfs_buf_ops xfs_agi_buf_ops = {
2706 : .name = "xfs_agi",
2707 : .magic = { cpu_to_be32(XFS_AGI_MAGIC), cpu_to_be32(XFS_AGI_MAGIC) },
2708 : .verify_read = xfs_agi_read_verify,
2709 : .verify_write = xfs_agi_write_verify,
2710 : .verify_struct = xfs_agi_verify,
2711 : };
2712 :
2713 : /*
2714 : * Read in the allocation group header (inode allocation section)
2715 : */
2716 : int
2717 3284691859 : xfs_read_agi(
2718 : struct xfs_perag *pag,
2719 : struct xfs_trans *tp,
2720 : struct xfs_buf **agibpp)
2721 : {
2722 3284691859 : struct xfs_mount *mp = pag->pag_mount;
2723 3284691859 : int error;
2724 :
2725 3284691859 : trace_xfs_read_agi(pag->pag_mount, pag->pag_agno);
2726 :
2727 13130952748 : error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp,
2728 3282738187 : XFS_AG_DADDR(mp, pag->pag_agno, XFS_AGI_DADDR(mp)),
2729 3282738187 : XFS_FSS_TO_BB(mp, 1), 0, agibpp, &xfs_agi_buf_ops);
2730 3284912187 : if (xfs_metadata_is_sick(error))
2731 10 : xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI);
2732 3284912187 : if (error)
2733 : return error;
2734 3284905719 : if (tp)
2735 3278749649 : xfs_trans_buf_set_type(tp, *agibpp, XFS_BLFT_AGI_BUF);
2736 :
2737 3284181802 : xfs_buf_set_ref(*agibpp, XFS_AGI_REF);
2738 3284181802 : return 0;
2739 : }
2740 :
2741 : /*
2742 : * Read in the agi and initialise the per-ag data. If the caller supplies a
2743 : * @agibpp, return the locked AGI buffer to them, otherwise release it.
2744 : */
2745 : int
2746 3122086408 : xfs_ialloc_read_agi(
2747 : struct xfs_perag *pag,
2748 : struct xfs_trans *tp,
2749 : struct xfs_buf **agibpp)
2750 : {
2751 3122086408 : struct xfs_buf *agibp;
2752 3122086408 : struct xfs_agi *agi;
2753 3122086408 : int error;
2754 :
2755 3122086408 : trace_xfs_ialloc_read_agi(pag->pag_mount, pag->pag_agno);
2756 :
2757 3121050220 : error = xfs_read_agi(pag, tp, &agibp);
2758 3122344078 : if (error)
2759 : return error;
2760 :
2761 3122337682 : agi = agibp->b_addr;
2762 6244675364 : if (!xfs_perag_initialised_agi(pag)) {
2763 658716 : pag->pagi_freecount = be32_to_cpu(agi->agi_freecount);
2764 658716 : pag->pagi_count = be32_to_cpu(agi->agi_count);
2765 658716 : set_bit(XFS_AGSTATE_AGI_INIT, &pag->pag_opstate);
2766 : }
2767 :
2768 : /*
2769 : * It's possible for these to be out of sync if
2770 : * we are in the middle of a forced shutdown.
2771 : */
2772 3122337791 : ASSERT(pag->pagi_freecount == be32_to_cpu(agi->agi_freecount) ||
2773 : xfs_is_shutdown(pag->pag_mount));
2774 3122337791 : if (agibpp)
2775 3122278939 : *agibpp = agibp;
2776 : else
2777 58852 : xfs_trans_brelse(tp, agibp);
2778 : return 0;
2779 : }
2780 :
2781 : /* How many inodes are backed by inode clusters ondisk? */
2782 : STATIC int
2783 4199492615 : xfs_ialloc_count_ondisk(
2784 : struct xfs_btree_cur *cur,
2785 : xfs_agino_t low,
2786 : xfs_agino_t high,
2787 : unsigned int *allocated)
2788 : {
2789 4199492615 : struct xfs_inobt_rec_incore irec;
2790 4199492615 : unsigned int ret = 0;
2791 4199492615 : int has_record;
2792 4199492615 : int error;
2793 :
2794 4199492615 : error = xfs_inobt_lookup(cur, low, XFS_LOOKUP_LE, &has_record);
2795 4197376977 : if (error)
2796 : return error;
2797 :
2798 7497739354 : while (has_record) {
2799 4670920960 : unsigned int i, hole_idx;
2800 :
2801 4670920960 : error = xfs_inobt_get_rec(cur, &irec, &has_record);
2802 4671986039 : if (error)
2803 0 : return error;
2804 4671986039 : if (irec.ir_startino > high)
2805 : break;
2806 :
2807 >21260*10^7 : for (i = 0; i < XFS_INODES_PER_CHUNK; i++) {
2808 >20935*10^7 : if (irec.ir_startino + i < low)
2809 >20794*10^7 : continue;
2810 1409958501 : if (irec.ir_startino + i > high)
2811 : break;
2812 :
2813 1361240496 : hole_idx = i / XFS_INODES_PER_HOLEMASK_BIT;
2814 1361240496 : if (!(irec.ir_holemask & (1U << hole_idx)))
2815 894296828 : ret++;
2816 : }
2817 :
2818 3300403952 : error = xfs_btree_increment(cur, 0, &has_record);
2819 3300362377 : if (error)
2820 0 : return error;
2821 : }
2822 :
2823 4198400481 : *allocated = ret;
2824 4198400481 : return 0;
2825 : }
2826 :
2827 : /* Is there an inode record covering a given extent? */
2828 : int
2829 4198425002 : xfs_ialloc_has_inodes_at_extent(
2830 : struct xfs_btree_cur *cur,
2831 : xfs_agblock_t bno,
2832 : xfs_extlen_t len,
2833 : enum xbtree_recpacking *outcome)
2834 : {
2835 4198425002 : xfs_agino_t agino;
2836 4198425002 : xfs_agino_t last_agino;
2837 4198425002 : unsigned int allocated;
2838 4198425002 : int error;
2839 :
2840 4198425002 : agino = XFS_AGB_TO_AGINO(cur->bc_mp, bno);
2841 4198425002 : last_agino = XFS_AGB_TO_AGINO(cur->bc_mp, bno + len) - 1;
2842 :
2843 4198425002 : error = xfs_ialloc_count_ondisk(cur, agino, last_agino, &allocated);
2844 4198315088 : if (error)
2845 : return error;
2846 :
2847 4198315088 : if (allocated == 0)
2848 4180873467 : *outcome = XBTREE_RECPACKING_EMPTY;
2849 17441621 : else if (allocated == last_agino - agino + 1)
2850 17441621 : *outcome = XBTREE_RECPACKING_FULL;
2851 : else
2852 0 : *outcome = XBTREE_RECPACKING_SPARSE;
2853 : return 0;
2854 : }
2855 :
2856 : struct xfs_ialloc_count_inodes {
2857 : xfs_agino_t count;
2858 : xfs_agino_t freecount;
2859 : };
2860 :
2861 : /* Record inode counts across all inobt records. */
2862 : STATIC int
2863 258912963 : xfs_ialloc_count_inodes_rec(
2864 : struct xfs_btree_cur *cur,
2865 : const union xfs_btree_rec *rec,
2866 : void *priv)
2867 : {
2868 258912963 : struct xfs_inobt_rec_incore irec;
2869 258912963 : struct xfs_ialloc_count_inodes *ci = priv;
2870 258912963 : xfs_failaddr_t fa;
2871 :
2872 258912963 : xfs_inobt_btrec_to_irec(cur->bc_mp, rec, &irec);
2873 258915639 : fa = xfs_inobt_check_irec(cur, &irec);
2874 258921730 : if (fa)
2875 0 : return xfs_inobt_complain_bad_rec(cur, fa, &irec);
2876 :
2877 258921730 : ci->count += irec.ir_count;
2878 258921730 : ci->freecount += irec.ir_freecount;
2879 :
2880 258921730 : return 0;
2881 : }
2882 :
2883 : /* Count allocated and free inodes under an inobt. */
2884 : int
2885 1144464 : xfs_ialloc_count_inodes(
2886 : struct xfs_btree_cur *cur,
2887 : xfs_agino_t *count,
2888 : xfs_agino_t *freecount)
2889 : {
2890 1144464 : struct xfs_ialloc_count_inodes ci = {0};
2891 1144464 : int error;
2892 :
2893 1144464 : ASSERT(cur->bc_btnum == XFS_BTNUM_INO);
2894 1144464 : error = xfs_btree_query_all(cur, xfs_ialloc_count_inodes_rec, &ci);
2895 1144371 : if (error)
2896 : return error;
2897 :
2898 1144371 : *count = ci.count;
2899 1144371 : *freecount = ci.freecount;
2900 1144371 : return 0;
2901 : }
2902 :
2903 : /*
2904 : * Initialize inode-related geometry information.
2905 : *
2906 : * Compute the inode btree min and max levels and set maxicount.
2907 : *
2908 : * Set the inode cluster size. This may still be overridden by the file
2909 : * system block size if it is larger than the chosen cluster size.
2910 : *
2911 : * For v5 filesystems, scale the cluster size with the inode size to keep a
2912 : * constant ratio of inode per cluster buffer, but only if mkfs has set the
2913 : * inode alignment value appropriately for larger cluster sizes.
2914 : *
2915 : * Then compute the inode cluster alignment information.
2916 : */
2917 : void
2918 66972 : xfs_ialloc_setup_geometry(
2919 : struct xfs_mount *mp)
2920 : {
2921 66972 : struct xfs_sb *sbp = &mp->m_sb;
2922 66972 : struct xfs_ino_geometry *igeo = M_IGEO(mp);
2923 66972 : uint64_t icount;
2924 66972 : uint inodes;
2925 :
2926 66972 : igeo->new_diflags2 = 0;
2927 66972 : if (xfs_has_bigtime(mp))
2928 66555 : igeo->new_diflags2 |= XFS_DIFLAG2_BIGTIME;
2929 66972 : if (xfs_has_large_extent_counts(mp))
2930 66725 : igeo->new_diflags2 |= XFS_DIFLAG2_NREXT64;
2931 :
2932 : /* Compute inode btree geometry. */
2933 66972 : igeo->agino_log = sbp->sb_inopblog + sbp->sb_agblklog;
2934 66972 : igeo->inobt_mxr[0] = xfs_inobt_maxrecs(mp, sbp->sb_blocksize, true);
2935 66972 : igeo->inobt_mxr[1] = xfs_inobt_maxrecs(mp, sbp->sb_blocksize, false);
2936 66972 : igeo->inobt_mnr[0] = igeo->inobt_mxr[0] / 2;
2937 66972 : igeo->inobt_mnr[1] = igeo->inobt_mxr[1] / 2;
2938 :
2939 66972 : igeo->ialloc_inos = max_t(uint16_t, XFS_INODES_PER_CHUNK,
2940 : sbp->sb_inopblock);
2941 66972 : igeo->ialloc_blks = igeo->ialloc_inos >> sbp->sb_inopblog;
2942 :
2943 66972 : if (sbp->sb_spino_align)
2944 66736 : igeo->ialloc_min_blks = sbp->sb_spino_align;
2945 : else
2946 236 : igeo->ialloc_min_blks = igeo->ialloc_blks;
2947 :
2948 : /* Compute and fill in value of m_ino_geo.inobt_maxlevels. */
2949 66972 : inodes = (1LL << XFS_INO_AGINO_BITS(mp)) >> XFS_INODES_PER_CHUNK_LOG;
2950 66972 : igeo->inobt_maxlevels = xfs_btree_compute_maxlevels(igeo->inobt_mnr,
2951 : inodes);
2952 66972 : ASSERT(igeo->inobt_maxlevels <= xfs_iallocbt_maxlevels_ondisk());
2953 :
2954 : /*
2955 : * Set the maximum inode count for this filesystem, being careful not
2956 : * to use obviously garbage sb_inopblog/sb_inopblock values. Regular
2957 : * users should never get here due to failing sb verification, but
2958 : * certain users (xfs_db) need to be usable even with corrupt metadata.
2959 : */
2960 66972 : if (sbp->sb_imax_pct && igeo->ialloc_blks) {
2961 : /*
2962 : * Make sure the maximum inode count is a multiple
2963 : * of the units we allocate inodes in.
2964 : */
2965 66972 : icount = sbp->sb_dblocks * sbp->sb_imax_pct;
2966 66972 : do_div(icount, 100);
2967 66972 : do_div(icount, igeo->ialloc_blks);
2968 66972 : igeo->maxicount = XFS_FSB_TO_INO(mp,
2969 : icount * igeo->ialloc_blks);
2970 : } else {
2971 0 : igeo->maxicount = 0;
2972 : }
2973 :
2974 : /*
2975 : * Compute the desired size of an inode cluster buffer size, which
2976 : * starts at 8K and (on v5 filesystems) scales up with larger inode
2977 : * sizes.
2978 : *
2979 : * Preserve the desired inode cluster size because the sparse inodes
2980 : * feature uses that desired size (not the actual size) to compute the
2981 : * sparse inode alignment. The mount code validates this value, so we
2982 : * cannot change the behavior.
2983 : */
2984 66972 : igeo->inode_cluster_size_raw = XFS_INODE_BIG_CLUSTER_SIZE;
2985 66972 : if (xfs_has_v3inodes(mp)) {
2986 66736 : int new_size = igeo->inode_cluster_size_raw;
2987 :
2988 66736 : new_size *= mp->m_sb.sb_inodesize / XFS_DINODE_MIN_SIZE;
2989 66736 : if (mp->m_sb.sb_inoalignmt >= XFS_B_TO_FSBT(mp, new_size))
2990 66736 : igeo->inode_cluster_size_raw = new_size;
2991 : }
2992 :
2993 : /* Calculate inode cluster ratios. */
2994 66972 : if (igeo->inode_cluster_size_raw > mp->m_sb.sb_blocksize)
2995 66972 : igeo->blocks_per_cluster = XFS_B_TO_FSBT(mp,
2996 : igeo->inode_cluster_size_raw);
2997 : else
2998 0 : igeo->blocks_per_cluster = 1;
2999 66972 : igeo->inode_cluster_size = XFS_FSB_TO_B(mp, igeo->blocks_per_cluster);
3000 66972 : igeo->inodes_per_cluster = XFS_FSB_TO_INO(mp, igeo->blocks_per_cluster);
3001 :
3002 : /* Calculate inode cluster alignment. */
3003 66972 : if (xfs_has_align(mp) &&
3004 66972 : mp->m_sb.sb_inoalignmt >= igeo->blocks_per_cluster)
3005 66972 : igeo->cluster_align = mp->m_sb.sb_inoalignmt;
3006 : else
3007 0 : igeo->cluster_align = 1;
3008 66972 : igeo->inoalign_mask = igeo->cluster_align - 1;
3009 66972 : igeo->cluster_align_inodes = XFS_FSB_TO_INO(mp, igeo->cluster_align);
3010 :
3011 : /*
3012 : * If we are using stripe alignment, check whether
3013 : * the stripe unit is a multiple of the inode alignment
3014 : */
3015 66972 : if (mp->m_dalign && igeo->inoalign_mask &&
3016 106 : !(mp->m_dalign & igeo->inoalign_mask))
3017 75 : igeo->ialloc_align = mp->m_dalign;
3018 : else
3019 66897 : igeo->ialloc_align = 0;
3020 66972 : }
3021 :
3022 : /* Compute the location of the root directory inode that is laid out by mkfs. */
3023 : xfs_ino_t
3024 95 : xfs_ialloc_calc_rootino(
3025 : struct xfs_mount *mp,
3026 : int sunit)
3027 : {
3028 95 : struct xfs_ino_geometry *igeo = M_IGEO(mp);
3029 95 : xfs_agblock_t first_bno;
3030 :
3031 : /*
3032 : * Pre-calculate the geometry of AG 0. We know what it looks like
3033 : * because libxfs knows how to create allocation groups now.
3034 : *
3035 : * first_bno is the first block in which mkfs could possibly have
3036 : * allocated the root directory inode, once we factor in the metadata
3037 : * that mkfs formats before it. Namely, the four AG headers...
3038 : */
3039 95 : first_bno = howmany(4 * mp->m_sb.sb_sectsize, mp->m_sb.sb_blocksize);
3040 :
3041 : /* ...the two free space btree roots... */
3042 95 : first_bno += 2;
3043 :
3044 : /* ...the inode btree root... */
3045 95 : first_bno += 1;
3046 :
3047 : /* ...the initial AGFL... */
3048 95 : first_bno += xfs_alloc_min_freelist(mp, NULL);
3049 :
3050 : /* ...the free inode btree root... */
3051 95 : if (xfs_has_finobt(mp))
3052 95 : first_bno++;
3053 :
3054 : /* ...the reverse mapping btree root... */
3055 95 : if (xfs_has_rmapbt(mp))
3056 90 : first_bno++;
3057 :
3058 : /* ...the reference count btree... */
3059 95 : if (xfs_has_reflink(mp))
3060 90 : first_bno++;
3061 :
3062 : /*
3063 : * ...and the log, if it is allocated in the first allocation group.
3064 : *
3065 : * This can happen with filesystems that only have a single
3066 : * allocation group, or very odd geometries created by old mkfs
3067 : * versions on very small filesystems.
3068 : */
3069 95 : if (xfs_ag_contains_log(mp, 0))
3070 0 : first_bno += mp->m_sb.sb_logblocks;
3071 :
3072 : /*
3073 : * Now round first_bno up to whatever allocation alignment is given
3074 : * by the filesystem or was passed in.
3075 : */
3076 95 : if (xfs_has_dalign(mp) && igeo->ialloc_align > 0)
3077 64 : first_bno = roundup(first_bno, sunit);
3078 31 : else if (xfs_has_align(mp) &&
3079 31 : mp->m_sb.sb_inoalignmt > 1)
3080 31 : first_bno = roundup(first_bno, mp->m_sb.sb_inoalignmt);
3081 :
3082 95 : return XFS_AGINO_TO_INO(mp, 0, XFS_AGB_TO_AGINO(mp, first_bno));
3083 : }
3084 :
3085 : /*
3086 : * Ensure there are not sparse inode clusters that cross the new EOAG.
3087 : *
3088 : * This is a no-op for non-spinode filesystems since clusters are always fully
3089 : * allocated and checking the bnobt suffices. However, a spinode filesystem
3090 : * could have a record where the upper inodes are free blocks. If those blocks
3091 : * were removed from the filesystem, the inode record would extend beyond EOAG,
3092 : * which will be flagged as corruption.
3093 : */
3094 : int
3095 641 : xfs_ialloc_check_shrink(
3096 : struct xfs_perag *pag,
3097 : struct xfs_trans *tp,
3098 : struct xfs_buf *agibp,
3099 : xfs_agblock_t new_length)
3100 : {
3101 641 : struct xfs_inobt_rec_incore rec;
3102 641 : struct xfs_btree_cur *cur;
3103 641 : xfs_agino_t agino;
3104 641 : int has;
3105 641 : int error;
3106 :
3107 641 : if (!xfs_has_sparseinodes(pag->pag_mount))
3108 : return 0;
3109 :
3110 641 : cur = xfs_inobt_init_cursor(pag, tp, agibp, XFS_BTNUM_INO);
3111 :
3112 : /* Look up the inobt record that would correspond to the new EOFS. */
3113 641 : agino = XFS_AGB_TO_AGINO(pag->pag_mount, new_length);
3114 641 : error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &has);
3115 641 : if (error || !has)
3116 82 : goto out;
3117 :
3118 559 : error = xfs_inobt_get_rec(cur, &rec, &has);
3119 559 : if (error)
3120 0 : goto out;
3121 :
3122 559 : if (!has) {
3123 0 : xfs_ag_mark_sick(pag, XFS_SICK_AG_INOBT);
3124 0 : error = -EFSCORRUPTED;
3125 0 : goto out;
3126 : }
3127 :
3128 : /* If the record covers inodes that would be beyond EOFS, bail out. */
3129 559 : if (rec.ir_startino + XFS_INODES_PER_CHUNK > agino) {
3130 191 : error = -ENOSPC;
3131 191 : goto out;
3132 : }
3133 368 : out:
3134 641 : xfs_btree_del_cursor(cur, error);
3135 641 : return error;
3136 : }
|
