Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
4 : * All Rights Reserved.
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_bit.h"
13 : #include "xfs_mount.h"
14 : #include "xfs_inode.h"
15 : #include "xfs_btree.h"
16 : #include "xfs_ialloc.h"
17 : #include "xfs_ialloc_btree.h"
18 : #include "xfs_alloc.h"
19 : #include "xfs_errortag.h"
20 : #include "xfs_error.h"
21 : #include "xfs_bmap.h"
22 : #include "xfs_trans.h"
23 : #include "xfs_buf_item.h"
24 : #include "xfs_icreate_item.h"
25 : #include "xfs_icache.h"
26 : #include "xfs_trace.h"
27 : #include "xfs_log.h"
28 : #include "xfs_rmap.h"
29 : #include "xfs_ag.h"
30 : #include "xfs_health.h"
31 :
32 : /*
33 : * Lookup a record by ino in the btree given by cur.
34 : */
35 : int /* error */
36 4359093657 : xfs_inobt_lookup(
37 : struct xfs_btree_cur *cur, /* btree cursor */
38 : xfs_agino_t ino, /* starting inode of chunk */
39 : xfs_lookup_t dir, /* <=, >=, == */
40 : int *stat) /* success/failure */
41 : {
42 8793245171 : cur->bc_rec.i.ir_startino = ino;
43 8793245171 : cur->bc_rec.i.ir_holemask = 0;
44 8793245171 : cur->bc_rec.i.ir_count = 0;
45 8793245171 : cur->bc_rec.i.ir_freecount = 0;
46 8793245171 : cur->bc_rec.i.ir_free = 0;
47 4359093657 : return xfs_btree_lookup(cur, dir, stat);
48 : }
49 :
50 : /*
51 : * Update the record referred to by cur to the value given.
52 : * This either works (return 0) or gets an EFSCORRUPTED error.
53 : */
54 : STATIC int /* error */
55 382106576 : xfs_inobt_update(
56 : struct xfs_btree_cur *cur, /* btree cursor */
57 : xfs_inobt_rec_incore_t *irec) /* btree record */
58 : {
59 382106576 : union xfs_btree_rec rec;
60 :
61 382106576 : rec.inobt.ir_startino = cpu_to_be32(irec->ir_startino);
62 382106576 : if (xfs_has_sparseinodes(cur->bc_mp)) {
63 382105181 : rec.inobt.ir_u.sp.ir_holemask = cpu_to_be16(irec->ir_holemask);
64 382105181 : rec.inobt.ir_u.sp.ir_count = irec->ir_count;
65 382105181 : rec.inobt.ir_u.sp.ir_freecount = irec->ir_freecount;
66 : } else {
67 : /* ir_holemask/ir_count not supported on-disk */
68 1395 : rec.inobt.ir_u.f.ir_freecount = cpu_to_be32(irec->ir_freecount);
69 : }
70 382106576 : rec.inobt.ir_free = cpu_to_be64(irec->ir_free);
71 382106576 : return xfs_btree_update(cur, &rec);
72 : }
73 :
74 : /* Convert on-disk btree record to incore inobt record. */
75 : void
76 32238290051 : xfs_inobt_btrec_to_irec(
77 : struct xfs_mount *mp,
78 : const union xfs_btree_rec *rec,
79 : struct xfs_inobt_rec_incore *irec)
80 : {
81 32238290051 : irec->ir_startino = be32_to_cpu(rec->inobt.ir_startino);
82 32238290051 : if (xfs_has_sparseinodes(mp)) {
83 32238284009 : irec->ir_holemask = be16_to_cpu(rec->inobt.ir_u.sp.ir_holemask);
84 32238284009 : irec->ir_count = rec->inobt.ir_u.sp.ir_count;
85 32238284009 : irec->ir_freecount = rec->inobt.ir_u.sp.ir_freecount;
86 : } else {
87 : /*
88 : * ir_holemask/ir_count not supported on-disk. Fill in hardcoded
89 : * values for full inode chunks.
90 : */
91 6042 : irec->ir_holemask = XFS_INOBT_HOLEMASK_FULL;
92 6042 : irec->ir_count = XFS_INODES_PER_CHUNK;
93 6042 : irec->ir_freecount =
94 6042 : be32_to_cpu(rec->inobt.ir_u.f.ir_freecount);
95 : }
96 32238290051 : irec->ir_free = be64_to_cpu(rec->inobt.ir_free);
97 32238290051 : }
98 :
99 : /* Compute the freecount of an incore inode record. */
100 : uint8_t
101 32263292874 : xfs_inobt_rec_freecount(
102 : const struct xfs_inobt_rec_incore *irec)
103 : {
104 32263292874 : uint64_t realfree;
105 :
106 32263292874 : if (!xfs_inobt_issparse(irec->ir_holemask))
107 21993189787 : realfree = irec->ir_free;
108 : else
109 10270103087 : realfree = irec->ir_free & xfs_inobt_irec_to_allocmask(irec);
110 32279358459 : return hweight64(realfree);
111 : }
112 :
113 : inline xfs_failaddr_t
114 32245216852 : xfs_inobt_check_perag_irec(
115 : struct xfs_perag *pag,
116 : const struct xfs_inobt_rec_incore *irec)
117 : {
118 : /* Record has to be properly aligned within the AG. */
119 32245216852 : if (!xfs_verify_agino(pag, irec->ir_startino))
120 0 : return __this_address;
121 32245216852 : if (!xfs_verify_agino(pag,
122 : irec->ir_startino + XFS_INODES_PER_CHUNK - 1))
123 0 : return __this_address;
124 32245216852 : if (irec->ir_count < XFS_INODES_PER_HOLEMASK_BIT ||
125 : irec->ir_count > XFS_INODES_PER_CHUNK)
126 0 : return __this_address;
127 32245216852 : if (irec->ir_freecount > XFS_INODES_PER_CHUNK)
128 0 : return __this_address;
129 :
130 32245216852 : if (xfs_inobt_rec_freecount(irec) != irec->ir_freecount)
131 0 : return __this_address;
132 :
133 : return NULL;
134 : }
135 :
136 : /* Simple checks for inode records. */
137 : xfs_failaddr_t
138 56938397 : xfs_inobt_check_irec(
139 : struct xfs_btree_cur *cur,
140 : const struct xfs_inobt_rec_incore *irec)
141 : {
142 56938397 : return xfs_inobt_check_perag_irec(cur->bc_ag.pag, irec);
143 : }
144 :
145 : static inline int
146 0 : xfs_inobt_complain_bad_rec(
147 : struct xfs_btree_cur *cur,
148 : xfs_failaddr_t fa,
149 : const struct xfs_inobt_rec_incore *irec)
150 : {
151 0 : struct xfs_mount *mp = cur->bc_mp;
152 :
153 0 : xfs_warn(mp,
154 : "%s Inode BTree record corruption in AG %d detected at %pS!",
155 : cur->bc_btnum == XFS_BTNUM_INO ? "Used" : "Free",
156 : cur->bc_ag.pag->pag_agno, fa);
157 0 : xfs_warn(mp,
158 : "start inode 0x%x, count 0x%x, free 0x%x freemask 0x%llx, holemask 0x%x",
159 : irec->ir_startino, irec->ir_count, irec->ir_freecount,
160 : irec->ir_free, irec->ir_holemask);
161 0 : xfs_btree_mark_sick(cur);
162 0 : return -EFSCORRUPTED;
163 : }
164 :
165 : /*
166 : * Get the data from the pointed-to record.
167 : */
168 : int
169 32071133032 : xfs_inobt_get_rec(
170 : struct xfs_btree_cur *cur,
171 : struct xfs_inobt_rec_incore *irec,
172 : int *stat)
173 : {
174 32071133032 : struct xfs_mount *mp = cur->bc_mp;
175 32071133032 : union xfs_btree_rec *rec;
176 32071133032 : xfs_failaddr_t fa;
177 32071133032 : int error;
178 :
179 32071133032 : error = xfs_btree_get_rec(cur, &rec, stat);
180 32002212543 : if (error || *stat == 0)
181 : return error;
182 :
183 32009001812 : xfs_inobt_btrec_to_irec(mp, rec, irec);
184 31966581084 : fa = xfs_inobt_check_irec(cur, irec);
185 32047827752 : if (fa)
186 0 : return xfs_inobt_complain_bad_rec(cur, fa, irec);
187 :
188 : return 0;
189 : }
190 :
191 : /*
192 : * Insert a single inobt record. Cursor must already point to desired location.
193 : */
194 : int
195 0 : xfs_inobt_insert_rec(
196 : struct xfs_btree_cur *cur,
197 : uint16_t holemask,
198 : uint8_t count,
199 : int32_t freecount,
200 : xfs_inofree_t free,
201 : int *stat)
202 : {
203 23945315 : cur->bc_rec.i.ir_holemask = holemask;
204 23945315 : cur->bc_rec.i.ir_count = count;
205 23945315 : cur->bc_rec.i.ir_freecount = freecount;
206 23945315 : cur->bc_rec.i.ir_free = free;
207 0 : return xfs_btree_insert(cur, stat);
208 : }
209 :
210 : /*
211 : * Insert records describing a newly allocated inode chunk into the inobt.
212 : */
213 : STATIC int
214 1357511 : xfs_inobt_insert(
215 : struct xfs_perag *pag,
216 : struct xfs_trans *tp,
217 : struct xfs_buf *agbp,
218 : xfs_agino_t newino,
219 : xfs_agino_t newlen,
220 : xfs_btnum_t btnum)
221 : {
222 1357511 : struct xfs_btree_cur *cur;
223 1357511 : xfs_agino_t thisino;
224 1357511 : int i;
225 1357511 : int error;
226 :
227 1357511 : cur = xfs_inobt_init_cursor(pag, tp, agbp, btnum);
228 :
229 1357511 : for (thisino = newino;
230 2715100 : thisino < newino + newlen;
231 1357497 : thisino += XFS_INODES_PER_CHUNK) {
232 1357597 : error = xfs_inobt_lookup(cur, thisino, XFS_LOOKUP_EQ, &i);
233 1357637 : if (error) {
234 4 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
235 4 : return error;
236 : }
237 1357633 : ASSERT(i == 0);
238 :
239 1357633 : error = xfs_inobt_insert_rec(cur, XFS_INOBT_HOLEMASK_FULL,
240 : XFS_INODES_PER_CHUNK,
241 : XFS_INODES_PER_CHUNK,
242 : XFS_INOBT_ALL_FREE, &i);
243 1357497 : if (error) {
244 0 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
245 0 : return error;
246 : }
247 1357497 : ASSERT(i == 1);
248 : }
249 :
250 1357503 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
251 :
252 1357503 : return 0;
253 : }
254 :
255 : /*
256 : * Verify that the number of free inodes in the AGI is correct.
257 : */
258 : #ifdef DEBUG
259 : static int
260 766320163 : xfs_check_agi_freecount(
261 : struct xfs_btree_cur *cur)
262 : {
263 766320163 : if (cur->bc_nlevels == 1) {
264 621123238 : xfs_inobt_rec_incore_t rec;
265 621123238 : int freecount = 0;
266 621123238 : int error;
267 621123238 : int i;
268 :
269 621123238 : error = xfs_inobt_lookup(cur, 0, XFS_LOOKUP_GE, &i);
270 621581600 : if (error)
271 191 : return error;
272 :
273 22848326490 : do {
274 22848326490 : error = xfs_inobt_get_rec(cur, &rec, &i);
275 22897473028 : if (error)
276 0 : return error;
277 :
278 22897473028 : if (i) {
279 22903097879 : freecount += rec.ir_freecount;
280 22903097879 : error = xfs_btree_increment(cur, 0, &i);
281 22853568188 : if (error)
282 0 : return error;
283 : }
284 22847943337 : } while (i == 1);
285 :
286 1242396512 : if (!xfs_is_shutdown(cur->bc_mp))
287 621231200 : ASSERT(freecount == cur->bc_ag.pag->pagi_freecount);
288 : }
289 : return 0;
290 : }
291 : #else
292 : #define xfs_check_agi_freecount(cur) 0
293 : #endif
294 :
295 : /*
296 : * Initialise a new set of inodes. When called without a transaction context
297 : * (e.g. from recovery) we initiate a delayed write of the inode buffers rather
298 : * than logging them (which in a transaction context puts them into the AIL
299 : * for writeback rather than the xfsbufd queue).
300 : */
301 : int
302 1396979 : xfs_ialloc_inode_init(
303 : struct xfs_mount *mp,
304 : struct xfs_trans *tp,
305 : struct list_head *buffer_list,
306 : int icount,
307 : xfs_agnumber_t agno,
308 : xfs_agblock_t agbno,
309 : xfs_agblock_t length,
310 : unsigned int gen)
311 : {
312 1396979 : struct xfs_buf *fbuf;
313 1396979 : struct xfs_dinode *free;
314 1396979 : int nbufs;
315 1396979 : int version;
316 1396979 : int i, j;
317 1396979 : xfs_daddr_t d;
318 1396979 : xfs_ino_t ino = 0;
319 1396979 : int error;
320 :
321 : /*
322 : * Loop over the new block(s), filling in the inodes. For small block
323 : * sizes, manipulate the inodes in buffers which are multiples of the
324 : * blocks size.
325 : */
326 1396979 : nbufs = length / M_IGEO(mp)->blocks_per_cluster;
327 :
328 : /*
329 : * Figure out what version number to use in the inodes we create. If
330 : * the superblock version has caught up to the one that supports the new
331 : * inode format, then use the new inode version. Otherwise use the old
332 : * version so that old kernels will continue to be able to use the file
333 : * system.
334 : *
335 : * For v3 inodes, we also need to write the inode number into the inode,
336 : * so calculate the first inode number of the chunk here as
337 : * XFS_AGB_TO_AGINO() only works within a filesystem block, not
338 : * across multiple filesystem blocks (such as a cluster) and so cannot
339 : * be used in the cluster buffer loop below.
340 : *
341 : * Further, because we are writing the inode directly into the buffer
342 : * and calculating a CRC on the entire inode, we have ot log the entire
343 : * inode so that the entire range the CRC covers is present in the log.
344 : * That means for v3 inode we log the entire buffer rather than just the
345 : * inode cores.
346 : */
347 1396979 : if (xfs_has_v3inodes(mp)) {
348 1396496 : version = 3;
349 1396496 : ino = XFS_AGINO_TO_INO(mp, agno, XFS_AGB_TO_AGINO(mp, agbno));
350 :
351 : /*
352 : * log the initialisation that is about to take place as an
353 : * logical operation. This means the transaction does not
354 : * need to log the physical changes to the inode buffers as log
355 : * recovery will know what initialisation is actually needed.
356 : * Hence we only need to log the buffers as "ordered" buffers so
357 : * they track in the AIL as if they were physically logged.
358 : */
359 1396496 : if (tp)
360 1367726 : xfs_icreate_log(tp, agno, agbno, icount,
361 1367726 : mp->m_sb.sb_inodesize, length, gen);
362 : } else
363 : version = 2;
364 :
365 3487900 : for (j = 0; j < nbufs; j++) {
366 : /*
367 : * Get the block.
368 : */
369 2090645 : d = XFS_AGB_TO_DADDR(mp, agno, agbno +
370 : (j * M_IGEO(mp)->blocks_per_cluster));
371 2090645 : error = xfs_trans_get_buf(tp, mp->m_ddev_targp, d,
372 2090645 : mp->m_bsize * M_IGEO(mp)->blocks_per_cluster,
373 : XBF_UNMAPPED, &fbuf);
374 2090020 : if (error)
375 0 : return error;
376 :
377 : /* Initialize the inode buffers and log them appropriately. */
378 2090020 : fbuf->b_ops = &xfs_inode_buf_ops;
379 2090020 : xfs_buf_zero(fbuf, 0, BBTOB(fbuf->b_length));
380 71074251 : for (i = 0; i < M_IGEO(mp)->inodes_per_cluster; i++) {
381 66893672 : int ioffset = i << mp->m_sb.sb_inodelog;
382 :
383 66893672 : free = xfs_make_iptr(mp, fbuf, i);
384 66891676 : free->di_magic = cpu_to_be16(XFS_DINODE_MAGIC);
385 66891676 : free->di_version = version;
386 66891676 : free->di_gen = cpu_to_be32(gen);
387 66891676 : free->di_next_unlinked = cpu_to_be32(NULLAGINO);
388 :
389 66891676 : if (version == 3) {
390 66890972 : free->di_ino = cpu_to_be64(ino);
391 66890972 : ino++;
392 66890972 : uuid_copy(&free->di_uuid,
393 66890972 : &mp->m_sb.sb_meta_uuid);
394 66889445 : xfs_dinode_calc_crc(mp, free);
395 704 : } else if (tp) {
396 : /* just log the inode core */
397 704 : xfs_trans_log_buf(tp, fbuf, ioffset,
398 1408 : ioffset + XFS_DINODE_SIZE(mp) - 1);
399 : }
400 : }
401 :
402 2090559 : if (tp) {
403 : /*
404 : * Mark the buffer as an inode allocation buffer so it
405 : * sticks in AIL at the point of this allocation
406 : * transaction. This ensures the they are on disk before
407 : * the tail of the log can be moved past this
408 : * transaction (i.e. by preventing relogging from moving
409 : * it forward in the log).
410 : */
411 2047437 : xfs_trans_inode_alloc_buf(tp, fbuf);
412 2046810 : if (version == 3) {
413 : /*
414 : * Mark the buffer as ordered so that they are
415 : * not physically logged in the transaction but
416 : * still tracked in the AIL as part of the
417 : * transaction and pin the log appropriately.
418 : */
419 2046804 : xfs_trans_ordered_buf(tp, fbuf);
420 : }
421 : } else {
422 43122 : fbuf->b_flags |= XBF_DONE;
423 43122 : xfs_buf_delwri_queue(fbuf, buffer_list);
424 43122 : xfs_buf_relse(fbuf);
425 : }
426 : }
427 : return 0;
428 : }
429 :
430 : /*
431 : * Align startino and allocmask for a recently allocated sparse chunk such that
432 : * they are fit for insertion (or merge) into the on-disk inode btrees.
433 : *
434 : * Background:
435 : *
436 : * When enabled, sparse inode support increases the inode alignment from cluster
437 : * size to inode chunk size. This means that the minimum range between two
438 : * non-adjacent inode records in the inobt is large enough for a full inode
439 : * record. This allows for cluster sized, cluster aligned block allocation
440 : * without need to worry about whether the resulting inode record overlaps with
441 : * another record in the tree. Without this basic rule, we would have to deal
442 : * with the consequences of overlap by potentially undoing recent allocations in
443 : * the inode allocation codepath.
444 : *
445 : * Because of this alignment rule (which is enforced on mount), there are two
446 : * inobt possibilities for newly allocated sparse chunks. One is that the
447 : * aligned inode record for the chunk covers a range of inodes not already
448 : * covered in the inobt (i.e., it is safe to insert a new sparse record). The
449 : * other is that a record already exists at the aligned startino that considers
450 : * the newly allocated range as sparse. In the latter case, record content is
451 : * merged in hope that sparse inode chunks fill to full chunks over time.
452 : */
453 : STATIC void
454 689570 : xfs_align_sparse_ino(
455 : struct xfs_mount *mp,
456 : xfs_agino_t *startino,
457 : uint16_t *allocmask)
458 : {
459 689570 : xfs_agblock_t agbno;
460 689570 : xfs_agblock_t mod;
461 689570 : int offset;
462 :
463 689570 : agbno = XFS_AGINO_TO_AGBNO(mp, *startino);
464 689570 : mod = agbno % mp->m_sb.sb_inoalignmt;
465 689570 : if (!mod)
466 : return;
467 :
468 : /* calculate the inode offset and align startino */
469 392437 : offset = XFS_AGB_TO_AGINO(mp, mod);
470 392437 : *startino -= offset;
471 :
472 : /*
473 : * Since startino has been aligned down, left shift allocmask such that
474 : * it continues to represent the same physical inodes relative to the
475 : * new startino.
476 : */
477 392437 : *allocmask <<= offset / XFS_INODES_PER_HOLEMASK_BIT;
478 : }
479 :
480 : /*
481 : * Determine whether the source inode record can merge into the target. Both
482 : * records must be sparse, the inode ranges must match and there must be no
483 : * allocation overlap between the records.
484 : */
485 : STATIC bool
486 133188 : __xfs_inobt_can_merge(
487 : struct xfs_inobt_rec_incore *trec, /* tgt record */
488 : struct xfs_inobt_rec_incore *srec) /* src record */
489 : {
490 133188 : uint64_t talloc;
491 133188 : uint64_t salloc;
492 :
493 : /* records must cover the same inode range */
494 133188 : if (trec->ir_startino != srec->ir_startino)
495 : return false;
496 :
497 : /* both records must be sparse */
498 133192 : if (!xfs_inobt_issparse(trec->ir_holemask) ||
499 133192 : !xfs_inobt_issparse(srec->ir_holemask))
500 : return false;
501 :
502 : /* both records must track some inodes */
503 133192 : if (!trec->ir_count || !srec->ir_count)
504 : return false;
505 :
506 : /* can't exceed capacity of a full record */
507 133192 : if (trec->ir_count + srec->ir_count > XFS_INODES_PER_CHUNK)
508 : return false;
509 :
510 : /* verify there is no allocation overlap */
511 133195 : talloc = xfs_inobt_irec_to_allocmask(trec);
512 133207 : salloc = xfs_inobt_irec_to_allocmask(srec);
513 133196 : if (talloc & salloc)
514 0 : return false;
515 :
516 : return true;
517 : }
518 :
519 : /*
520 : * Merge the source inode record into the target. The caller must call
521 : * __xfs_inobt_can_merge() to ensure the merge is valid.
522 : */
523 : STATIC void
524 133191 : __xfs_inobt_rec_merge(
525 : struct xfs_inobt_rec_incore *trec, /* target */
526 : struct xfs_inobt_rec_incore *srec) /* src */
527 : {
528 133191 : ASSERT(trec->ir_startino == srec->ir_startino);
529 :
530 : /* combine the counts */
531 133191 : trec->ir_count += srec->ir_count;
532 133191 : trec->ir_freecount += srec->ir_freecount;
533 :
534 : /*
535 : * Merge the holemask and free mask. For both fields, 0 bits refer to
536 : * allocated inodes. We combine the allocated ranges with bitwise AND.
537 : */
538 133191 : trec->ir_holemask &= srec->ir_holemask;
539 133191 : trec->ir_free &= srec->ir_free;
540 133191 : }
541 :
542 : /*
543 : * Insert a new sparse inode chunk into the associated inode btree. The inode
544 : * record for the sparse chunk is pre-aligned to a startino that should match
545 : * any pre-existing sparse inode record in the tree. This allows sparse chunks
546 : * to fill over time.
547 : *
548 : * This function supports two modes of handling preexisting records depending on
549 : * the merge flag. If merge is true, the provided record is merged with the
550 : * existing record and updated in place. The merged record is returned in nrec.
551 : * If merge is false, an existing record is replaced with the provided record.
552 : * If no preexisting record exists, the provided record is always inserted.
553 : *
554 : * It is considered corruption if a merge is requested and not possible. Given
555 : * the sparse inode alignment constraints, this should never happen.
556 : */
557 : STATIC int
558 1379091 : xfs_inobt_insert_sprec(
559 : struct xfs_perag *pag,
560 : struct xfs_trans *tp,
561 : struct xfs_buf *agbp,
562 : int btnum,
563 : struct xfs_inobt_rec_incore *nrec, /* in/out: new/merged rec. */
564 : bool merge) /* merge or replace */
565 : {
566 1379091 : struct xfs_mount *mp = pag->pag_mount;
567 1379091 : struct xfs_btree_cur *cur;
568 1379091 : int error;
569 1379091 : int i;
570 1379091 : struct xfs_inobt_rec_incore rec;
571 :
572 1379091 : cur = xfs_inobt_init_cursor(pag, tp, agbp, btnum);
573 :
574 : /* the new record is pre-aligned so we know where to look */
575 1379236 : error = xfs_inobt_lookup(cur, nrec->ir_startino, XFS_LOOKUP_EQ, &i);
576 1379283 : if (error)
577 3 : goto error;
578 : /* if nothing there, insert a new record and return */
579 1379280 : if (i == 0) {
580 1246060 : error = xfs_inobt_insert_rec(cur, nrec->ir_holemask,
581 1246060 : nrec->ir_count, nrec->ir_freecount,
582 : nrec->ir_free, &i);
583 1246011 : if (error)
584 0 : goto error;
585 1246011 : if (XFS_IS_CORRUPT(mp, i != 1)) {
586 0 : xfs_btree_mark_sick(cur);
587 0 : error = -EFSCORRUPTED;
588 0 : goto error;
589 : }
590 :
591 1246011 : goto out;
592 : }
593 :
594 : /*
595 : * A record exists at this startino. Merge or replace the record
596 : * depending on what we've been asked to do.
597 : */
598 133220 : if (merge) {
599 133211 : error = xfs_inobt_get_rec(cur, &rec, &i);
600 133209 : if (error)
601 0 : goto error;
602 133209 : if (XFS_IS_CORRUPT(mp, i != 1)) {
603 0 : xfs_btree_mark_sick(cur);
604 0 : error = -EFSCORRUPTED;
605 0 : goto error;
606 : }
607 133209 : if (XFS_IS_CORRUPT(mp, rec.ir_startino != nrec->ir_startino)) {
608 0 : xfs_btree_mark_sick(cur);
609 0 : error = -EFSCORRUPTED;
610 0 : goto error;
611 : }
612 :
613 : /*
614 : * This should never fail. If we have coexisting records that
615 : * cannot merge, something is seriously wrong.
616 : */
617 133209 : if (XFS_IS_CORRUPT(mp, !__xfs_inobt_can_merge(nrec, &rec))) {
618 0 : xfs_btree_mark_sick(cur);
619 0 : error = -EFSCORRUPTED;
620 0 : goto error;
621 : }
622 :
623 133200 : trace_xfs_irec_merge_pre(mp, pag->pag_agno, rec.ir_startino,
624 133200 : rec.ir_holemask, nrec->ir_startino,
625 133200 : nrec->ir_holemask);
626 :
627 : /* merge to nrec to output the updated record */
628 133193 : __xfs_inobt_rec_merge(nrec, &rec);
629 :
630 133190 : trace_xfs_irec_merge_post(mp, pag->pag_agno, nrec->ir_startino,
631 133190 : nrec->ir_holemask);
632 :
633 133187 : error = xfs_inobt_rec_check_count(mp, nrec);
634 133213 : if (error)
635 0 : goto error;
636 : }
637 :
638 133222 : error = xfs_inobt_update(cur, nrec);
639 133209 : if (error)
640 0 : goto error;
641 :
642 133209 : out:
643 1379220 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
644 1379220 : return 0;
645 3 : error:
646 3 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
647 3 : return error;
648 : }
649 :
650 : /*
651 : * Allocate new inodes in the allocation group specified by agbp. Returns 0 if
652 : * inodes were allocated in this AG; -EAGAIN if there was no space in this AG so
653 : * the caller knows it can try another AG, a hard -ENOSPC when over the maximum
654 : * inode count threshold, or the usual negative error code for other errors.
655 : */
656 : STATIC int
657 2964295 : xfs_ialloc_ag_alloc(
658 : struct xfs_perag *pag,
659 : struct xfs_trans *tp,
660 : struct xfs_buf *agbp)
661 : {
662 2964295 : struct xfs_agi *agi;
663 2964295 : struct xfs_alloc_arg args;
664 2964295 : int error;
665 2964295 : xfs_agino_t newino; /* new first inode's number */
666 2964295 : xfs_agino_t newlen; /* new number of inodes */
667 2964295 : int isaligned = 0; /* inode allocation at stripe */
668 : /* unit boundary */
669 : /* init. to full chunk */
670 2964295 : struct xfs_inobt_rec_incore rec;
671 2964295 : struct xfs_ino_geometry *igeo = M_IGEO(tp->t_mountp);
672 2964295 : uint16_t allocmask = (uint16_t) -1;
673 2964295 : int do_sparse = 0;
674 :
675 2964295 : memset(&args, 0, sizeof(args));
676 2964295 : args.tp = tp;
677 2964295 : args.mp = tp->t_mountp;
678 2964295 : args.fsbno = NULLFSBLOCK;
679 2964295 : args.oinfo = XFS_RMAP_OINFO_INODES;
680 2964295 : args.pag = pag;
681 :
682 : #ifdef DEBUG
683 : /* randomly do sparse inode allocations */
684 2964295 : if (xfs_has_sparseinodes(tp->t_mountp) &&
685 2963978 : igeo->ialloc_min_blks < igeo->ialloc_blks)
686 2963870 : do_sparse = get_random_u32_below(2);
687 : #endif
688 :
689 : /*
690 : * Locking will ensure that we don't have two callers in here
691 : * at one time.
692 : */
693 2964044 : newlen = igeo->ialloc_inos;
694 2964044 : if (igeo->maxicount &&
695 2963800 : percpu_counter_read_positive(&args.mp->m_icount) + newlen >
696 : igeo->maxicount)
697 : return -ENOSPC;
698 2964044 : args.minlen = args.maxlen = igeo->ialloc_blks;
699 : /*
700 : * First try to allocate inodes contiguous with the last-allocated
701 : * chunk of inodes. If the filesystem is striped, this will fill
702 : * an entire stripe unit with inodes.
703 : */
704 2964044 : agi = agbp->b_addr;
705 2964044 : newino = be32_to_cpu(agi->agi_newino);
706 2964044 : args.agbno = XFS_AGINO_TO_AGBNO(args.mp, newino) +
707 2964044 : igeo->ialloc_blks;
708 2964044 : if (do_sparse)
709 1481588 : goto sparse_alloc;
710 1482456 : if (likely(newino != NULLAGINO &&
711 : (args.agbno < be32_to_cpu(agi->agi_length)))) {
712 1425827 : args.prod = 1;
713 :
714 : /*
715 : * We need to take into account alignment here to ensure that
716 : * we don't modify the free list if we fail to have an exact
717 : * block. If we don't have an exact match, and every oher
718 : * attempt allocation attempt fails, we'll end up cancelling
719 : * a dirty transaction and shutting down.
720 : *
721 : * For an exact allocation, alignment must be 1,
722 : * however we need to take cluster alignment into account when
723 : * fixing up the freelist. Use the minalignslop field to
724 : * indicate that extra blocks might be required for alignment,
725 : * but not to use them in the actual exact allocation.
726 : */
727 1425827 : args.alignment = 1;
728 1425827 : args.minalignslop = igeo->cluster_align - 1;
729 :
730 : /* Allow space for the inode btree to split. */
731 1425827 : args.minleft = igeo->inobt_maxlevels;
732 2851654 : error = xfs_alloc_vextent_exact_bno(&args,
733 1425827 : XFS_AGB_TO_FSB(args.mp, pag->pag_agno,
734 : args.agbno));
735 1425862 : if (error)
736 : return error;
737 :
738 : /*
739 : * This request might have dirtied the transaction if the AG can
740 : * satisfy the request, but the exact block was not available.
741 : * If the allocation did fail, subsequent requests will relax
742 : * the exact agbno requirement and increase the alignment
743 : * instead. It is critical that the total size of the request
744 : * (len + alignment + slop) does not increase from this point
745 : * on, so reset minalignslop to ensure it is not included in
746 : * subsequent requests.
747 : */
748 1425862 : args.minalignslop = 0;
749 : }
750 :
751 1482491 : if (unlikely(args.fsbno == NULLFSBLOCK)) {
752 : /*
753 : * Set the alignment for the allocation.
754 : * If stripe alignment is turned on then align at stripe unit
755 : * boundary.
756 : * If the cluster size is smaller than a filesystem block
757 : * then we're doing I/O for inodes in filesystem block size
758 : * pieces, so don't need alignment anyway.
759 : */
760 1172890 : isaligned = 0;
761 1172890 : if (igeo->ialloc_align) {
762 0 : ASSERT(!xfs_has_noalign(args.mp));
763 0 : args.alignment = args.mp->m_dalign;
764 0 : isaligned = 1;
765 : } else
766 1172890 : args.alignment = igeo->cluster_align;
767 : /*
768 : * Allocate a fixed-size extent of inodes.
769 : */
770 1172890 : args.prod = 1;
771 : /*
772 : * Allow space for the inode btree to split.
773 : */
774 1172890 : args.minleft = igeo->inobt_maxlevels;
775 2345780 : error = xfs_alloc_vextent_near_bno(&args,
776 1172890 : XFS_AGB_TO_FSB(args.mp, pag->pag_agno,
777 : be32_to_cpu(agi->agi_root)));
778 1172893 : if (error)
779 : return error;
780 : }
781 :
782 : /*
783 : * If stripe alignment is turned on, then try again with cluster
784 : * alignment.
785 : */
786 1172885 : if (isaligned && args.fsbno == NULLFSBLOCK) {
787 0 : args.alignment = igeo->cluster_align;
788 0 : error = xfs_alloc_vextent_near_bno(&args,
789 0 : XFS_AGB_TO_FSB(args.mp, pag->pag_agno,
790 : be32_to_cpu(agi->agi_root)));
791 0 : if (error)
792 : return error;
793 : }
794 :
795 : /*
796 : * Finally, try a sparse allocation if the filesystem supports it and
797 : * the sparse allocation length is smaller than a full chunk.
798 : */
799 1482486 : if (xfs_has_sparseinodes(args.mp) &&
800 1482413 : igeo->ialloc_min_blks < igeo->ialloc_blks &&
801 1482404 : args.fsbno == NULLFSBLOCK) {
802 803690 : sparse_alloc:
803 2285278 : args.alignment = args.mp->m_sb.sb_spino_align;
804 2285278 : args.prod = 1;
805 :
806 2285278 : args.minlen = igeo->ialloc_min_blks;
807 2285278 : args.maxlen = args.minlen;
808 :
809 : /*
810 : * The inode record will be aligned to full chunk size. We must
811 : * prevent sparse allocation from AG boundaries that result in
812 : * invalid inode records, such as records that start at agbno 0
813 : * or extend beyond the AG.
814 : *
815 : * Set min agbno to the first aligned, non-zero agbno and max to
816 : * the last aligned agbno that is at least one full chunk from
817 : * the end of the AG.
818 : */
819 2285278 : args.min_agbno = args.mp->m_sb.sb_inoalignmt;
820 2285278 : args.max_agbno = round_down(args.mp->m_sb.sb_agblocks,
821 2285278 : args.mp->m_sb.sb_inoalignmt) -
822 2285278 : igeo->ialloc_blks;
823 :
824 4570556 : error = xfs_alloc_vextent_near_bno(&args,
825 2285278 : XFS_AGB_TO_FSB(args.mp, pag->pag_agno,
826 : be32_to_cpu(agi->agi_root)));
827 2285330 : if (error)
828 : return error;
829 :
830 2285323 : newlen = XFS_AGB_TO_AGINO(args.mp, args.len);
831 2285323 : ASSERT(newlen <= XFS_INODES_PER_CHUNK);
832 2285323 : allocmask = (1 << (newlen / XFS_INODES_PER_HOLEMASK_BIT)) - 1;
833 : }
834 :
835 2964119 : if (args.fsbno == NULLFSBLOCK)
836 : return -EAGAIN;
837 :
838 1368418 : ASSERT(args.len == args.minlen);
839 :
840 : /*
841 : * Stamp and write the inode buffers.
842 : *
843 : * Seed the new inode cluster with a random generation number. This
844 : * prevents short-term reuse of generation numbers if a chunk is
845 : * freed and then immediately reallocated. We use random numbers
846 : * rather than a linear progression to prevent the next generation
847 : * number from being easily guessable.
848 : */
849 1368418 : error = xfs_ialloc_inode_init(args.mp, tp, NULL, newlen, pag->pag_agno,
850 : args.agbno, args.len, get_random_u32());
851 :
852 1368478 : if (error)
853 : return error;
854 : /*
855 : * Convert the results.
856 : */
857 1368478 : newino = XFS_AGB_TO_AGINO(args.mp, args.agbno);
858 :
859 1368478 : if (xfs_inobt_issparse(~allocmask)) {
860 : /*
861 : * We've allocated a sparse chunk. Align the startino and mask.
862 : */
863 689632 : xfs_align_sparse_ino(args.mp, &newino, &allocmask);
864 :
865 689491 : rec.ir_startino = newino;
866 689491 : rec.ir_holemask = ~allocmask;
867 689491 : rec.ir_count = newlen;
868 689491 : rec.ir_freecount = newlen;
869 689491 : rec.ir_free = XFS_INOBT_ALL_FREE;
870 :
871 : /*
872 : * Insert the sparse record into the inobt and allow for a merge
873 : * if necessary. If a merge does occur, rec is updated to the
874 : * merged record.
875 : */
876 689491 : error = xfs_inobt_insert_sprec(pag, tp, agbp,
877 : XFS_BTNUM_INO, &rec, true);
878 689659 : if (error == -EFSCORRUPTED) {
879 0 : xfs_alert(args.mp,
880 : "invalid sparse inode record: ino 0x%llx holemask 0x%x count %u",
881 : XFS_AGINO_TO_INO(args.mp, pag->pag_agno,
882 : rec.ir_startino),
883 : rec.ir_holemask, rec.ir_count);
884 0 : xfs_force_shutdown(args.mp, SHUTDOWN_CORRUPT_INCORE);
885 : }
886 689659 : if (error)
887 : return error;
888 :
889 : /*
890 : * We can't merge the part we've just allocated as for the inobt
891 : * due to finobt semantics. The original record may or may not
892 : * exist independent of whether physical inodes exist in this
893 : * sparse chunk.
894 : *
895 : * We must update the finobt record based on the inobt record.
896 : * rec contains the fully merged and up to date inobt record
897 : * from the previous call. Set merge false to replace any
898 : * existing record with this one.
899 : */
900 689659 : if (xfs_has_finobt(args.mp)) {
901 689575 : error = xfs_inobt_insert_sprec(pag, tp, agbp,
902 : XFS_BTNUM_FINO, &rec, false);
903 689618 : if (error)
904 : return error;
905 : }
906 : } else {
907 : /* full chunk - insert new records to both btrees */
908 678846 : error = xfs_inobt_insert(pag, tp, agbp, newino, newlen,
909 : XFS_BTNUM_INO);
910 678834 : if (error)
911 : return error;
912 :
913 678834 : if (xfs_has_finobt(args.mp)) {
914 678764 : error = xfs_inobt_insert(pag, tp, agbp, newino,
915 : newlen, XFS_BTNUM_FINO);
916 678792 : if (error)
917 : return error;
918 : }
919 : }
920 :
921 : /*
922 : * Update AGI counts and newino.
923 : */
924 1368557 : be32_add_cpu(&agi->agi_count, newlen);
925 1368557 : be32_add_cpu(&agi->agi_freecount, newlen);
926 1368557 : pag->pagi_freecount += newlen;
927 1368557 : pag->pagi_count += newlen;
928 1368557 : agi->agi_newino = cpu_to_be32(newino);
929 :
930 : /*
931 : * Log allocation group header fields
932 : */
933 1368557 : xfs_ialloc_log_agi(tp, agbp,
934 : XFS_AGI_COUNT | XFS_AGI_FREECOUNT | XFS_AGI_NEWINO);
935 : /*
936 : * Modify/log superblock values for inode count and inode free count.
937 : */
938 1368525 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_ICOUNT, (long)newlen);
939 1368363 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, (long)newlen);
940 1368363 : return 0;
941 : }
942 :
943 : /*
944 : * Try to retrieve the next record to the left/right from the current one.
945 : */
946 : STATIC int
947 2612 : xfs_ialloc_next_rec(
948 : struct xfs_btree_cur *cur,
949 : xfs_inobt_rec_incore_t *rec,
950 : int *done,
951 : int left)
952 : {
953 2612 : int error;
954 2612 : int i;
955 :
956 2612 : if (left)
957 1305 : error = xfs_btree_decrement(cur, 0, &i);
958 : else
959 1307 : error = xfs_btree_increment(cur, 0, &i);
960 :
961 2612 : if (error)
962 : return error;
963 2612 : *done = !i;
964 2612 : if (i) {
965 1409 : error = xfs_inobt_get_rec(cur, rec, &i);
966 1409 : if (error)
967 : return error;
968 1409 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
969 0 : xfs_btree_mark_sick(cur);
970 0 : return -EFSCORRUPTED;
971 : }
972 : }
973 :
974 : return 0;
975 : }
976 :
977 : STATIC int
978 1372 : xfs_ialloc_get_rec(
979 : struct xfs_btree_cur *cur,
980 : xfs_agino_t agino,
981 : xfs_inobt_rec_incore_t *rec,
982 : int *done)
983 : {
984 1372 : int error;
985 1372 : int i;
986 :
987 1372 : error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_EQ, &i);
988 1372 : if (error)
989 : return error;
990 1372 : *done = !i;
991 1372 : if (i) {
992 729 : error = xfs_inobt_get_rec(cur, rec, &i);
993 729 : if (error)
994 : return error;
995 729 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
996 0 : xfs_btree_mark_sick(cur);
997 0 : return -EFSCORRUPTED;
998 : }
999 : }
1000 :
1001 : return 0;
1002 : }
1003 :
1004 : /*
1005 : * Return the offset of the first free inode in the record. If the inode chunk
1006 : * is sparsely allocated, we convert the record holemask to inode granularity
1007 : * and mask off the unallocated regions from the inode free mask.
1008 : */
1009 : STATIC int
1010 127315291 : xfs_inobt_first_free_inode(
1011 : struct xfs_inobt_rec_incore *rec)
1012 : {
1013 127315291 : xfs_inofree_t realfree;
1014 :
1015 : /* if there are no holes, return the first available offset */
1016 127315291 : if (!xfs_inobt_issparse(rec->ir_holemask))
1017 91031556 : return xfs_lowbit64(rec->ir_free);
1018 :
1019 36283735 : realfree = xfs_inobt_irec_to_allocmask(rec);
1020 36285226 : realfree &= rec->ir_free;
1021 :
1022 36285226 : return xfs_lowbit64(realfree);
1023 : }
1024 :
1025 : /*
1026 : * If this AG has corrupt inodes, check if allocating this inode would fail
1027 : * with corruption errors. Returns 0 if we're clear, or EAGAIN to try again
1028 : * somewhere else.
1029 : */
1030 : static int
1031 0 : xfs_dialloc_check_ino(
1032 : struct xfs_perag *pag,
1033 : struct xfs_trans *tp,
1034 : xfs_ino_t ino)
1035 : {
1036 0 : struct xfs_imap imap;
1037 0 : struct xfs_buf *bp;
1038 0 : int error;
1039 :
1040 0 : error = xfs_imap(pag, tp, ino, &imap, 0);
1041 0 : if (error)
1042 : return -EAGAIN;
1043 :
1044 0 : error = xfs_imap_to_bp(pag->pag_mount, tp, &imap, &bp);
1045 0 : if (error)
1046 : return -EAGAIN;
1047 :
1048 0 : xfs_trans_brelse(tp, bp);
1049 0 : return 0;
1050 : }
1051 :
1052 : /*
1053 : * Allocate an inode using the inobt-only algorithm.
1054 : */
1055 : STATIC int
1056 7209 : xfs_dialloc_ag_inobt(
1057 : struct xfs_perag *pag,
1058 : struct xfs_trans *tp,
1059 : struct xfs_buf *agbp,
1060 : xfs_ino_t parent,
1061 : xfs_ino_t *inop)
1062 : {
1063 7209 : struct xfs_mount *mp = tp->t_mountp;
1064 7209 : struct xfs_agi *agi = agbp->b_addr;
1065 7209 : xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent);
1066 7209 : xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent);
1067 7209 : struct xfs_btree_cur *cur, *tcur;
1068 7209 : struct xfs_inobt_rec_incore rec, trec;
1069 7209 : xfs_ino_t ino;
1070 7209 : int error;
1071 7209 : int offset;
1072 7209 : int i, j;
1073 7209 : int searchdistance = 10;
1074 :
1075 14418 : ASSERT(xfs_perag_initialised_agi(pag));
1076 14418 : ASSERT(xfs_perag_allows_inodes(pag));
1077 7209 : ASSERT(pag->pagi_freecount > 0);
1078 :
1079 7209 : restart_pagno:
1080 7264 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_INO);
1081 : /*
1082 : * If pagino is 0 (this is the root inode allocation) use newino.
1083 : * This must work because we've just allocated some.
1084 : */
1085 7264 : if (!pagino)
1086 72 : pagino = be32_to_cpu(agi->agi_newino);
1087 :
1088 7264 : error = xfs_check_agi_freecount(cur);
1089 7263 : if (error)
1090 0 : goto error0;
1091 :
1092 : /*
1093 : * If in the same AG as the parent, try to get near the parent.
1094 : */
1095 7263 : if (pagno == pag->pag_agno) {
1096 6461 : int doneleft; /* done, to the left */
1097 6461 : int doneright; /* done, to the right */
1098 :
1099 6461 : error = xfs_inobt_lookup(cur, pagino, XFS_LOOKUP_LE, &i);
1100 6461 : if (error)
1101 0 : goto error0;
1102 6461 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1103 0 : xfs_btree_mark_sick(cur);
1104 0 : error = -EFSCORRUPTED;
1105 0 : goto error0;
1106 : }
1107 :
1108 6461 : error = xfs_inobt_get_rec(cur, &rec, &j);
1109 6462 : if (error)
1110 0 : goto error0;
1111 6462 : if (XFS_IS_CORRUPT(mp, j != 1)) {
1112 0 : xfs_btree_mark_sick(cur);
1113 0 : error = -EFSCORRUPTED;
1114 0 : goto error0;
1115 : }
1116 :
1117 6462 : if (rec.ir_freecount > 0) {
1118 : /*
1119 : * Found a free inode in the same chunk
1120 : * as the parent, done.
1121 : */
1122 6401 : goto alloc_inode;
1123 : }
1124 :
1125 :
1126 : /*
1127 : * In the same AG as parent, but parent's chunk is full.
1128 : */
1129 :
1130 : /* duplicate the cursor, search left & right simultaneously */
1131 1851 : error = xfs_btree_dup_cursor(cur, &tcur);
1132 1851 : if (error)
1133 0 : goto error0;
1134 :
1135 : /*
1136 : * Skip to last blocks looked up if same parent inode.
1137 : */
1138 1851 : if (pagino != NULLAGINO &&
1139 1851 : pag->pagl_pagino == pagino &&
1140 686 : pag->pagl_leftrec != NULLAGINO &&
1141 686 : pag->pagl_rightrec != NULLAGINO) {
1142 686 : error = xfs_ialloc_get_rec(tcur, pag->pagl_leftrec,
1143 : &trec, &doneleft);
1144 686 : if (error)
1145 0 : goto error1;
1146 :
1147 686 : error = xfs_ialloc_get_rec(cur, pag->pagl_rightrec,
1148 : &rec, &doneright);
1149 686 : if (error)
1150 0 : goto error1;
1151 : } else {
1152 : /* search left with tcur, back up 1 record */
1153 1165 : error = xfs_ialloc_next_rec(tcur, &trec, &doneleft, 1);
1154 1165 : if (error)
1155 0 : goto error1;
1156 :
1157 : /* search right with cur, go forward 1 record. */
1158 1165 : error = xfs_ialloc_next_rec(cur, &rec, &doneright, 0);
1159 1165 : if (error)
1160 0 : goto error1;
1161 : }
1162 :
1163 : /*
1164 : * Loop until we find an inode chunk with a free inode.
1165 : */
1166 2133 : while (--searchdistance > 0 && (!doneleft || !doneright)) {
1167 2072 : int useleft; /* using left inode chunk this time */
1168 :
1169 : /* figure out the closer block if both are valid. */
1170 2072 : if (!doneleft && !doneright) {
1171 201 : useleft = pagino -
1172 201 : (trec.ir_startino + XFS_INODES_PER_CHUNK - 1) <
1173 201 : rec.ir_startino - pagino;
1174 : } else {
1175 1871 : useleft = !doneleft;
1176 : }
1177 :
1178 : /* free inodes to the left? */
1179 2072 : if (useleft && trec.ir_freecount) {
1180 59 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
1181 59 : cur = tcur;
1182 :
1183 59 : pag->pagl_leftrec = trec.ir_startino;
1184 59 : pag->pagl_rightrec = rec.ir_startino;
1185 59 : pag->pagl_pagino = pagino;
1186 59 : rec = trec;
1187 59 : goto alloc_inode;
1188 : }
1189 :
1190 : /* free inodes to the right? */
1191 2013 : if (!useleft && rec.ir_freecount) {
1192 1731 : xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR);
1193 :
1194 1731 : pag->pagl_leftrec = trec.ir_startino;
1195 1731 : pag->pagl_rightrec = rec.ir_startino;
1196 1731 : pag->pagl_pagino = pagino;
1197 1731 : goto alloc_inode;
1198 : }
1199 :
1200 : /* get next record to check */
1201 282 : if (useleft) {
1202 140 : error = xfs_ialloc_next_rec(tcur, &trec,
1203 : &doneleft, 1);
1204 : } else {
1205 142 : error = xfs_ialloc_next_rec(cur, &rec,
1206 : &doneright, 0);
1207 : }
1208 282 : if (error)
1209 0 : goto error1;
1210 : }
1211 :
1212 61 : if (searchdistance <= 0) {
1213 : /*
1214 : * Not in range - save last search
1215 : * location and allocate a new inode
1216 : */
1217 6 : xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR);
1218 6 : pag->pagl_leftrec = trec.ir_startino;
1219 6 : pag->pagl_rightrec = rec.ir_startino;
1220 6 : pag->pagl_pagino = pagino;
1221 :
1222 : } else {
1223 : /*
1224 : * We've reached the end of the btree. because
1225 : * we are only searching a small chunk of the
1226 : * btree each search, there is obviously free
1227 : * inodes closer to the parent inode than we
1228 : * are now. restart the search again.
1229 : */
1230 55 : pag->pagl_pagino = NULLAGINO;
1231 55 : pag->pagl_leftrec = NULLAGINO;
1232 55 : pag->pagl_rightrec = NULLAGINO;
1233 55 : xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR);
1234 55 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
1235 55 : goto restart_pagno;
1236 : }
1237 : }
1238 :
1239 : /*
1240 : * In a different AG from the parent.
1241 : * See if the most recently allocated block has any free.
1242 : */
1243 808 : if (agi->agi_newino != cpu_to_be32(NULLAGINO)) {
1244 808 : error = xfs_inobt_lookup(cur, be32_to_cpu(agi->agi_newino),
1245 : XFS_LOOKUP_EQ, &i);
1246 808 : if (error)
1247 0 : goto error0;
1248 :
1249 808 : if (i == 1) {
1250 808 : error = xfs_inobt_get_rec(cur, &rec, &j);
1251 808 : if (error)
1252 0 : goto error0;
1253 :
1254 808 : if (j == 1 && rec.ir_freecount > 0) {
1255 : /*
1256 : * The last chunk allocated in the group
1257 : * still has a free inode.
1258 : */
1259 801 : goto alloc_inode;
1260 : }
1261 : }
1262 : }
1263 :
1264 : /*
1265 : * None left in the last group, search the whole AG
1266 : */
1267 7 : error = xfs_inobt_lookup(cur, 0, XFS_LOOKUP_GE, &i);
1268 7 : if (error)
1269 0 : goto error0;
1270 7 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1271 0 : xfs_btree_mark_sick(cur);
1272 0 : error = -EFSCORRUPTED;
1273 0 : goto error0;
1274 : }
1275 :
1276 7 : for (;;) {
1277 7 : error = xfs_inobt_get_rec(cur, &rec, &i);
1278 7 : if (error)
1279 0 : goto error0;
1280 7 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1281 0 : xfs_btree_mark_sick(cur);
1282 0 : error = -EFSCORRUPTED;
1283 0 : goto error0;
1284 : }
1285 7 : if (rec.ir_freecount > 0)
1286 : break;
1287 6 : error = xfs_btree_increment(cur, 0, &i);
1288 6 : if (error)
1289 0 : goto error0;
1290 6 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1291 6 : xfs_btree_mark_sick(cur);
1292 6 : error = -EFSCORRUPTED;
1293 6 : goto error0;
1294 : }
1295 : }
1296 :
1297 1 : alloc_inode:
1298 7203 : offset = xfs_inobt_first_free_inode(&rec);
1299 7203 : ASSERT(offset >= 0);
1300 7203 : ASSERT(offset < XFS_INODES_PER_CHUNK);
1301 7203 : ASSERT((XFS_AGINO_TO_OFFSET(mp, rec.ir_startino) %
1302 : XFS_INODES_PER_CHUNK) == 0);
1303 7203 : ino = XFS_AGINO_TO_INO(mp, pag->pag_agno, rec.ir_startino + offset);
1304 :
1305 7203 : if (xfs_ag_has_sickness(pag, XFS_SICK_AG_INODES)) {
1306 0 : error = xfs_dialloc_check_ino(pag, tp, ino);
1307 0 : if (error)
1308 0 : goto error0;
1309 : }
1310 :
1311 7202 : rec.ir_free &= ~XFS_INOBT_MASK(offset);
1312 7202 : rec.ir_freecount--;
1313 7202 : error = xfs_inobt_update(cur, &rec);
1314 7202 : if (error)
1315 0 : goto error0;
1316 7202 : be32_add_cpu(&agi->agi_freecount, -1);
1317 7202 : xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT);
1318 7202 : pag->pagi_freecount--;
1319 :
1320 7202 : error = xfs_check_agi_freecount(cur);
1321 7203 : if (error)
1322 0 : goto error0;
1323 :
1324 7203 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
1325 7202 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -1);
1326 7201 : *inop = ino;
1327 7201 : return 0;
1328 : error1:
1329 0 : xfs_btree_del_cursor(tcur, XFS_BTREE_ERROR);
1330 6 : error0:
1331 6 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
1332 6 : return error;
1333 : }
1334 :
1335 : /*
1336 : * Use the free inode btree to allocate an inode based on distance from the
1337 : * parent. Note that the provided cursor may be deleted and replaced.
1338 : */
1339 : STATIC int
1340 116831511 : xfs_dialloc_ag_finobt_near(
1341 : xfs_agino_t pagino,
1342 : struct xfs_btree_cur **ocur,
1343 : struct xfs_inobt_rec_incore *rec)
1344 : {
1345 116831511 : struct xfs_btree_cur *lcur = *ocur; /* left search cursor */
1346 116831511 : struct xfs_btree_cur *rcur; /* right search cursor */
1347 116831511 : struct xfs_inobt_rec_incore rrec;
1348 116831511 : int error;
1349 116831511 : int i, j;
1350 :
1351 116831511 : error = xfs_inobt_lookup(lcur, pagino, XFS_LOOKUP_LE, &i);
1352 116937210 : if (error)
1353 : return error;
1354 :
1355 116937210 : if (i == 1) {
1356 16440044 : error = xfs_inobt_get_rec(lcur, rec, &i);
1357 16440553 : if (error)
1358 : return error;
1359 16440553 : if (XFS_IS_CORRUPT(lcur->bc_mp, i != 1)) {
1360 0 : xfs_btree_mark_sick(lcur);
1361 0 : return -EFSCORRUPTED;
1362 : }
1363 :
1364 : /*
1365 : * See if we've landed in the parent inode record. The finobt
1366 : * only tracks chunks with at least one free inode, so record
1367 : * existence is enough.
1368 : */
1369 16440553 : if (pagino >= rec->ir_startino &&
1370 16440548 : pagino < (rec->ir_startino + XFS_INODES_PER_CHUNK))
1371 : return 0;
1372 : }
1373 :
1374 110085625 : error = xfs_btree_dup_cursor(lcur, &rcur);
1375 109593277 : if (error)
1376 : return error;
1377 :
1378 109604963 : error = xfs_inobt_lookup(rcur, pagino, XFS_LOOKUP_GE, &j);
1379 109937724 : if (error)
1380 0 : goto error_rcur;
1381 109937724 : if (j == 1) {
1382 106319630 : error = xfs_inobt_get_rec(rcur, &rrec, &j);
1383 106447007 : if (error)
1384 0 : goto error_rcur;
1385 106447007 : if (XFS_IS_CORRUPT(lcur->bc_mp, j != 1)) {
1386 0 : xfs_btree_mark_sick(lcur);
1387 0 : error = -EFSCORRUPTED;
1388 0 : goto error_rcur;
1389 : }
1390 : }
1391 :
1392 110065101 : if (XFS_IS_CORRUPT(lcur->bc_mp, i != 1 && j != 1)) {
1393 0 : xfs_btree_mark_sick(lcur);
1394 0 : error = -EFSCORRUPTED;
1395 0 : goto error_rcur;
1396 : }
1397 110065101 : if (i == 1 && j == 1) {
1398 : /*
1399 : * Both the left and right records are valid. Choose the closer
1400 : * inode chunk to the target.
1401 : */
1402 5816307 : if ((pagino - rec->ir_startino + XFS_INODES_PER_CHUNK - 1) >
1403 5816307 : (rrec.ir_startino - pagino)) {
1404 2923463 : *rec = rrec;
1405 2923463 : xfs_btree_del_cursor(lcur, XFS_BTREE_NOERROR);
1406 2923399 : *ocur = rcur;
1407 : } else {
1408 2892844 : xfs_btree_del_cursor(rcur, XFS_BTREE_NOERROR);
1409 : }
1410 104248794 : } else if (j == 1) {
1411 : /* only the right record is valid */
1412 100477242 : *rec = rrec;
1413 100477242 : xfs_btree_del_cursor(lcur, XFS_BTREE_NOERROR);
1414 100575404 : *ocur = rcur;
1415 3771552 : } else if (i == 1) {
1416 : /* only the left record is valid */
1417 3771556 : xfs_btree_del_cursor(rcur, XFS_BTREE_NOERROR);
1418 : }
1419 :
1420 : return 0;
1421 :
1422 0 : error_rcur:
1423 0 : xfs_btree_del_cursor(rcur, XFS_BTREE_ERROR);
1424 0 : return error;
1425 : }
1426 :
1427 : /*
1428 : * Use the free inode btree to find a free inode based on a newino hint. If
1429 : * the hint is NULL, find the first free inode in the AG.
1430 : */
1431 : STATIC int
1432 10592729 : xfs_dialloc_ag_finobt_newino(
1433 : struct xfs_agi *agi,
1434 : struct xfs_btree_cur *cur,
1435 : struct xfs_inobt_rec_incore *rec)
1436 : {
1437 10592729 : int error;
1438 10592729 : int i;
1439 :
1440 10592729 : if (agi->agi_newino != cpu_to_be32(NULLAGINO)) {
1441 10449406 : error = xfs_inobt_lookup(cur, be32_to_cpu(agi->agi_newino),
1442 : XFS_LOOKUP_EQ, &i);
1443 10450131 : if (error)
1444 : return error;
1445 10450131 : if (i == 1) {
1446 10123787 : error = xfs_inobt_get_rec(cur, rec, &i);
1447 10125421 : if (error)
1448 : return error;
1449 10125421 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1450 0 : xfs_btree_mark_sick(cur);
1451 0 : return -EFSCORRUPTED;
1452 : }
1453 : return 0;
1454 : }
1455 : }
1456 :
1457 : /*
1458 : * Find the first inode available in the AG.
1459 : */
1460 469667 : error = xfs_inobt_lookup(cur, 0, XFS_LOOKUP_GE, &i);
1461 469225 : if (error)
1462 : return error;
1463 469225 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1464 0 : xfs_btree_mark_sick(cur);
1465 0 : return -EFSCORRUPTED;
1466 : }
1467 :
1468 469225 : error = xfs_inobt_get_rec(cur, rec, &i);
1469 469230 : if (error)
1470 : return error;
1471 469230 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1472 0 : xfs_btree_mark_sick(cur);
1473 0 : return -EFSCORRUPTED;
1474 : }
1475 :
1476 : return 0;
1477 : }
1478 :
1479 : /*
1480 : * Update the inobt based on a modification made to the finobt. Also ensure that
1481 : * the records from both trees are equivalent post-modification.
1482 : */
1483 : STATIC int
1484 127555466 : xfs_dialloc_ag_update_inobt(
1485 : struct xfs_btree_cur *cur, /* inobt cursor */
1486 : struct xfs_inobt_rec_incore *frec, /* finobt record */
1487 : int offset) /* inode offset */
1488 : {
1489 127555466 : struct xfs_inobt_rec_incore rec;
1490 127555466 : int error;
1491 127555466 : int i;
1492 :
1493 127555466 : error = xfs_inobt_lookup(cur, frec->ir_startino, XFS_LOOKUP_EQ, &i);
1494 127771305 : if (error)
1495 : return error;
1496 127771265 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1497 0 : xfs_btree_mark_sick(cur);
1498 0 : return -EFSCORRUPTED;
1499 : }
1500 :
1501 127771265 : error = xfs_inobt_get_rec(cur, &rec, &i);
1502 127777061 : if (error)
1503 : return error;
1504 127777061 : if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) {
1505 0 : xfs_btree_mark_sick(cur);
1506 0 : return -EFSCORRUPTED;
1507 : }
1508 127777061 : ASSERT((XFS_AGINO_TO_OFFSET(cur->bc_mp, rec.ir_startino) %
1509 : XFS_INODES_PER_CHUNK) == 0);
1510 :
1511 127777061 : rec.ir_free &= ~XFS_INOBT_MASK(offset);
1512 127777061 : rec.ir_freecount--;
1513 :
1514 127777061 : if (XFS_IS_CORRUPT(cur->bc_mp,
1515 : rec.ir_free != frec->ir_free ||
1516 : rec.ir_freecount != frec->ir_freecount)) {
1517 0 : xfs_btree_mark_sick(cur);
1518 0 : return -EFSCORRUPTED;
1519 : }
1520 :
1521 127777061 : return xfs_inobt_update(cur, &rec);
1522 : }
1523 :
1524 : /*
1525 : * Allocate an inode using the free inode btree, if available. Otherwise, fall
1526 : * back to the inobt search algorithm.
1527 : *
1528 : * The caller selected an AG for us, and made sure that free inodes are
1529 : * available.
1530 : */
1531 : static int
1532 127426756 : xfs_dialloc_ag(
1533 : struct xfs_perag *pag,
1534 : struct xfs_trans *tp,
1535 : struct xfs_buf *agbp,
1536 : xfs_ino_t parent,
1537 : xfs_ino_t *inop)
1538 : {
1539 127426756 : struct xfs_mount *mp = tp->t_mountp;
1540 127426756 : struct xfs_agi *agi = agbp->b_addr;
1541 127426756 : xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent);
1542 127426756 : xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent);
1543 127426756 : struct xfs_btree_cur *cur; /* finobt cursor */
1544 127426756 : struct xfs_btree_cur *icur; /* inobt cursor */
1545 127426756 : struct xfs_inobt_rec_incore rec;
1546 127426756 : xfs_ino_t ino;
1547 127426756 : int error;
1548 127426756 : int offset;
1549 127426756 : int i;
1550 :
1551 127426756 : if (!xfs_has_finobt(mp))
1552 7209 : return xfs_dialloc_ag_inobt(pag, tp, agbp, parent, inop);
1553 :
1554 : /*
1555 : * If pagino is 0 (this is the root inode allocation) use newino.
1556 : * This must work because we've just allocated some.
1557 : */
1558 127419547 : if (!pagino)
1559 20166 : pagino = be32_to_cpu(agi->agi_newino);
1560 :
1561 127419547 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_FINO);
1562 :
1563 127637415 : error = xfs_check_agi_freecount(cur);
1564 126949325 : if (error)
1565 174 : goto error_cur;
1566 :
1567 : /*
1568 : * The search algorithm depends on whether we're in the same AG as the
1569 : * parent. If so, find the closest available inode to the parent. If
1570 : * not, consider the agi hint or find the first free inode in the AG.
1571 : */
1572 126949151 : if (pag->pag_agno == pagno)
1573 116355852 : error = xfs_dialloc_ag_finobt_near(pagino, &cur, &rec);
1574 : else
1575 10593299 : error = xfs_dialloc_ag_finobt_newino(agi, cur, &rec);
1576 127360703 : if (error)
1577 1 : goto error_cur;
1578 :
1579 127360702 : offset = xfs_inobt_first_free_inode(&rec);
1580 127083130 : ASSERT(offset >= 0);
1581 127083130 : ASSERT(offset < XFS_INODES_PER_CHUNK);
1582 127083130 : ASSERT((XFS_AGINO_TO_OFFSET(mp, rec.ir_startino) %
1583 : XFS_INODES_PER_CHUNK) == 0);
1584 127083130 : ino = XFS_AGINO_TO_INO(mp, pag->pag_agno, rec.ir_startino + offset);
1585 :
1586 127083130 : if (xfs_ag_has_sickness(pag, XFS_SICK_AG_INODES)) {
1587 0 : error = xfs_dialloc_check_ino(pag, tp, ino);
1588 0 : if (error)
1589 0 : goto error_cur;
1590 : }
1591 :
1592 : /*
1593 : * Modify or remove the finobt record.
1594 : */
1595 127586309 : rec.ir_free &= ~XFS_INOBT_MASK(offset);
1596 127586309 : rec.ir_freecount--;
1597 127586309 : if (rec.ir_freecount)
1598 105291788 : error = xfs_inobt_update(cur, &rec);
1599 : else
1600 22294521 : error = xfs_btree_delete(cur, &i);
1601 126988868 : if (error)
1602 0 : goto error_cur;
1603 :
1604 : /*
1605 : * The finobt has now been updated appropriately. We haven't updated the
1606 : * agi and superblock yet, so we can create an inobt cursor and validate
1607 : * the original freecount. If all is well, make the equivalent update to
1608 : * the inobt using the finobt record and offset information.
1609 : */
1610 126988868 : icur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_INO);
1611 :
1612 127532120 : error = xfs_check_agi_freecount(icur);
1613 127487113 : if (error)
1614 14 : goto error_icur;
1615 :
1616 127487099 : error = xfs_dialloc_ag_update_inobt(icur, &rec, offset);
1617 127736027 : if (error)
1618 40 : goto error_icur;
1619 :
1620 : /*
1621 : * Both trees have now been updated. We must update the perag and
1622 : * superblock before we can check the freecount for each btree.
1623 : */
1624 127735987 : be32_add_cpu(&agi->agi_freecount, -1);
1625 127735987 : xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT);
1626 127622382 : pag->pagi_freecount--;
1627 :
1628 127622382 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -1);
1629 :
1630 127155570 : error = xfs_check_agi_freecount(icur);
1631 127277523 : if (error)
1632 0 : goto error_icur;
1633 127277523 : error = xfs_check_agi_freecount(cur);
1634 127663337 : if (error)
1635 0 : goto error_icur;
1636 :
1637 127663337 : xfs_btree_del_cursor(icur, XFS_BTREE_NOERROR);
1638 127803639 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
1639 127669760 : *inop = ino;
1640 127669760 : return 0;
1641 :
1642 54 : error_icur:
1643 54 : xfs_btree_del_cursor(icur, XFS_BTREE_ERROR);
1644 229 : error_cur:
1645 229 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
1646 229 : return error;
1647 : }
1648 :
1649 : static int
1650 1368212 : xfs_dialloc_roll(
1651 : struct xfs_trans **tpp,
1652 : struct xfs_buf *agibp)
1653 : {
1654 1368212 : struct xfs_trans *tp = *tpp;
1655 1368212 : struct xfs_dquot_acct *dqinfo;
1656 1368212 : int error;
1657 :
1658 : /*
1659 : * Hold to on to the agibp across the commit so no other allocation can
1660 : * come in and take the free inodes we just allocated for our caller.
1661 : */
1662 1368212 : xfs_trans_bhold(tp, agibp);
1663 :
1664 : /*
1665 : * We want the quota changes to be associated with the next transaction,
1666 : * NOT this one. So, detach the dqinfo from this and attach it to the
1667 : * next transaction.
1668 : */
1669 1368195 : dqinfo = tp->t_dqinfo;
1670 1368195 : tp->t_dqinfo = NULL;
1671 :
1672 1368195 : error = xfs_trans_roll(&tp);
1673 :
1674 : /* Re-attach the quota info that we detached from prev trx. */
1675 1368371 : tp->t_dqinfo = dqinfo;
1676 :
1677 : /*
1678 : * Join the buffer even on commit error so that the buffer is released
1679 : * when the caller cancels the transaction and doesn't have to handle
1680 : * this error case specially.
1681 : */
1682 1368371 : xfs_trans_bjoin(tp, agibp);
1683 1368370 : *tpp = tp;
1684 1368370 : return error;
1685 : }
1686 :
1687 : static bool
1688 232149146 : xfs_dialloc_good_ag(
1689 : struct xfs_perag *pag,
1690 : struct xfs_trans *tp,
1691 : umode_t mode,
1692 : int flags,
1693 : bool ok_alloc)
1694 : {
1695 232149146 : struct xfs_mount *mp = tp->t_mountp;
1696 232149146 : xfs_extlen_t ineed;
1697 232149146 : xfs_extlen_t longest = 0;
1698 232149146 : int needspace;
1699 232149146 : int error;
1700 :
1701 232149146 : if (!pag)
1702 : return false;
1703 464298292 : if (!xfs_perag_allows_inodes(pag))
1704 : return false;
1705 :
1706 464298292 : if (!xfs_perag_initialised_agi(pag)) {
1707 108 : error = xfs_ialloc_read_agi(pag, tp, NULL);
1708 108 : if (error)
1709 : return false;
1710 : }
1711 :
1712 232149146 : if (pag->pagi_freecount)
1713 : return true;
1714 105997344 : if (!ok_alloc)
1715 : return false;
1716 :
1717 6455816 : if (!xfs_perag_initialised_agf(pag)) {
1718 75 : error = xfs_alloc_read_agf(pag, tp, flags, NULL);
1719 75 : if (error)
1720 : return false;
1721 : }
1722 :
1723 : /*
1724 : * Check that there is enough free space for the file plus a chunk of
1725 : * inodes if we need to allocate some. If this is the first pass across
1726 : * the AGs, take into account the potential space needed for alignment
1727 : * of inode chunks when checking the longest contiguous free space in
1728 : * the AG - this prevents us from getting ENOSPC because we have free
1729 : * space larger than ialloc_blks but alignment constraints prevent us
1730 : * from using it.
1731 : *
1732 : * If we can't find an AG with space for full alignment slack to be
1733 : * taken into account, we must be near ENOSPC in all AGs. Hence we
1734 : * don't include alignment for the second pass and so if we fail
1735 : * allocation due to alignment issues then it is most likely a real
1736 : * ENOSPC condition.
1737 : *
1738 : * XXX(dgc): this calculation is now bogus thanks to the per-ag
1739 : * reservations that xfs_alloc_fix_freelist() now does via
1740 : * xfs_alloc_space_available(). When the AG fills up, pagf_freeblks will
1741 : * be more than large enough for the check below to succeed, but
1742 : * xfs_alloc_space_available() will fail because of the non-zero
1743 : * metadata reservation and hence we won't actually be able to allocate
1744 : * more inodes in this AG. We do soooo much unnecessary work near ENOSPC
1745 : * because of this.
1746 : */
1747 3227878 : ineed = M_IGEO(mp)->ialloc_min_blks;
1748 3227878 : if (flags && ineed > 1)
1749 3167179 : ineed += M_IGEO(mp)->cluster_align;
1750 3227878 : longest = pag->pagf_longest;
1751 3227878 : if (!longest)
1752 147 : longest = pag->pagf_flcount > 0;
1753 3227878 : needspace = S_ISDIR(mode) || S_ISREG(mode) || S_ISLNK(mode);
1754 :
1755 3227878 : if (pag->pagf_freeblks < needspace + ineed || longest < ineed)
1756 224280 : return false;
1757 : return true;
1758 : }
1759 :
1760 : static int
1761 129013470 : xfs_dialloc_try_ag(
1762 : struct xfs_perag *pag,
1763 : struct xfs_trans **tpp,
1764 : xfs_ino_t parent,
1765 : xfs_ino_t *new_ino,
1766 : bool ok_alloc)
1767 : {
1768 129013470 : struct xfs_buf *agbp;
1769 129013470 : xfs_ino_t ino;
1770 129013470 : int error;
1771 :
1772 : /*
1773 : * Then read in the AGI buffer and recheck with the AGI buffer
1774 : * lock held.
1775 : */
1776 129013470 : error = xfs_ialloc_read_agi(pag, *tpp, &agbp);
1777 129115737 : if (error)
1778 : return error;
1779 :
1780 129115707 : if (!pag->pagi_freecount) {
1781 2984583 : if (!ok_alloc) {
1782 20506 : error = -EAGAIN;
1783 20506 : goto out_release;
1784 : }
1785 :
1786 2964077 : error = xfs_ialloc_ag_alloc(pag, *tpp, agbp);
1787 2964147 : if (error < 0)
1788 1595725 : goto out_release;
1789 :
1790 : /*
1791 : * We successfully allocated space for an inode cluster in this
1792 : * AG. Roll the transaction so that we can allocate one of the
1793 : * new inodes.
1794 : */
1795 1368422 : ASSERT(pag->pagi_freecount > 0);
1796 1368422 : error = xfs_dialloc_roll(tpp, agbp);
1797 1368361 : if (error)
1798 0 : goto out_release;
1799 : }
1800 :
1801 : /* Allocate an inode in the found AG */
1802 127499485 : error = xfs_dialloc_ag(pag, *tpp, agbp, parent, &ino);
1803 127608778 : if (!error)
1804 127412596 : *new_ino = ino;
1805 : return error;
1806 :
1807 1616231 : out_release:
1808 1616231 : xfs_trans_brelse(*tpp, agbp);
1809 1616231 : return error;
1810 : }
1811 :
1812 : /*
1813 : * Allocate an on-disk inode.
1814 : *
1815 : * Mode is used to tell whether the new inode is a directory and hence where to
1816 : * locate it. The on-disk inode that is allocated will be returned in @new_ino
1817 : * on success, otherwise an error will be set to indicate the failure (e.g.
1818 : * -ENOSPC).
1819 : */
1820 : int
1821 134571877 : xfs_dialloc(
1822 : struct xfs_trans **tpp,
1823 : xfs_ino_t parent,
1824 : umode_t mode,
1825 : xfs_ino_t *new_ino)
1826 : {
1827 134571877 : struct xfs_mount *mp = (*tpp)->t_mountp;
1828 134571877 : xfs_agnumber_t agno;
1829 134571877 : int error = 0;
1830 134571877 : xfs_agnumber_t start_agno;
1831 134571877 : struct xfs_perag *pag;
1832 134571877 : struct xfs_ino_geometry *igeo = M_IGEO(mp);
1833 134571877 : bool ok_alloc = true;
1834 134571877 : bool low_space = false;
1835 134571877 : int flags;
1836 134571877 : xfs_ino_t ino = NULLFSINO;
1837 :
1838 : /*
1839 : * Directories, symlinks, and regular files frequently allocate at least
1840 : * one block, so factor that potential expansion when we examine whether
1841 : * an AG has enough space for file creation.
1842 : */
1843 134571877 : if (S_ISDIR(mode))
1844 12199612 : start_agno = (atomic_inc_return(&mp->m_agirotor) - 1) %
1845 12199957 : mp->m_maxagi;
1846 : else {
1847 122371920 : start_agno = XFS_INO_TO_AGNO(mp, parent);
1848 122371920 : if (start_agno >= mp->m_maxagi)
1849 0 : start_agno = 0;
1850 : }
1851 :
1852 : /*
1853 : * If we have already hit the ceiling of inode blocks then clear
1854 : * ok_alloc so we scan all available agi structures for a free
1855 : * inode.
1856 : *
1857 : * Read rough value of mp->m_icount by percpu_counter_read_positive,
1858 : * which will sacrifice the preciseness but improve the performance.
1859 : */
1860 134571532 : if (igeo->maxicount &&
1861 134245366 : percpu_counter_read_positive(&mp->m_icount) + igeo->ialloc_inos
1862 : > igeo->maxicount) {
1863 7137861 : ok_alloc = false;
1864 : }
1865 :
1866 : /*
1867 : * If we are near to ENOSPC, we want to prefer allocation from AGs that
1868 : * have free inodes in them rather than use up free space allocating new
1869 : * inode chunks. Hence we turn off allocation for the first non-blocking
1870 : * pass through the AGs if we are near ENOSPC to consume free inodes
1871 : * that we can immediately allocate, but then we allow allocation on the
1872 : * second pass if we fail to find an AG with free inodes in it.
1873 : */
1874 134571532 : if (percpu_counter_read_positive(&mp->m_fdblocks) <
1875 134571532 : mp->m_low_space[XFS_LOWSP_1_PCNT]) {
1876 903429 : ok_alloc = false;
1877 903429 : low_space = true;
1878 : }
1879 :
1880 : /*
1881 : * Loop until we find an allocation group that either has free inodes
1882 : * or in which we can allocate some inodes. Iterate through the
1883 : * allocation groups upward, wrapping at the end.
1884 : */
1885 134571532 : flags = XFS_ALLOC_FLAG_TRYLOCK;
1886 141937762 : retry:
1887 246433936 : for_each_perag_wrap_at(mp, start_agno, mp->m_maxagi, agno, pag) {
1888 232209238 : if (xfs_dialloc_good_ag(pag, *tpp, mode, flags, ok_alloc)) {
1889 129069381 : error = xfs_dialloc_try_ag(pag, tpp, parent,
1890 : &ino, ok_alloc);
1891 128947582 : if (error != -EAGAIN)
1892 : break;
1893 : error = 0;
1894 : }
1895 :
1896 208992348 : if (xfs_is_shutdown(mp)) {
1897 : error = -EFSCORRUPTED;
1898 : break;
1899 : }
1900 : }
1901 141990133 : if (pag)
1902 127218133 : xfs_perag_rele(pag);
1903 142250055 : if (error)
1904 287 : return error;
1905 142249768 : if (ino == NULLFSINO) {
1906 14684797 : if (flags) {
1907 7366230 : flags = 0;
1908 7366230 : if (low_space)
1909 22126 : ok_alloc = true;
1910 7366230 : goto retry;
1911 : }
1912 : return -ENOSPC;
1913 : }
1914 127564971 : *new_ino = ino;
1915 127564971 : return 0;
1916 : }
1917 :
1918 : /*
1919 : * Free the blocks of an inode chunk. We must consider that the inode chunk
1920 : * might be sparse and only free the regions that are allocated as part of the
1921 : * chunk.
1922 : */
1923 : static int
1924 388796 : xfs_difree_inode_chunk(
1925 : struct xfs_trans *tp,
1926 : xfs_agnumber_t agno,
1927 : struct xfs_inobt_rec_incore *rec)
1928 : {
1929 388796 : struct xfs_mount *mp = tp->t_mountp;
1930 388796 : xfs_agblock_t sagbno = XFS_AGINO_TO_AGBNO(mp,
1931 : rec->ir_startino);
1932 388796 : int startidx, endidx;
1933 388796 : int nextbit;
1934 388796 : xfs_agblock_t agbno;
1935 388796 : int contigblk;
1936 388796 : DECLARE_BITMAP(holemask, XFS_INOBT_HOLEMASK_BITS);
1937 :
1938 388796 : if (!xfs_inobt_issparse(rec->ir_holemask)) {
1939 : /* not sparse, calculate extent info directly */
1940 648554 : return xfs_free_extent_later(tp,
1941 324277 : XFS_AGB_TO_FSB(mp, agno, sagbno),
1942 324277 : M_IGEO(mp)->ialloc_blks, &XFS_RMAP_OINFO_INODES,
1943 : XFS_AG_RESV_NONE);
1944 : }
1945 :
1946 : /* holemask is only 16-bits (fits in an unsigned long) */
1947 64519 : ASSERT(sizeof(rec->ir_holemask) <= sizeof(holemask[0]));
1948 64519 : holemask[0] = rec->ir_holemask;
1949 :
1950 : /*
1951 : * Find contiguous ranges of zeroes (i.e., allocated regions) in the
1952 : * holemask and convert the start/end index of each range to an extent.
1953 : * We start with the start and end index both pointing at the first 0 in
1954 : * the mask.
1955 : */
1956 64519 : startidx = endidx = find_first_zero_bit(holemask,
1957 : XFS_INOBT_HOLEMASK_BITS);
1958 64517 : nextbit = startidx + 1;
1959 580638 : while (startidx < XFS_INOBT_HOLEMASK_BITS) {
1960 516120 : int error;
1961 :
1962 516120 : nextbit = find_next_zero_bit(holemask, XFS_INOBT_HOLEMASK_BITS,
1963 : nextbit);
1964 : /*
1965 : * If the next zero bit is contiguous, update the end index of
1966 : * the current range and continue.
1967 : */
1968 516123 : if (nextbit != XFS_INOBT_HOLEMASK_BITS &&
1969 451603 : nextbit == endidx + 1) {
1970 451602 : endidx = nextbit;
1971 451602 : goto next;
1972 : }
1973 :
1974 : /*
1975 : * nextbit is not contiguous with the current end index. Convert
1976 : * the current start/end to an extent and add it to the free
1977 : * list.
1978 : */
1979 0 : agbno = sagbno + (startidx * XFS_INODES_PER_HOLEMASK_BIT) /
1980 64521 : mp->m_sb.sb_inopblock;
1981 129042 : contigblk = ((endidx - startidx + 1) *
1982 0 : XFS_INODES_PER_HOLEMASK_BIT) /
1983 64521 : mp->m_sb.sb_inopblock;
1984 :
1985 64521 : ASSERT(agbno % mp->m_sb.sb_spino_align == 0);
1986 64521 : ASSERT(contigblk % mp->m_sb.sb_spino_align == 0);
1987 129042 : error = xfs_free_extent_later(tp,
1988 64521 : XFS_AGB_TO_FSB(mp, agno, agbno), contigblk,
1989 : &XFS_RMAP_OINFO_INODES, XFS_AG_RESV_NONE);
1990 64519 : if (error)
1991 0 : return error;
1992 :
1993 : /* reset range to current bit and carry on... */
1994 : startidx = endidx = nextbit;
1995 :
1996 516121 : next:
1997 516121 : nextbit++;
1998 : }
1999 : return 0;
2000 : }
2001 :
2002 : STATIC int
2003 85747293 : xfs_difree_inobt(
2004 : struct xfs_perag *pag,
2005 : struct xfs_trans *tp,
2006 : struct xfs_buf *agbp,
2007 : xfs_agino_t agino,
2008 : struct xfs_icluster *xic,
2009 : struct xfs_inobt_rec_incore *orec)
2010 : {
2011 85747293 : struct xfs_mount *mp = pag->pag_mount;
2012 85747293 : struct xfs_agi *agi = agbp->b_addr;
2013 85747293 : struct xfs_btree_cur *cur;
2014 85747293 : struct xfs_inobt_rec_incore rec;
2015 85747293 : int ilen;
2016 85747293 : int error;
2017 85747293 : int i;
2018 85747293 : int off;
2019 :
2020 85747293 : ASSERT(agi->agi_magicnum == cpu_to_be32(XFS_AGI_MAGIC));
2021 85747293 : ASSERT(XFS_AGINO_TO_AGBNO(mp, agino) < be32_to_cpu(agi->agi_length));
2022 :
2023 : /*
2024 : * Initialize the cursor.
2025 : */
2026 85747293 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_INO);
2027 :
2028 85801753 : error = xfs_check_agi_freecount(cur);
2029 85769342 : if (error)
2030 3 : goto error0;
2031 :
2032 : /*
2033 : * Look for the entry describing this inode.
2034 : */
2035 85769339 : if ((error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &i))) {
2036 4 : xfs_warn(mp, "%s: xfs_inobt_lookup() returned error %d.",
2037 : __func__, error);
2038 4 : goto error0;
2039 : }
2040 85805313 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2041 0 : xfs_btree_mark_sick(cur);
2042 0 : error = -EFSCORRUPTED;
2043 0 : goto error0;
2044 : }
2045 85805313 : error = xfs_inobt_get_rec(cur, &rec, &i);
2046 85778412 : if (error) {
2047 0 : xfs_warn(mp, "%s: xfs_inobt_get_rec() returned error %d.",
2048 : __func__, error);
2049 0 : goto error0;
2050 : }
2051 85778412 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2052 0 : xfs_btree_mark_sick(cur);
2053 0 : error = -EFSCORRUPTED;
2054 0 : goto error0;
2055 : }
2056 : /*
2057 : * Get the offset in the inode chunk.
2058 : */
2059 85778412 : off = agino - rec.ir_startino;
2060 85778412 : ASSERT(off >= 0 && off < XFS_INODES_PER_CHUNK);
2061 85778412 : ASSERT(!(rec.ir_free & XFS_INOBT_MASK(off)));
2062 : /*
2063 : * Mark the inode free & increment the count.
2064 : */
2065 85778412 : rec.ir_free |= XFS_INOBT_MASK(off);
2066 85778412 : rec.ir_freecount++;
2067 :
2068 : /*
2069 : * When an inode chunk is free, it becomes eligible for removal. Don't
2070 : * remove the chunk if the block size is large enough for multiple inode
2071 : * chunks (that might not be free).
2072 : */
2073 85778412 : if (!xfs_has_ikeep(mp) && rec.ir_free == XFS_INOBT_ALL_FREE &&
2074 388814 : mp->m_sb.sb_inopblock <= XFS_INODES_PER_CHUNK) {
2075 388814 : xic->deleted = true;
2076 388814 : xic->first_ino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
2077 : rec.ir_startino);
2078 388814 : xic->alloc = xfs_inobt_irec_to_allocmask(&rec);
2079 :
2080 : /*
2081 : * Remove the inode cluster from the AGI B+Tree, adjust the
2082 : * AGI and Superblock inode counts, and mark the disk space
2083 : * to be freed when the transaction is committed.
2084 : */
2085 388793 : ilen = rec.ir_freecount;
2086 388793 : be32_add_cpu(&agi->agi_count, -ilen);
2087 388793 : be32_add_cpu(&agi->agi_freecount, -(ilen - 1));
2088 388793 : xfs_ialloc_log_agi(tp, agbp, XFS_AGI_COUNT | XFS_AGI_FREECOUNT);
2089 388803 : pag->pagi_freecount -= ilen - 1;
2090 388803 : pag->pagi_count -= ilen;
2091 388803 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_ICOUNT, -ilen);
2092 388782 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -(ilen - 1));
2093 :
2094 388787 : if ((error = xfs_btree_delete(cur, &i))) {
2095 0 : xfs_warn(mp, "%s: xfs_btree_delete returned error %d.",
2096 : __func__, error);
2097 0 : goto error0;
2098 : }
2099 :
2100 388794 : error = xfs_difree_inode_chunk(tp, pag->pag_agno, &rec);
2101 388780 : if (error)
2102 0 : goto error0;
2103 : } else {
2104 85389598 : xic->deleted = false;
2105 :
2106 85389598 : error = xfs_inobt_update(cur, &rec);
2107 85324192 : if (error) {
2108 0 : xfs_warn(mp, "%s: xfs_inobt_update returned error %d.",
2109 : __func__, error);
2110 0 : goto error0;
2111 : }
2112 :
2113 : /*
2114 : * Change the inode free counts and log the ag/sb changes.
2115 : */
2116 85324192 : be32_add_cpu(&agi->agi_freecount, 1);
2117 85324192 : xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT);
2118 85386887 : pag->pagi_freecount++;
2119 85386887 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, 1);
2120 : }
2121 :
2122 85655240 : error = xfs_check_agi_freecount(cur);
2123 85712221 : if (error)
2124 0 : goto error0;
2125 :
2126 85712221 : *orec = rec;
2127 85712221 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
2128 85712221 : return 0;
2129 :
2130 7 : error0:
2131 7 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
2132 7 : return error;
2133 : }
2134 :
2135 : /*
2136 : * Free an inode in the free inode btree.
2137 : */
2138 : STATIC int
2139 85673000 : xfs_difree_finobt(
2140 : struct xfs_perag *pag,
2141 : struct xfs_trans *tp,
2142 : struct xfs_buf *agbp,
2143 : xfs_agino_t agino,
2144 : struct xfs_inobt_rec_incore *ibtrec) /* inobt record */
2145 : {
2146 85673000 : struct xfs_mount *mp = pag->pag_mount;
2147 85673000 : struct xfs_btree_cur *cur;
2148 85673000 : struct xfs_inobt_rec_incore rec;
2149 85673000 : int offset = agino - ibtrec->ir_startino;
2150 85673000 : int error;
2151 85673000 : int i;
2152 :
2153 85673000 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_FINO);
2154 :
2155 85767569 : error = xfs_inobt_lookup(cur, ibtrec->ir_startino, XFS_LOOKUP_EQ, &i);
2156 85815680 : if (error)
2157 4 : goto error;
2158 85815676 : if (i == 0) {
2159 : /*
2160 : * If the record does not exist in the finobt, we must have just
2161 : * freed an inode in a previously fully allocated chunk. If not,
2162 : * something is out of sync.
2163 : */
2164 21341622 : if (XFS_IS_CORRUPT(mp, ibtrec->ir_freecount != 1)) {
2165 0 : xfs_btree_mark_sick(cur);
2166 0 : error = -EFSCORRUPTED;
2167 0 : goto error;
2168 : }
2169 :
2170 21341622 : error = xfs_inobt_insert_rec(cur, ibtrec->ir_holemask,
2171 21341622 : ibtrec->ir_count,
2172 : ibtrec->ir_freecount,
2173 : ibtrec->ir_free, &i);
2174 21322749 : if (error)
2175 0 : goto error;
2176 21322749 : ASSERT(i == 1);
2177 :
2178 21322749 : goto out;
2179 : }
2180 :
2181 : /*
2182 : * Read and update the existing record. We could just copy the ibtrec
2183 : * across here, but that would defeat the purpose of having redundant
2184 : * metadata. By making the modifications independently, we can catch
2185 : * corruptions that we wouldn't see if we just copied from one record
2186 : * to another.
2187 : */
2188 64474054 : error = xfs_inobt_get_rec(cur, &rec, &i);
2189 64461098 : if (error)
2190 0 : goto error;
2191 64461098 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2192 0 : xfs_btree_mark_sick(cur);
2193 0 : error = -EFSCORRUPTED;
2194 0 : goto error;
2195 : }
2196 :
2197 64461098 : rec.ir_free |= XFS_INOBT_MASK(offset);
2198 64461098 : rec.ir_freecount++;
2199 :
2200 64461098 : if (XFS_IS_CORRUPT(mp,
2201 : rec.ir_free != ibtrec->ir_free ||
2202 : rec.ir_freecount != ibtrec->ir_freecount)) {
2203 0 : xfs_btree_mark_sick(cur);
2204 0 : error = -EFSCORRUPTED;
2205 0 : goto error;
2206 : }
2207 :
2208 : /*
2209 : * The content of inobt records should always match between the inobt
2210 : * and finobt. The lifecycle of records in the finobt is different from
2211 : * the inobt in that the finobt only tracks records with at least one
2212 : * free inode. Hence, if all of the inodes are free and we aren't
2213 : * keeping inode chunks permanently on disk, remove the record.
2214 : * Otherwise, update the record with the new information.
2215 : *
2216 : * Note that we currently can't free chunks when the block size is large
2217 : * enough for multiple chunks. Leave the finobt record to remain in sync
2218 : * with the inobt.
2219 : */
2220 64461098 : if (!xfs_has_ikeep(mp) && rec.ir_free == XFS_INOBT_ALL_FREE &&
2221 388811 : mp->m_sb.sb_inopblock <= XFS_INODES_PER_CHUNK) {
2222 388810 : error = xfs_btree_delete(cur, &i);
2223 388804 : if (error)
2224 0 : goto error;
2225 388804 : ASSERT(i == 1);
2226 : } else {
2227 64072288 : error = xfs_inobt_update(cur, &rec);
2228 64067004 : if (error)
2229 0 : goto error;
2230 : }
2231 :
2232 64067004 : out:
2233 85778557 : error = xfs_check_agi_freecount(cur);
2234 85816328 : if (error)
2235 0 : goto error;
2236 :
2237 85816328 : xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
2238 85816328 : return 0;
2239 :
2240 4 : error:
2241 4 : xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
2242 4 : return error;
2243 : }
2244 :
2245 : /*
2246 : * Free disk inode. Carefully avoids touching the incore inode, all
2247 : * manipulations incore are the caller's responsibility.
2248 : * The on-disk inode is not changed by this operation, only the
2249 : * btree (free inode mask) is changed.
2250 : */
2251 : int
2252 85728920 : xfs_difree(
2253 : struct xfs_trans *tp,
2254 : struct xfs_perag *pag,
2255 : xfs_ino_t inode,
2256 : struct xfs_icluster *xic)
2257 : {
2258 : /* REFERENCED */
2259 85728920 : xfs_agblock_t agbno; /* block number containing inode */
2260 85728920 : struct xfs_buf *agbp; /* buffer for allocation group header */
2261 85728920 : xfs_agino_t agino; /* allocation group inode number */
2262 85728920 : int error; /* error return value */
2263 85728920 : struct xfs_mount *mp = tp->t_mountp;
2264 85728920 : struct xfs_inobt_rec_incore rec;/* btree record */
2265 :
2266 : /*
2267 : * Break up inode number into its components.
2268 : */
2269 85728920 : if (pag->pag_agno != XFS_INO_TO_AGNO(mp, inode)) {
2270 0 : xfs_warn(mp, "%s: agno != pag->pag_agno (%d != %d).",
2271 : __func__, XFS_INO_TO_AGNO(mp, inode), pag->pag_agno);
2272 0 : ASSERT(0);
2273 0 : return -EINVAL;
2274 : }
2275 85728920 : agino = XFS_INO_TO_AGINO(mp, inode);
2276 85728920 : if (inode != XFS_AGINO_TO_INO(mp, pag->pag_agno, agino)) {
2277 0 : xfs_warn(mp, "%s: inode != XFS_AGINO_TO_INO() (%llu != %llu).",
2278 : __func__, (unsigned long long)inode,
2279 : (unsigned long long)XFS_AGINO_TO_INO(mp, pag->pag_agno, agino));
2280 0 : ASSERT(0);
2281 0 : return -EINVAL;
2282 : }
2283 85728920 : agbno = XFS_AGINO_TO_AGBNO(mp, agino);
2284 85728920 : if (agbno >= mp->m_sb.sb_agblocks) {
2285 0 : xfs_warn(mp, "%s: agbno >= mp->m_sb.sb_agblocks (%d >= %d).",
2286 : __func__, agbno, mp->m_sb.sb_agblocks);
2287 0 : ASSERT(0);
2288 0 : return -EINVAL;
2289 : }
2290 : /*
2291 : * Get the allocation group header.
2292 : */
2293 85728920 : error = xfs_ialloc_read_agi(pag, tp, &agbp);
2294 85769757 : if (error) {
2295 136 : xfs_warn(mp, "%s: xfs_ialloc_read_agi() returned error %d.",
2296 : __func__, error);
2297 136 : return error;
2298 : }
2299 :
2300 : /*
2301 : * Fix up the inode allocation btree.
2302 : */
2303 85769621 : error = xfs_difree_inobt(pag, tp, agbp, agino, xic, &rec);
2304 85764301 : if (error)
2305 7 : goto error0;
2306 :
2307 : /*
2308 : * Fix up the free inode btree.
2309 : */
2310 85764294 : if (xfs_has_finobt(mp)) {
2311 85742297 : error = xfs_difree_finobt(pag, tp, agbp, agino, &rec);
2312 85829971 : if (error)
2313 4 : goto error0;
2314 : }
2315 :
2316 : return 0;
2317 :
2318 : error0:
2319 : return error;
2320 : }
2321 :
2322 : STATIC int
2323 406054625 : xfs_imap_lookup(
2324 : struct xfs_perag *pag,
2325 : struct xfs_trans *tp,
2326 : xfs_agino_t agino,
2327 : xfs_agblock_t agbno,
2328 : xfs_agblock_t *chunk_agbno,
2329 : xfs_agblock_t *offset_agbno,
2330 : int flags)
2331 : {
2332 406054625 : struct xfs_mount *mp = pag->pag_mount;
2333 406054625 : struct xfs_inobt_rec_incore rec;
2334 406054625 : struct xfs_btree_cur *cur;
2335 406054625 : struct xfs_buf *agbp;
2336 406054625 : int error;
2337 406054625 : int i;
2338 :
2339 406054625 : error = xfs_ialloc_read_agi(pag, tp, &agbp);
2340 406056883 : if (error) {
2341 121 : xfs_alert(mp,
2342 : "%s: xfs_ialloc_read_agi() returned error %d, agno %d",
2343 : __func__, error, pag->pag_agno);
2344 121 : return error;
2345 : }
2346 :
2347 : /*
2348 : * Lookup the inode record for the given agino. If the record cannot be
2349 : * found, then it's an invalid inode number and we should abort. Once
2350 : * we have a record, we need to ensure it contains the inode number
2351 : * we are looking up.
2352 : */
2353 406056762 : cur = xfs_inobt_init_cursor(pag, tp, agbp, XFS_BTNUM_INO);
2354 406063993 : error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &i);
2355 406066379 : if (!error) {
2356 406066056 : if (i)
2357 406064213 : error = xfs_inobt_get_rec(cur, &rec, &i);
2358 406053018 : if (!error && i == 0)
2359 865 : error = -EINVAL;
2360 : }
2361 :
2362 406053341 : xfs_trans_brelse(tp, agbp);
2363 406059463 : xfs_btree_del_cursor(cur, error);
2364 406069052 : if (error)
2365 : return error;
2366 :
2367 : /* check that the returned record contains the required inode */
2368 406068174 : if (rec.ir_startino > agino ||
2369 406068174 : rec.ir_startino + M_IGEO(mp)->ialloc_inos <= agino)
2370 : return -EINVAL;
2371 :
2372 : /* for untrusted inodes check it is allocated first */
2373 405988608 : if ((flags & XFS_IGET_UNTRUSTED) &&
2374 405988951 : (rec.ir_free & XFS_INOBT_MASK(agino - rec.ir_startino)))
2375 : return -EINVAL;
2376 :
2377 405983736 : *chunk_agbno = XFS_AGINO_TO_AGBNO(mp, rec.ir_startino);
2378 405983736 : *offset_agbno = agbno - *chunk_agbno;
2379 405983736 : return 0;
2380 : }
2381 :
2382 : /*
2383 : * Return the location of the inode in imap, for mapping it into a buffer.
2384 : */
2385 : int
2386 479926407 : xfs_imap(
2387 : struct xfs_perag *pag,
2388 : struct xfs_trans *tp,
2389 : xfs_ino_t ino, /* inode to locate */
2390 : struct xfs_imap *imap, /* location map structure */
2391 : uint flags) /* flags for inode btree lookup */
2392 : {
2393 479926407 : struct xfs_mount *mp = pag->pag_mount;
2394 479926407 : xfs_agblock_t agbno; /* block number of inode in the alloc group */
2395 479926407 : xfs_agino_t agino; /* inode number within alloc group */
2396 479926407 : xfs_agblock_t chunk_agbno; /* first block in inode chunk */
2397 479926407 : xfs_agblock_t cluster_agbno; /* first block in inode cluster */
2398 479926407 : int error; /* error code */
2399 479926407 : int offset; /* index of inode in its buffer */
2400 479926407 : xfs_agblock_t offset_agbno; /* blks from chunk start to inode */
2401 :
2402 479926407 : ASSERT(ino != NULLFSINO);
2403 :
2404 : /*
2405 : * Split up the inode number into its parts.
2406 : */
2407 479926407 : agino = XFS_INO_TO_AGINO(mp, ino);
2408 479926407 : agbno = XFS_AGINO_TO_AGBNO(mp, agino);
2409 479926407 : if (agbno >= mp->m_sb.sb_agblocks ||
2410 480013399 : ino != XFS_AGINO_TO_INO(mp, pag->pag_agno, agino)) {
2411 26356 : error = -EINVAL;
2412 : #ifdef DEBUG
2413 : /*
2414 : * Don't output diagnostic information for untrusted inodes
2415 : * as they can be invalid without implying corruption.
2416 : */
2417 26356 : if (flags & XFS_IGET_UNTRUSTED)
2418 : return error;
2419 0 : if (agbno >= mp->m_sb.sb_agblocks) {
2420 0 : xfs_alert(mp,
2421 : "%s: agbno (0x%llx) >= mp->m_sb.sb_agblocks (0x%lx)",
2422 : __func__, (unsigned long long)agbno,
2423 : (unsigned long)mp->m_sb.sb_agblocks);
2424 : }
2425 0 : if (ino != XFS_AGINO_TO_INO(mp, pag->pag_agno, agino)) {
2426 0 : xfs_alert(mp,
2427 : "%s: ino (0x%llx) != XFS_AGINO_TO_INO() (0x%llx)",
2428 : __func__, ino,
2429 : XFS_AGINO_TO_INO(mp, pag->pag_agno, agino));
2430 : }
2431 0 : xfs_stack_trace();
2432 : #endif /* DEBUG */
2433 0 : return error;
2434 : }
2435 :
2436 : /*
2437 : * For bulkstat and handle lookups, we have an untrusted inode number
2438 : * that we have to verify is valid. We cannot do this just by reading
2439 : * the inode buffer as it may have been unlinked and removed leaving
2440 : * inodes in stale state on disk. Hence we have to do a btree lookup
2441 : * in all cases where an untrusted inode number is passed.
2442 : */
2443 479900051 : if (flags & XFS_IGET_UNTRUSTED) {
2444 406051643 : error = xfs_imap_lookup(pag, tp, agino, agbno,
2445 : &chunk_agbno, &offset_agbno, flags);
2446 406067543 : if (error)
2447 : return error;
2448 405981580 : goto out_map;
2449 : }
2450 :
2451 : /*
2452 : * If the inode cluster size is the same as the blocksize or
2453 : * smaller we get to the buffer by simple arithmetics.
2454 : */
2455 73848408 : if (M_IGEO(mp)->blocks_per_cluster == 1) {
2456 0 : offset = XFS_INO_TO_OFFSET(mp, ino);
2457 0 : ASSERT(offset < mp->m_sb.sb_inopblock);
2458 :
2459 0 : imap->im_blkno = XFS_AGB_TO_DADDR(mp, pag->pag_agno, agbno);
2460 0 : imap->im_len = XFS_FSB_TO_BB(mp, 1);
2461 0 : imap->im_boffset = (unsigned short)(offset <<
2462 0 : mp->m_sb.sb_inodelog);
2463 0 : return 0;
2464 : }
2465 :
2466 : /*
2467 : * If the inode chunks are aligned then use simple maths to
2468 : * find the location. Otherwise we have to do a btree
2469 : * lookup to find the location.
2470 : */
2471 73848408 : if (M_IGEO(mp)->inoalign_mask) {
2472 73848408 : offset_agbno = agbno & M_IGEO(mp)->inoalign_mask;
2473 73848408 : chunk_agbno = agbno - offset_agbno;
2474 : } else {
2475 0 : error = xfs_imap_lookup(pag, tp, agino, agbno,
2476 : &chunk_agbno, &offset_agbno, flags);
2477 0 : if (error)
2478 : return error;
2479 : }
2480 :
2481 0 : out_map:
2482 479829988 : ASSERT(agbno >= chunk_agbno);
2483 959659976 : cluster_agbno = chunk_agbno +
2484 479829988 : ((offset_agbno / M_IGEO(mp)->blocks_per_cluster) *
2485 479829988 : M_IGEO(mp)->blocks_per_cluster);
2486 959659976 : offset = ((agbno - cluster_agbno) * mp->m_sb.sb_inopblock) +
2487 479829988 : XFS_INO_TO_OFFSET(mp, ino);
2488 :
2489 479829988 : imap->im_blkno = XFS_AGB_TO_DADDR(mp, pag->pag_agno, cluster_agbno);
2490 479829988 : imap->im_len = XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster);
2491 479829988 : imap->im_boffset = (unsigned short)(offset << mp->m_sb.sb_inodelog);
2492 :
2493 : /*
2494 : * If the inode number maps to a block outside the bounds
2495 : * of the file system then return NULL rather than calling
2496 : * read_buf and panicing when we get an error from the
2497 : * driver.
2498 : */
2499 959659976 : if ((imap->im_blkno + imap->im_len) >
2500 479829988 : XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks)) {
2501 0 : xfs_alert(mp,
2502 : "%s: (im_blkno (0x%llx) + im_len (0x%llx)) > sb_dblocks (0x%llx)",
2503 : __func__, (unsigned long long) imap->im_blkno,
2504 : (unsigned long long) imap->im_len,
2505 : XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks));
2506 0 : return -EINVAL;
2507 : }
2508 : return 0;
2509 : }
2510 :
2511 : /*
2512 : * Log specified fields for the ag hdr (inode section). The growth of the agi
2513 : * structure over time requires that we interpret the buffer as two logical
2514 : * regions delineated by the end of the unlinked list. This is due to the size
2515 : * of the hash table and its location in the middle of the agi.
2516 : *
2517 : * For example, a request to log a field before agi_unlinked and a field after
2518 : * agi_unlinked could cause us to log the entire hash table and use an excessive
2519 : * amount of log space. To avoid this behavior, log the region up through
2520 : * agi_unlinked in one call and the region after agi_unlinked through the end of
2521 : * the structure in another.
2522 : */
2523 : void
2524 215465203 : xfs_ialloc_log_agi(
2525 : struct xfs_trans *tp,
2526 : struct xfs_buf *bp,
2527 : uint32_t fields)
2528 : {
2529 215465203 : int first; /* first byte number */
2530 215465203 : int last; /* last byte number */
2531 215465203 : static const short offsets[] = { /* field starting offsets */
2532 : /* keep in sync with bit definitions */
2533 : offsetof(xfs_agi_t, agi_magicnum),
2534 : offsetof(xfs_agi_t, agi_versionnum),
2535 : offsetof(xfs_agi_t, agi_seqno),
2536 : offsetof(xfs_agi_t, agi_length),
2537 : offsetof(xfs_agi_t, agi_count),
2538 : offsetof(xfs_agi_t, agi_root),
2539 : offsetof(xfs_agi_t, agi_level),
2540 : offsetof(xfs_agi_t, agi_freecount),
2541 : offsetof(xfs_agi_t, agi_newino),
2542 : offsetof(xfs_agi_t, agi_dirino),
2543 : offsetof(xfs_agi_t, agi_unlinked),
2544 : offsetof(xfs_agi_t, agi_free_root),
2545 : offsetof(xfs_agi_t, agi_free_level),
2546 : offsetof(xfs_agi_t, agi_iblocks),
2547 : sizeof(xfs_agi_t)
2548 : };
2549 : #ifdef DEBUG
2550 215465203 : struct xfs_agi *agi = bp->b_addr;
2551 :
2552 215465203 : ASSERT(agi->agi_magicnum == cpu_to_be32(XFS_AGI_MAGIC));
2553 : #endif
2554 :
2555 : /*
2556 : * Compute byte offsets for the first and last fields in the first
2557 : * region and log the agi buffer. This only logs up through
2558 : * agi_unlinked.
2559 : */
2560 215465203 : if (fields & XFS_AGI_ALL_BITS_R1) {
2561 215325049 : xfs_btree_offsets(fields, offsets, XFS_AGI_NUM_BITS_R1,
2562 : &first, &last);
2563 214999224 : xfs_trans_log_buf(tp, bp, first, last);
2564 : }
2565 :
2566 : /*
2567 : * Mask off the bits in the first region and calculate the first and
2568 : * last field offsets for any bits in the second region.
2569 : */
2570 215781020 : fields &= ~XFS_AGI_ALL_BITS_R1;
2571 215781020 : if (fields) {
2572 156872 : xfs_btree_offsets(fields, offsets, XFS_AGI_NUM_BITS_R2,
2573 : &first, &last);
2574 156862 : xfs_trans_log_buf(tp, bp, first, last);
2575 : }
2576 215781073 : }
2577 :
2578 : static xfs_failaddr_t
2579 2215609 : xfs_agi_verify(
2580 : struct xfs_buf *bp)
2581 : {
2582 2215609 : struct xfs_mount *mp = bp->b_mount;
2583 2215609 : struct xfs_agi *agi = bp->b_addr;
2584 2215609 : xfs_failaddr_t fa;
2585 2215609 : uint32_t agi_seqno = be32_to_cpu(agi->agi_seqno);
2586 2215609 : uint32_t agi_length = be32_to_cpu(agi->agi_length);
2587 2215609 : int i;
2588 :
2589 2215609 : if (xfs_has_crc(mp)) {
2590 2200950 : if (!uuid_equal(&agi->agi_uuid, &mp->m_sb.sb_meta_uuid))
2591 0 : return __this_address;
2592 2201365 : if (!xfs_log_check_lsn(mp, be64_to_cpu(agi->agi_lsn)))
2593 0 : return __this_address;
2594 : }
2595 :
2596 : /*
2597 : * Validate the magic number of the agi block.
2598 : */
2599 2216100 : if (!xfs_verify_magic(bp, agi->agi_magicnum))
2600 0 : return __this_address;
2601 2215514 : if (!XFS_AGI_GOOD_VERSION(be32_to_cpu(agi->agi_versionnum)))
2602 0 : return __this_address;
2603 :
2604 2215514 : fa = xfs_validate_ag_length(bp, agi_seqno, agi_length);
2605 2215214 : if (fa)
2606 : return fa;
2607 :
2608 2215246 : if (be32_to_cpu(agi->agi_level) < 1 ||
2609 2215246 : be32_to_cpu(agi->agi_level) > M_IGEO(mp)->inobt_maxlevels)
2610 0 : return __this_address;
2611 :
2612 2215246 : if (xfs_has_finobt(mp) &&
2613 2200903 : (be32_to_cpu(agi->agi_free_level) < 1 ||
2614 : be32_to_cpu(agi->agi_free_level) > M_IGEO(mp)->inobt_maxlevels))
2615 0 : return __this_address;
2616 :
2617 143959663 : for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) {
2618 141743653 : if (agi->agi_unlinked[i] == cpu_to_be32(NULLAGINO))
2619 141536666 : continue;
2620 207751 : if (!xfs_verify_ino(mp, be32_to_cpu(agi->agi_unlinked[i])))
2621 0 : return __this_address;
2622 : }
2623 :
2624 : return NULL;
2625 : }
2626 :
2627 : static void
2628 823295 : xfs_agi_read_verify(
2629 : struct xfs_buf *bp)
2630 : {
2631 823295 : struct xfs_mount *mp = bp->b_mount;
2632 823295 : xfs_failaddr_t fa;
2633 :
2634 1646131 : if (xfs_has_crc(mp) &&
2635 : !xfs_buf_verify_cksum(bp, XFS_AGI_CRC_OFF))
2636 10 : xfs_verifier_error(bp, -EFSBADCRC, __this_address);
2637 : else {
2638 823285 : fa = xfs_agi_verify(bp);
2639 823285 : if (XFS_TEST_ERROR(fa, mp, XFS_ERRTAG_IALLOC_READ_AGI))
2640 0 : xfs_verifier_error(bp, -EFSCORRUPTED, fa);
2641 : }
2642 823295 : }
2643 :
2644 : static void
2645 769104 : xfs_agi_write_verify(
2646 : struct xfs_buf *bp)
2647 : {
2648 769104 : struct xfs_mount *mp = bp->b_mount;
2649 769104 : struct xfs_buf_log_item *bip = bp->b_log_item;
2650 769104 : struct xfs_agi *agi = bp->b_addr;
2651 769104 : xfs_failaddr_t fa;
2652 :
2653 769104 : fa = xfs_agi_verify(bp);
2654 769104 : if (fa) {
2655 0 : xfs_verifier_error(bp, -EFSCORRUPTED, fa);
2656 0 : return;
2657 : }
2658 :
2659 769104 : if (!xfs_has_crc(mp))
2660 : return;
2661 :
2662 755452 : if (bip)
2663 737444 : agi->agi_lsn = cpu_to_be64(bip->bli_item.li_lsn);
2664 755452 : xfs_buf_update_cksum(bp, XFS_AGI_CRC_OFF);
2665 : }
2666 :
2667 : const struct xfs_buf_ops xfs_agi_buf_ops = {
2668 : .name = "xfs_agi",
2669 : .magic = { cpu_to_be32(XFS_AGI_MAGIC), cpu_to_be32(XFS_AGI_MAGIC) },
2670 : .verify_read = xfs_agi_read_verify,
2671 : .verify_write = xfs_agi_write_verify,
2672 : .verify_struct = xfs_agi_verify,
2673 : };
2674 :
2675 : /*
2676 : * Read in the allocation group header (inode allocation section)
2677 : */
2678 : int
2679 2634438948 : xfs_read_agi(
2680 : struct xfs_perag *pag,
2681 : struct xfs_trans *tp,
2682 : struct xfs_buf **agibpp)
2683 : {
2684 2634438948 : struct xfs_mount *mp = pag->pag_mount;
2685 2634438948 : int error;
2686 :
2687 2634438948 : trace_xfs_read_agi(pag->pag_mount, pag->pag_agno);
2688 :
2689 10531559156 : error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp,
2690 2632889789 : XFS_AG_DADDR(mp, pag->pag_agno, XFS_AGI_DADDR(mp)),
2691 2632889789 : XFS_FSS_TO_BB(mp, 1), 0, agibpp, &xfs_agi_buf_ops);
2692 2636418815 : if (xfs_metadata_is_sick(error))
2693 10 : xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI);
2694 2636418815 : if (error)
2695 : return error;
2696 2636412081 : if (tp)
2697 2635244829 : xfs_trans_buf_set_type(tp, *agibpp, XFS_BLFT_AGI_BUF);
2698 :
2699 2635818810 : xfs_buf_set_ref(*agibpp, XFS_AGI_REF);
2700 2635818810 : return 0;
2701 : }
2702 :
2703 : /*
2704 : * Read in the agi and initialise the per-ag data. If the caller supplies a
2705 : * @agibpp, return the locked AGI buffer to them, otherwise release it.
2706 : */
2707 : int
2708 2454548024 : xfs_ialloc_read_agi(
2709 : struct xfs_perag *pag,
2710 : struct xfs_trans *tp,
2711 : struct xfs_buf **agibpp)
2712 : {
2713 2454548024 : struct xfs_buf *agibp;
2714 2454548024 : struct xfs_agi *agi;
2715 2454548024 : int error;
2716 :
2717 2454548024 : trace_xfs_ialloc_read_agi(pag->pag_mount, pag->pag_agno);
2718 :
2719 2453054735 : error = xfs_read_agi(pag, tp, &agibp);
2720 2456088873 : if (error)
2721 : return error;
2722 :
2723 2456082206 : agi = agibp->b_addr;
2724 4912164412 : if (!xfs_perag_initialised_agi(pag)) {
2725 512022 : pag->pagi_freecount = be32_to_cpu(agi->agi_freecount);
2726 512022 : pag->pagi_count = be32_to_cpu(agi->agi_count);
2727 512022 : set_bit(XFS_AGSTATE_AGI_INIT, &pag->pag_opstate);
2728 : }
2729 :
2730 : /*
2731 : * It's possible for these to be out of sync if
2732 : * we are in the middle of a forced shutdown.
2733 : */
2734 2456082254 : ASSERT(pag->pagi_freecount == be32_to_cpu(agi->agi_freecount) ||
2735 : xfs_is_shutdown(pag->pag_mount));
2736 2456082254 : if (agibpp)
2737 2456022748 : *agibpp = agibp;
2738 : else
2739 59506 : xfs_trans_brelse(tp, agibp);
2740 : return 0;
2741 : }
2742 :
2743 : /* How many inodes are backed by inode clusters ondisk? */
2744 : STATIC int
2745 2867769987 : xfs_ialloc_count_ondisk(
2746 : struct xfs_btree_cur *cur,
2747 : xfs_agino_t low,
2748 : xfs_agino_t high,
2749 : unsigned int *allocated)
2750 : {
2751 2867769987 : struct xfs_inobt_rec_incore irec;
2752 2867769987 : unsigned int ret = 0;
2753 2867769987 : int has_record;
2754 2867769987 : int error;
2755 :
2756 2867769987 : error = xfs_inobt_lookup(cur, low, XFS_LOOKUP_LE, &has_record);
2757 2866704616 : if (error)
2758 : return error;
2759 :
2760 5331831128 : while (has_record) {
2761 3241611577 : unsigned int i, hole_idx;
2762 :
2763 3241611577 : error = xfs_inobt_get_rec(cur, &irec, &has_record);
2764 3241462394 : if (error)
2765 0 : return error;
2766 3241462394 : if (irec.ir_startino > high)
2767 : break;
2768 :
2769 >15950*10^7 : for (i = 0; i < XFS_INODES_PER_CHUNK; i++) {
2770 >15705*10^7 : if (irec.ir_startino + i < low)
2771 >15667*10^7 : continue;
2772 376074341 : if (irec.ir_startino + i > high)
2773 : break;
2774 :
2775 365735671 : hole_idx = i / XFS_INODES_PER_HOLEMASK_BIT;
2776 365735671 : if (!(irec.ir_holemask & (1U << hole_idx)))
2777 226887745 : ret++;
2778 : }
2779 :
2780 2464426419 : error = xfs_btree_increment(cur, 0, &has_record);
2781 2465126512 : if (error)
2782 0 : return error;
2783 : }
2784 :
2785 2867255526 : *allocated = ret;
2786 2867255526 : return 0;
2787 : }
2788 :
2789 : /* Is there an inode record covering a given extent? */
2790 : int
2791 2867216731 : xfs_ialloc_has_inodes_at_extent(
2792 : struct xfs_btree_cur *cur,
2793 : xfs_agblock_t bno,
2794 : xfs_extlen_t len,
2795 : enum xbtree_recpacking *outcome)
2796 : {
2797 2867216731 : xfs_agino_t agino;
2798 2867216731 : xfs_agino_t last_agino;
2799 2867216731 : unsigned int allocated;
2800 2867216731 : int error;
2801 :
2802 2867216731 : agino = XFS_AGB_TO_AGINO(cur->bc_mp, bno);
2803 2867216731 : last_agino = XFS_AGB_TO_AGINO(cur->bc_mp, bno + len) - 1;
2804 :
2805 2867216731 : error = xfs_ialloc_count_ondisk(cur, agino, last_agino, &allocated);
2806 2867322182 : if (error)
2807 : return error;
2808 :
2809 2867322182 : if (allocated == 0)
2810 2863268917 : *outcome = XBTREE_RECPACKING_EMPTY;
2811 4053265 : else if (allocated == last_agino - agino + 1)
2812 4053265 : *outcome = XBTREE_RECPACKING_FULL;
2813 : else
2814 0 : *outcome = XBTREE_RECPACKING_SPARSE;
2815 : return 0;
2816 : }
2817 :
2818 : struct xfs_ialloc_count_inodes {
2819 : xfs_agino_t count;
2820 : xfs_agino_t freecount;
2821 : };
2822 :
2823 : /* Record inode counts across all inobt records. */
2824 : STATIC int
2825 170107695 : xfs_ialloc_count_inodes_rec(
2826 : struct xfs_btree_cur *cur,
2827 : const union xfs_btree_rec *rec,
2828 : void *priv)
2829 : {
2830 170107695 : struct xfs_inobt_rec_incore irec;
2831 170107695 : struct xfs_ialloc_count_inodes *ci = priv;
2832 170107695 : xfs_failaddr_t fa;
2833 :
2834 170107695 : xfs_inobt_btrec_to_irec(cur->bc_mp, rec, &irec);
2835 170106578 : fa = xfs_inobt_check_irec(cur, &irec);
2836 170112506 : if (fa)
2837 0 : return xfs_inobt_complain_bad_rec(cur, fa, &irec);
2838 :
2839 170112506 : ci->count += irec.ir_count;
2840 170112506 : ci->freecount += irec.ir_freecount;
2841 :
2842 170112506 : return 0;
2843 : }
2844 :
2845 : /* Count allocated and free inodes under an inobt. */
2846 : int
2847 719631 : xfs_ialloc_count_inodes(
2848 : struct xfs_btree_cur *cur,
2849 : xfs_agino_t *count,
2850 : xfs_agino_t *freecount)
2851 : {
2852 719631 : struct xfs_ialloc_count_inodes ci = {0};
2853 719631 : int error;
2854 :
2855 719631 : ASSERT(cur->bc_btnum == XFS_BTNUM_INO);
2856 719631 : error = xfs_btree_query_all(cur, xfs_ialloc_count_inodes_rec, &ci);
2857 719370 : if (error)
2858 : return error;
2859 :
2860 719370 : *count = ci.count;
2861 719370 : *freecount = ci.freecount;
2862 719370 : return 0;
2863 : }
2864 :
2865 : /*
2866 : * Initialize inode-related geometry information.
2867 : *
2868 : * Compute the inode btree min and max levels and set maxicount.
2869 : *
2870 : * Set the inode cluster size. This may still be overridden by the file
2871 : * system block size if it is larger than the chosen cluster size.
2872 : *
2873 : * For v5 filesystems, scale the cluster size with the inode size to keep a
2874 : * constant ratio of inode per cluster buffer, but only if mkfs has set the
2875 : * inode alignment value appropriately for larger cluster sizes.
2876 : *
2877 : * Then compute the inode cluster alignment information.
2878 : */
2879 : void
2880 60880 : xfs_ialloc_setup_geometry(
2881 : struct xfs_mount *mp)
2882 : {
2883 60880 : struct xfs_sb *sbp = &mp->m_sb;
2884 60880 : struct xfs_ino_geometry *igeo = M_IGEO(mp);
2885 60880 : uint64_t icount;
2886 60880 : uint inodes;
2887 :
2888 60880 : igeo->new_diflags2 = 0;
2889 60880 : if (xfs_has_bigtime(mp))
2890 60498 : igeo->new_diflags2 |= XFS_DIFLAG2_BIGTIME;
2891 60880 : if (xfs_has_large_extent_counts(mp))
2892 60663 : igeo->new_diflags2 |= XFS_DIFLAG2_NREXT64;
2893 :
2894 : /* Compute inode btree geometry. */
2895 60880 : igeo->agino_log = sbp->sb_inopblog + sbp->sb_agblklog;
2896 60880 : igeo->inobt_mxr[0] = xfs_inobt_maxrecs(mp, sbp->sb_blocksize, 1);
2897 60880 : igeo->inobt_mxr[1] = xfs_inobt_maxrecs(mp, sbp->sb_blocksize, 0);
2898 60880 : igeo->inobt_mnr[0] = igeo->inobt_mxr[0] / 2;
2899 60880 : igeo->inobt_mnr[1] = igeo->inobt_mxr[1] / 2;
2900 :
2901 60880 : igeo->ialloc_inos = max_t(uint16_t, XFS_INODES_PER_CHUNK,
2902 : sbp->sb_inopblock);
2903 60880 : igeo->ialloc_blks = igeo->ialloc_inos >> sbp->sb_inopblog;
2904 :
2905 60880 : if (sbp->sb_spino_align)
2906 60674 : igeo->ialloc_min_blks = sbp->sb_spino_align;
2907 : else
2908 206 : igeo->ialloc_min_blks = igeo->ialloc_blks;
2909 :
2910 : /* Compute and fill in value of m_ino_geo.inobt_maxlevels. */
2911 60880 : inodes = (1LL << XFS_INO_AGINO_BITS(mp)) >> XFS_INODES_PER_CHUNK_LOG;
2912 60880 : igeo->inobt_maxlevels = xfs_btree_compute_maxlevels(igeo->inobt_mnr,
2913 : inodes);
2914 60880 : ASSERT(igeo->inobt_maxlevels <= xfs_iallocbt_maxlevels_ondisk());
2915 :
2916 : /*
2917 : * Set the maximum inode count for this filesystem, being careful not
2918 : * to use obviously garbage sb_inopblog/sb_inopblock values. Regular
2919 : * users should never get here due to failing sb verification, but
2920 : * certain users (xfs_db) need to be usable even with corrupt metadata.
2921 : */
2922 60880 : if (sbp->sb_imax_pct && igeo->ialloc_blks) {
2923 : /*
2924 : * Make sure the maximum inode count is a multiple
2925 : * of the units we allocate inodes in.
2926 : */
2927 60880 : icount = sbp->sb_dblocks * sbp->sb_imax_pct;
2928 60880 : do_div(icount, 100);
2929 60880 : do_div(icount, igeo->ialloc_blks);
2930 60880 : igeo->maxicount = XFS_FSB_TO_INO(mp,
2931 : icount * igeo->ialloc_blks);
2932 : } else {
2933 0 : igeo->maxicount = 0;
2934 : }
2935 :
2936 : /*
2937 : * Compute the desired size of an inode cluster buffer size, which
2938 : * starts at 8K and (on v5 filesystems) scales up with larger inode
2939 : * sizes.
2940 : *
2941 : * Preserve the desired inode cluster size because the sparse inodes
2942 : * feature uses that desired size (not the actual size) to compute the
2943 : * sparse inode alignment. The mount code validates this value, so we
2944 : * cannot change the behavior.
2945 : */
2946 60880 : igeo->inode_cluster_size_raw = XFS_INODE_BIG_CLUSTER_SIZE;
2947 60880 : if (xfs_has_v3inodes(mp)) {
2948 60674 : int new_size = igeo->inode_cluster_size_raw;
2949 :
2950 60674 : new_size *= mp->m_sb.sb_inodesize / XFS_DINODE_MIN_SIZE;
2951 60674 : if (mp->m_sb.sb_inoalignmt >= XFS_B_TO_FSBT(mp, new_size))
2952 60674 : igeo->inode_cluster_size_raw = new_size;
2953 : }
2954 :
2955 : /* Calculate inode cluster ratios. */
2956 60880 : if (igeo->inode_cluster_size_raw > mp->m_sb.sb_blocksize)
2957 60880 : igeo->blocks_per_cluster = XFS_B_TO_FSBT(mp,
2958 : igeo->inode_cluster_size_raw);
2959 : else
2960 0 : igeo->blocks_per_cluster = 1;
2961 60880 : igeo->inode_cluster_size = XFS_FSB_TO_B(mp, igeo->blocks_per_cluster);
2962 60880 : igeo->inodes_per_cluster = XFS_FSB_TO_INO(mp, igeo->blocks_per_cluster);
2963 :
2964 : /* Calculate inode cluster alignment. */
2965 60880 : if (xfs_has_align(mp) &&
2966 60880 : mp->m_sb.sb_inoalignmt >= igeo->blocks_per_cluster)
2967 60880 : igeo->cluster_align = mp->m_sb.sb_inoalignmt;
2968 : else
2969 0 : igeo->cluster_align = 1;
2970 60880 : igeo->inoalign_mask = igeo->cluster_align - 1;
2971 60880 : igeo->cluster_align_inodes = XFS_FSB_TO_INO(mp, igeo->cluster_align);
2972 :
2973 : /*
2974 : * If we are using stripe alignment, check whether
2975 : * the stripe unit is a multiple of the inode alignment
2976 : */
2977 60880 : if (mp->m_dalign && igeo->inoalign_mask &&
2978 106 : !(mp->m_dalign & igeo->inoalign_mask))
2979 75 : igeo->ialloc_align = mp->m_dalign;
2980 : else
2981 60805 : igeo->ialloc_align = 0;
2982 60880 : }
2983 :
2984 : /* Compute the location of the root directory inode that is laid out by mkfs. */
2985 : xfs_ino_t
2986 95 : xfs_ialloc_calc_rootino(
2987 : struct xfs_mount *mp,
2988 : int sunit)
2989 : {
2990 95 : struct xfs_ino_geometry *igeo = M_IGEO(mp);
2991 95 : xfs_agblock_t first_bno;
2992 :
2993 : /*
2994 : * Pre-calculate the geometry of AG 0. We know what it looks like
2995 : * because libxfs knows how to create allocation groups now.
2996 : *
2997 : * first_bno is the first block in which mkfs could possibly have
2998 : * allocated the root directory inode, once we factor in the metadata
2999 : * that mkfs formats before it. Namely, the four AG headers...
3000 : */
3001 95 : first_bno = howmany(4 * mp->m_sb.sb_sectsize, mp->m_sb.sb_blocksize);
3002 :
3003 : /* ...the two free space btree roots... */
3004 95 : first_bno += 2;
3005 :
3006 : /* ...the inode btree root... */
3007 95 : first_bno += 1;
3008 :
3009 : /* ...the initial AGFL... */
3010 95 : first_bno += xfs_alloc_min_freelist(mp, NULL);
3011 :
3012 : /* ...the free inode btree root... */
3013 95 : if (xfs_has_finobt(mp))
3014 95 : first_bno++;
3015 :
3016 : /* ...the reverse mapping btree root... */
3017 95 : if (xfs_has_rmapbt(mp))
3018 70 : first_bno++;
3019 :
3020 : /* ...the reference count btree... */
3021 95 : if (xfs_has_reflink(mp))
3022 70 : first_bno++;
3023 :
3024 : /*
3025 : * ...and the log, if it is allocated in the first allocation group.
3026 : *
3027 : * This can happen with filesystems that only have a single
3028 : * allocation group, or very odd geometries created by old mkfs
3029 : * versions on very small filesystems.
3030 : */
3031 95 : if (xfs_ag_contains_log(mp, 0))
3032 0 : first_bno += mp->m_sb.sb_logblocks;
3033 :
3034 : /*
3035 : * Now round first_bno up to whatever allocation alignment is given
3036 : * by the filesystem or was passed in.
3037 : */
3038 95 : if (xfs_has_dalign(mp) && igeo->ialloc_align > 0)
3039 64 : first_bno = roundup(first_bno, sunit);
3040 31 : else if (xfs_has_align(mp) &&
3041 31 : mp->m_sb.sb_inoalignmt > 1)
3042 31 : first_bno = roundup(first_bno, mp->m_sb.sb_inoalignmt);
3043 :
3044 95 : return XFS_AGINO_TO_INO(mp, 0, XFS_AGB_TO_AGINO(mp, first_bno));
3045 : }
3046 :
3047 : /*
3048 : * Ensure there are not sparse inode clusters that cross the new EOAG.
3049 : *
3050 : * This is a no-op for non-spinode filesystems since clusters are always fully
3051 : * allocated and checking the bnobt suffices. However, a spinode filesystem
3052 : * could have a record where the upper inodes are free blocks. If those blocks
3053 : * were removed from the filesystem, the inode record would extend beyond EOAG,
3054 : * which will be flagged as corruption.
3055 : */
3056 : int
3057 894 : xfs_ialloc_check_shrink(
3058 : struct xfs_perag *pag,
3059 : struct xfs_trans *tp,
3060 : struct xfs_buf *agibp,
3061 : xfs_agblock_t new_length)
3062 : {
3063 894 : struct xfs_inobt_rec_incore rec;
3064 894 : struct xfs_btree_cur *cur;
3065 894 : xfs_agino_t agino;
3066 894 : int has;
3067 894 : int error;
3068 :
3069 894 : if (!xfs_has_sparseinodes(pag->pag_mount))
3070 : return 0;
3071 :
3072 894 : cur = xfs_inobt_init_cursor(pag, tp, agibp, XFS_BTNUM_INO);
3073 :
3074 : /* Look up the inobt record that would correspond to the new EOFS. */
3075 894 : agino = XFS_AGB_TO_AGINO(pag->pag_mount, new_length);
3076 894 : error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &has);
3077 894 : if (error || !has)
3078 80 : goto out;
3079 :
3080 814 : error = xfs_inobt_get_rec(cur, &rec, &has);
3081 814 : if (error)
3082 0 : goto out;
3083 :
3084 814 : if (!has) {
3085 0 : xfs_ag_mark_sick(pag, XFS_SICK_AG_INOBT);
3086 0 : error = -EFSCORRUPTED;
3087 0 : goto out;
3088 : }
3089 :
3090 : /* If the record covers inodes that would be beyond EOFS, bail out. */
3091 814 : if (rec.ir_startino + XFS_INODES_PER_CHUNK > agino) {
3092 375 : error = -ENOSPC;
3093 375 : goto out;
3094 : }
3095 439 : out:
3096 894 : xfs_btree_del_cursor(cur, error);
3097 894 : return error;
3098 : }
|
