Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2000-2005 Silicon Graphics, Inc.
4 : * All Rights Reserved.
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_bit.h"
13 : #include "xfs_mount.h"
14 : #include "xfs_inode.h"
15 : #include "xfs_bmap.h"
16 : #include "xfs_bmap_btree.h"
17 : #include "xfs_trans.h"
18 : #include "xfs_trans_space.h"
19 : #include "xfs_icache.h"
20 : #include "xfs_rtalloc.h"
21 : #include "xfs_sb.h"
22 : #include "xfs_log_priv.h"
23 : #include "xfs_health.h"
24 : #include "xfs_trace.h"
25 :
26 : /*
27 : * Read and return the summary information for a given extent size,
28 : * bitmap block combination.
29 : * Keeps track of a current summary block, so we don't keep reading
30 : * it from the buffer cache.
31 : */
32 : static int
33 : xfs_rtget_summary(
34 : xfs_mount_t *mp, /* file system mount structure */
35 : xfs_trans_t *tp, /* transaction pointer */
36 : int log, /* log2 of extent size */
37 : xfs_rtblock_t bbno, /* bitmap block number */
38 : struct xfs_buf **rbpp, /* in/out: summary block buffer */
39 : xfs_fsblock_t *rsb, /* in/out: summary block number */
40 : xfs_suminfo_t *sum) /* out: summary info for this block */
41 : {
42 664128036 : return xfs_rtmodify_summary_int(mp, tp, log, bbno, 0, rbpp, rsb, sum);
43 : }
44 :
45 : /*
46 : * Return whether there are any free extents in the size range given
47 : * by low and high, for the bitmap block bbno.
48 : */
49 : STATIC int /* error */
50 241545038 : xfs_rtany_summary(
51 : xfs_mount_t *mp, /* file system mount structure */
52 : xfs_trans_t *tp, /* transaction pointer */
53 : int low, /* low log2 extent size */
54 : int high, /* high log2 extent size */
55 : xfs_rtblock_t bbno, /* bitmap block number */
56 : struct xfs_buf **rbpp, /* in/out: summary block buffer */
57 : xfs_fsblock_t *rsb, /* in/out: summary block number */
58 : int *stat) /* out: any good extents here? */
59 : {
60 241545038 : int error; /* error value */
61 241545038 : int log; /* loop counter, log2 of ext. size */
62 241545038 : xfs_suminfo_t sum; /* summary data */
63 :
64 : /* There are no extents at levels < m_rsum_cache[bbno]. */
65 241545038 : if (mp->m_rsum_cache && low < mp->m_rsum_cache[bbno])
66 : low = mp->m_rsum_cache[bbno];
67 :
68 : /*
69 : * Loop over logs of extent sizes.
70 : */
71 270604861 : for (log = low; log <= high; log++) {
72 : /*
73 : * Get one summary datum.
74 : */
75 49462697 : error = xfs_rtget_summary(mp, tp, log, bbno, rbpp, rsb, &sum);
76 49462697 : if (error) {
77 0 : return error;
78 : }
79 : /*
80 : * If there are any, return success.
81 : */
82 49462697 : if (sum) {
83 20402874 : *stat = 1;
84 20402874 : goto out;
85 : }
86 : }
87 : /*
88 : * Found nothing, return failure.
89 : */
90 221142164 : *stat = 0;
91 241545038 : out:
92 : /* There were no extents at levels < log. */
93 241545038 : if (mp->m_rsum_cache && log > mp->m_rsum_cache[bbno])
94 1571119 : mp->m_rsum_cache[bbno] = log;
95 : return 0;
96 : }
97 :
98 :
99 : /*
100 : * Copy and transform the summary file, given the old and new
101 : * parameters in the mount structures.
102 : */
103 : STATIC int /* error */
104 43 : xfs_rtcopy_summary(
105 : xfs_mount_t *omp, /* old file system mount point */
106 : xfs_mount_t *nmp, /* new file system mount point */
107 : xfs_trans_t *tp) /* transaction pointer */
108 : {
109 43 : xfs_rtblock_t bbno; /* bitmap block number */
110 43 : struct xfs_buf *bp; /* summary buffer */
111 43 : int error; /* error return value */
112 43 : int log; /* summary level number (log length) */
113 43 : xfs_suminfo_t sum; /* summary data */
114 43 : xfs_fsblock_t sumbno; /* summary block number */
115 :
116 43 : bp = NULL;
117 714 : for (log = omp->m_rsumlevels - 1; log >= 0; log--) {
118 671 : for (bbno = omp->m_sb.sb_rbmblocks - 1;
119 1801 : (xfs_srtblock_t)bbno >= 0;
120 1130 : bbno--) {
121 1130 : error = xfs_rtget_summary(omp, tp, log, bbno, &bp,
122 : &sumbno, &sum);
123 1130 : if (error)
124 0 : return error;
125 1130 : if (sum == 0)
126 1088 : continue;
127 42 : error = xfs_rtmodify_summary(omp, tp, log, bbno, -sum,
128 : &bp, &sumbno);
129 42 : if (error)
130 0 : return error;
131 42 : error = xfs_rtmodify_summary(nmp, tp, log, bbno, sum,
132 : &bp, &sumbno);
133 42 : if (error)
134 0 : return error;
135 42 : ASSERT(sum > 0);
136 : }
137 : }
138 : return 0;
139 : }
140 : /*
141 : * Mark an extent specified by start and len allocated.
142 : * Updates all the summary information as well as the bitmap.
143 : */
144 : STATIC int /* error */
145 57355577 : xfs_rtallocate_range(
146 : xfs_mount_t *mp, /* file system mount point */
147 : xfs_trans_t *tp, /* transaction pointer */
148 : xfs_rtblock_t start, /* start block to allocate */
149 : xfs_extlen_t len, /* length to allocate */
150 : struct xfs_buf **rbpp, /* in/out: summary block buffer */
151 : xfs_fsblock_t *rsb) /* in/out: summary block number */
152 : {
153 57355577 : xfs_rtblock_t end; /* end of the allocated extent */
154 57355577 : int error; /* error value */
155 57355577 : xfs_rtblock_t postblock = 0; /* first block allocated > end */
156 57355577 : xfs_rtblock_t preblock = 0; /* first block allocated < start */
157 :
158 57355577 : end = start + len - 1;
159 : /*
160 : * Assume we're allocating out of the middle of a free extent.
161 : * We need to find the beginning and end of the extent so we can
162 : * properly update the summary.
163 : */
164 57355577 : error = xfs_rtfind_back(mp, tp, start, 0, &preblock);
165 57355577 : if (error) {
166 : return error;
167 : }
168 : /*
169 : * Find the next allocated block (end of free extent).
170 : */
171 57355577 : error = xfs_rtfind_forw(mp, tp, end, mp->m_sb.sb_rextents - 1,
172 : &postblock);
173 57355577 : if (error) {
174 : return error;
175 : }
176 : /*
177 : * Decrement the summary information corresponding to the entire
178 : * (old) free extent.
179 : */
180 114711140 : error = xfs_rtmodify_summary(mp, tp,
181 57355570 : XFS_RTBLOCKLOG(postblock + 1 - preblock),
182 57355570 : XFS_BITTOBLOCK(mp, preblock), -1, rbpp, rsb);
183 57355570 : if (error) {
184 : return error;
185 : }
186 : /*
187 : * If there are blocks not being allocated at the front of the
188 : * old extent, add summary data for them to be free.
189 : */
190 57355570 : if (preblock < start) {
191 17299988 : error = xfs_rtmodify_summary(mp, tp,
192 : XFS_RTBLOCKLOG(start - preblock),
193 8649994 : XFS_BITTOBLOCK(mp, preblock), 1, rbpp, rsb);
194 8649994 : if (error) {
195 : return error;
196 : }
197 : }
198 : /*
199 : * If there are blocks not being allocated at the end of the
200 : * old extent, add summary data for them to be free.
201 : */
202 57355570 : if (postblock > end) {
203 97527874 : error = xfs_rtmodify_summary(mp, tp,
204 : XFS_RTBLOCKLOG(postblock - end),
205 48763937 : XFS_BITTOBLOCK(mp, end + 1), 1, rbpp, rsb);
206 48763937 : if (error) {
207 : return error;
208 : }
209 : }
210 : /*
211 : * Modify the bitmap to mark this extent allocated.
212 : */
213 57355570 : error = xfs_rtmodify_range(mp, tp, start, len, 0);
214 57355570 : return error;
215 : }
216 :
217 : /*
218 : * Attempt to allocate an extent minlen<=len<=maxlen starting from
219 : * bitmap block bbno. If we don't get maxlen then use prod to trim
220 : * the length, if given. Returns error; returns starting block in *rtblock.
221 : * The lengths are all in rtextents.
222 : */
223 : STATIC int /* error */
224 45164493 : xfs_rtallocate_extent_block(
225 : xfs_mount_t *mp, /* file system mount point */
226 : xfs_trans_t *tp, /* transaction pointer */
227 : xfs_rtblock_t bbno, /* bitmap block number */
228 : xfs_extlen_t minlen, /* minimum length to allocate */
229 : xfs_extlen_t maxlen, /* maximum length to allocate */
230 : xfs_extlen_t *len, /* out: actual length allocated */
231 : xfs_rtblock_t *nextp, /* out: next block to try */
232 : struct xfs_buf **rbpp, /* in/out: summary block buffer */
233 : xfs_fsblock_t *rsb, /* in/out: summary block number */
234 : xfs_extlen_t prod, /* extent product factor */
235 : xfs_rtblock_t *rtblock) /* out: start block allocated */
236 : {
237 45164493 : xfs_rtblock_t besti; /* best rtblock found so far */
238 45164493 : xfs_rtblock_t bestlen; /* best length found so far */
239 45164493 : xfs_rtblock_t end; /* last rtblock in chunk */
240 45164493 : int error; /* error value */
241 45164493 : xfs_rtblock_t i; /* current rtblock trying */
242 45164493 : xfs_rtblock_t next; /* next rtblock to try */
243 45164493 : int stat; /* status from internal calls */
244 :
245 : /*
246 : * Loop over all the extents starting in this bitmap block,
247 : * looking for one that's long enough.
248 : */
249 45164493 : for (i = XFS_BLOCKTOBIT(mp, bbno), besti = -1, bestlen = 0,
250 45164493 : end = XFS_BLOCKTOBIT(mp, bbno + 1) - 1;
251 311931404 : i <= end;
252 266766911 : i++) {
253 : /* Make sure we don't scan off the end of the rt volume. */
254 286515489 : maxlen = min(mp->m_sb.sb_rextents, i + maxlen) - i;
255 :
256 : /*
257 : * See if there's a free extent of maxlen starting at i.
258 : * If it's not so then next will contain the first non-free.
259 : */
260 286515489 : error = xfs_rtcheck_range(mp, tp, i, maxlen, 1, &next, &stat);
261 286515489 : if (error) {
262 0 : return error;
263 : }
264 286515489 : if (stat) {
265 : /*
266 : * i for maxlen is all free, allocate and return that.
267 : */
268 19718691 : error = xfs_rtallocate_range(mp, tp, i, maxlen, rbpp,
269 : rsb);
270 19718691 : if (error) {
271 : return error;
272 : }
273 19718687 : *len = maxlen;
274 19718687 : *rtblock = i;
275 19718687 : return 0;
276 : }
277 : /*
278 : * In the case where we have a variable-sized allocation
279 : * request, figure out how big this free piece is,
280 : * and if it's big enough for the minimum, and the best
281 : * so far, remember it.
282 : */
283 266796798 : if (minlen < maxlen) {
284 231580774 : xfs_rtblock_t thislen; /* this extent size */
285 :
286 231580774 : thislen = next - i;
287 231580774 : if (thislen >= minlen && thislen > bestlen) {
288 11693625 : besti = i;
289 11693625 : bestlen = thislen;
290 : }
291 : }
292 : /*
293 : * If not done yet, find the start of the next free space.
294 : */
295 266796798 : if (next < end) {
296 266766911 : error = xfs_rtfind_forw(mp, tp, next, end, &i);
297 266766911 : if (error) {
298 0 : return error;
299 : }
300 : } else
301 : break;
302 : }
303 : /*
304 : * Searched the whole thing & didn't find a maxlen free extent.
305 : */
306 25445802 : if (minlen < maxlen && besti != -1) {
307 2391495 : xfs_extlen_t p; /* amount to trim length by */
308 :
309 : /*
310 : * If size should be a multiple of prod, make that so.
311 : */
312 2391495 : if (prod > 1) {
313 0 : div_u64_rem(bestlen, prod, &p);
314 0 : if (p)
315 0 : bestlen -= p;
316 : }
317 :
318 : /*
319 : * Allocate besti for bestlen & return that.
320 : */
321 2391495 : error = xfs_rtallocate_range(mp, tp, besti, bestlen, rbpp, rsb);
322 2391495 : if (error) {
323 : return error;
324 : }
325 2391495 : *len = bestlen;
326 2391495 : *rtblock = besti;
327 2391495 : return 0;
328 : }
329 : /*
330 : * Allocation failed. Set *nextp to the next block to try.
331 : */
332 23054307 : *nextp = next;
333 23054307 : *rtblock = NULLRTBLOCK;
334 23054307 : return 0;
335 : }
336 :
337 : /*
338 : * Allocate an extent of length minlen<=len<=maxlen, starting at block
339 : * bno. If we don't get maxlen then use prod to trim the length, if given.
340 : * Returns error; returns starting block in *rtblock.
341 : * The lengths are all in rtextents.
342 : */
343 : STATIC int /* error */
344 55648299 : xfs_rtallocate_extent_exact(
345 : xfs_mount_t *mp, /* file system mount point */
346 : xfs_trans_t *tp, /* transaction pointer */
347 : xfs_rtblock_t bno, /* starting block number to allocate */
348 : xfs_extlen_t minlen, /* minimum length to allocate */
349 : xfs_extlen_t maxlen, /* maximum length to allocate */
350 : xfs_extlen_t *len, /* out: actual length allocated */
351 : struct xfs_buf **rbpp, /* in/out: summary block buffer */
352 : xfs_fsblock_t *rsb, /* in/out: summary block number */
353 : xfs_extlen_t prod, /* extent product factor */
354 : xfs_rtblock_t *rtblock) /* out: start block allocated */
355 : {
356 55648299 : int error; /* error value */
357 55648299 : xfs_extlen_t i; /* extent length trimmed due to prod */
358 55648299 : int isfree; /* extent is free */
359 55648299 : xfs_rtblock_t next; /* next block to try (dummy) */
360 :
361 55648299 : ASSERT(minlen % prod == 0 && maxlen % prod == 0);
362 : /*
363 : * Check if the range in question (for maxlen) is free.
364 : */
365 55648299 : error = xfs_rtcheck_range(mp, tp, bno, maxlen, 1, &next, &isfree);
366 55648299 : if (error) {
367 : return error;
368 : }
369 55648295 : if (isfree) {
370 : /*
371 : * If it is, allocate it and return success.
372 : */
373 34592511 : error = xfs_rtallocate_range(mp, tp, bno, maxlen, rbpp, rsb);
374 34592511 : if (error) {
375 : return error;
376 : }
377 34592508 : *len = maxlen;
378 34592508 : *rtblock = bno;
379 34592508 : return 0;
380 : }
381 : /*
382 : * If not, allocate what there is, if it's at least minlen.
383 : */
384 21055784 : maxlen = next - bno;
385 21055784 : if (maxlen < minlen) {
386 : /*
387 : * Failed, return failure status.
388 : */
389 20402904 : *rtblock = NULLRTBLOCK;
390 20402904 : return 0;
391 : }
392 : /*
393 : * Trim off tail of extent, if prod is specified.
394 : */
395 652880 : if (prod > 1 && (i = maxlen % prod)) {
396 0 : maxlen -= i;
397 0 : if (maxlen < minlen) {
398 : /*
399 : * Now we can't do it, return failure status.
400 : */
401 0 : *rtblock = NULLRTBLOCK;
402 0 : return 0;
403 : }
404 : }
405 : /*
406 : * Allocate what we can and return it.
407 : */
408 652880 : error = xfs_rtallocate_range(mp, tp, bno, maxlen, rbpp, rsb);
409 652880 : if (error) {
410 : return error;
411 : }
412 652880 : *len = maxlen;
413 652880 : *rtblock = bno;
414 652880 : return 0;
415 : }
416 :
417 : /*
418 : * Allocate an extent of length minlen<=len<=maxlen, starting as near
419 : * to bno as possible. If we don't get maxlen then use prod to trim
420 : * the length, if given. The lengths are all in rtextents.
421 : */
422 : STATIC int /* error */
423 55648299 : xfs_rtallocate_extent_near(
424 : xfs_mount_t *mp, /* file system mount point */
425 : xfs_trans_t *tp, /* transaction pointer */
426 : xfs_rtblock_t bno, /* starting block number to allocate */
427 : xfs_extlen_t minlen, /* minimum length to allocate */
428 : xfs_extlen_t maxlen, /* maximum length to allocate */
429 : xfs_extlen_t *len, /* out: actual length allocated */
430 : struct xfs_buf **rbpp, /* in/out: summary block buffer */
431 : xfs_fsblock_t *rsb, /* in/out: summary block number */
432 : xfs_extlen_t prod, /* extent product factor */
433 : xfs_rtblock_t *rtblock) /* out: start block allocated */
434 : {
435 55648299 : int any; /* any useful extents from summary */
436 55648299 : xfs_rtblock_t bbno; /* bitmap block number */
437 55648299 : int error; /* error value */
438 55648299 : int i; /* bitmap block offset (loop control) */
439 55648299 : int j; /* secondary loop control */
440 55648299 : int log2len; /* log2 of minlen */
441 55648299 : xfs_rtblock_t n; /* next block to try */
442 55648299 : xfs_rtblock_t r; /* result block */
443 :
444 55648299 : ASSERT(minlen % prod == 0 && maxlen % prod == 0);
445 : /*
446 : * If the block number given is off the end, silently set it to
447 : * the last block.
448 : */
449 55648299 : if (bno >= mp->m_sb.sb_rextents)
450 24 : bno = mp->m_sb.sb_rextents - 1;
451 :
452 : /* Make sure we don't run off the end of the rt volume. */
453 55648299 : maxlen = min(mp->m_sb.sb_rextents, bno + maxlen) - bno;
454 55648299 : if (maxlen < minlen) {
455 0 : *rtblock = NULLRTBLOCK;
456 0 : return 0;
457 : }
458 :
459 : /*
460 : * Try the exact allocation first.
461 : */
462 55648299 : error = xfs_rtallocate_extent_exact(mp, tp, bno, minlen, maxlen, len,
463 : rbpp, rsb, prod, &r);
464 55648299 : if (error) {
465 : return error;
466 : }
467 : /*
468 : * If the exact allocation worked, return that.
469 : */
470 55648292 : if (r != NULLRTBLOCK) {
471 35245388 : *rtblock = r;
472 35245388 : return 0;
473 : }
474 20402904 : bbno = XFS_BITTOBLOCK(mp, bno);
475 20402904 : i = 0;
476 20402904 : ASSERT(minlen != 0);
477 20402904 : log2len = xfs_highbit32(minlen);
478 : /*
479 : * Loop over all bitmap blocks (bbno + i is current block).
480 : */
481 218747237 : for (;;) {
482 : /*
483 : * Get summary information of extents of all useful levels
484 : * starting in this bitmap block.
485 : */
486 218747237 : error = xfs_rtany_summary(mp, tp, log2len, mp->m_rsumlevels - 1,
487 : bbno + i, rbpp, rsb, &any);
488 218747237 : if (error) {
489 0 : return error;
490 : }
491 : /*
492 : * If there are any useful extents starting here, try
493 : * allocating one.
494 : */
495 218747237 : if (any) {
496 : /*
497 : * On the positive side of the starting location.
498 : */
499 20402874 : if (i >= 0) {
500 : /*
501 : * Try to allocate an extent starting in
502 : * this block.
503 : */
504 17298143 : error = xfs_rtallocate_extent_block(mp, tp,
505 : bbno + i, minlen, maxlen, len, &n, rbpp,
506 : rsb, prod, &r);
507 17298143 : if (error) {
508 3 : return error;
509 : }
510 : /*
511 : * If it worked, return it.
512 : */
513 17298140 : if (r != NULLRTBLOCK) {
514 17298140 : *rtblock = r;
515 17298140 : return 0;
516 : }
517 : }
518 : /*
519 : * On the negative side of the starting location.
520 : */
521 : else { /* i < 0 */
522 : /*
523 : * Loop backwards through the bitmap blocks from
524 : * the starting point-1 up to where we are now.
525 : * There should be an extent which ends in this
526 : * bitmap block and is long enough.
527 : */
528 25901555 : for (j = -1; j > i; j--) {
529 : /*
530 : * Grab the summary information for
531 : * this bitmap block.
532 : */
533 22797801 : error = xfs_rtany_summary(mp, tp,
534 22797801 : log2len, mp->m_rsumlevels - 1,
535 : bbno + j, rbpp, rsb, &any);
536 22797801 : if (error) {
537 0 : return error;
538 : }
539 : /*
540 : * If there's no extent given in the
541 : * summary that means the extent we
542 : * found must carry over from an
543 : * earlier block. If there is an
544 : * extent given, we've already tried
545 : * that allocation, don't do it again.
546 : */
547 22797801 : if (any)
548 0 : continue;
549 22797801 : error = xfs_rtallocate_extent_block(mp,
550 : tp, bbno + j, minlen, maxlen,
551 : len, &n, rbpp, rsb, prod, &r);
552 22797801 : if (error) {
553 0 : return error;
554 : }
555 : /*
556 : * If it works, return the extent.
557 : */
558 22797801 : if (r != NULLRTBLOCK) {
559 977 : *rtblock = r;
560 977 : return 0;
561 : }
562 : }
563 : /*
564 : * There weren't intervening bitmap blocks
565 : * with a long enough extent, or the
566 : * allocation didn't work for some reason
567 : * (i.e. it's a little * too short).
568 : * Try to allocate from the summary block
569 : * that we found.
570 : */
571 3103754 : error = xfs_rtallocate_extent_block(mp, tp,
572 : bbno + i, minlen, maxlen, len, &n, rbpp,
573 : rsb, prod, &r);
574 3103754 : if (error) {
575 0 : return error;
576 : }
577 : /*
578 : * If it works, return the extent.
579 : */
580 3103754 : if (r != NULLRTBLOCK) {
581 3103754 : *rtblock = r;
582 3103754 : return 0;
583 : }
584 : }
585 : }
586 : /*
587 : * Loop control. If we were on the positive side, and there's
588 : * still more blocks on the negative side, go there.
589 : */
590 198344363 : if (i > 0 && (int)bbno - i >= 0)
591 71335830 : i = -i;
592 : /*
593 : * If positive, and no more negative, but there are more
594 : * positive, go there.
595 : */
596 127008533 : else if (i > 0 && (int)bbno + i < mp->m_sb.sb_rbmblocks - 1)
597 42171671 : i++;
598 : /*
599 : * If negative or 0 (just started), and there are positive
600 : * blocks to go, go there. The 0 case moves to block 1.
601 : */
602 84836862 : else if (i <= 0 && (int)bbno - i < mp->m_sb.sb_rbmblocks - 1)
603 76786834 : i = 1 - i;
604 : /*
605 : * If negative or 0 and there are more negative blocks,
606 : * go there.
607 : */
608 8050028 : else if (i <= 0 && (int)bbno + i > 0)
609 8049998 : i--;
610 : /*
611 : * Must be done. Return failure.
612 : */
613 : else
614 : break;
615 : }
616 30 : *rtblock = NULLRTBLOCK;
617 30 : return 0;
618 : }
619 :
620 : /*
621 : * Allocate an extent of length minlen<=len<=maxlen, with no position
622 : * specified. If we don't get maxlen then use prod to trim
623 : * the length, if given. The lengths are all in rtextents.
624 : */
625 : STATIC int /* error */
626 1707313 : xfs_rtallocate_extent_size(
627 : xfs_mount_t *mp, /* file system mount point */
628 : xfs_trans_t *tp, /* transaction pointer */
629 : xfs_extlen_t minlen, /* minimum length to allocate */
630 : xfs_extlen_t maxlen, /* maximum length to allocate */
631 : xfs_extlen_t *len, /* out: actual length allocated */
632 : struct xfs_buf **rbpp, /* in/out: summary block buffer */
633 : xfs_fsblock_t *rsb, /* in/out: summary block number */
634 : xfs_extlen_t prod, /* extent product factor */
635 : xfs_rtblock_t *rtblock) /* out: start block allocated */
636 : {
637 1707313 : int error; /* error value */
638 1707313 : int i; /* bitmap block number */
639 1707313 : int l; /* level number (loop control) */
640 1707313 : xfs_rtblock_t n; /* next block to be tried */
641 1707313 : xfs_rtblock_t r; /* result block number */
642 1707313 : xfs_suminfo_t sum; /* summary information for extents */
643 :
644 1707313 : ASSERT(minlen % prod == 0 && maxlen % prod == 0);
645 1707313 : ASSERT(maxlen != 0);
646 :
647 : /*
648 : * Loop over all the levels starting with maxlen.
649 : * At each level, look at all the bitmap blocks, to see if there
650 : * are extents starting there that are long enough (>= maxlen).
651 : * Note, only on the initial level can the allocation fail if
652 : * the summary says there's an extent.
653 : */
654 4119010 : for (l = xfs_highbit32(maxlen); l < mp->m_rsumlevels; l++) {
655 : /*
656 : * Loop over all the bitmap blocks.
657 : */
658 617064253 : for (i = 0; i < mp->m_sb.sb_rbmblocks; i++) {
659 : /*
660 : * Get the summary for this level/block.
661 : */
662 614652556 : error = xfs_rtget_summary(mp, tp, l, i, rbpp, rsb,
663 : &sum);
664 614652556 : if (error) {
665 1 : return error;
666 : }
667 : /*
668 : * Nothing there, on to the next block.
669 : */
670 614652555 : if (!sum)
671 612689917 : continue;
672 : /*
673 : * Try allocating the extent.
674 : */
675 1962638 : error = xfs_rtallocate_extent_block(mp, tp, i, maxlen,
676 : maxlen, len, &n, rbpp, rsb, prod, &r);
677 1962638 : if (error) {
678 1 : return error;
679 : }
680 : /*
681 : * If it worked, return that.
682 : */
683 1962637 : if (r != NULLRTBLOCK) {
684 1705157 : *rtblock = r;
685 1705157 : return 0;
686 : }
687 : /*
688 : * If the "next block to try" returned from the
689 : * allocator is beyond the next bitmap block,
690 : * skip to that bitmap block.
691 : */
692 257480 : if (XFS_BITTOBLOCK(mp, n) > i + 1)
693 2 : i = XFS_BITTOBLOCK(mp, n) - 1;
694 : }
695 : }
696 : /*
697 : * Didn't find any maxlen blocks. Try smaller ones, unless
698 : * we're asking for a fixed size extent.
699 : */
700 2154 : if (minlen > --maxlen) {
701 0 : *rtblock = NULLRTBLOCK;
702 0 : return 0;
703 : }
704 2154 : ASSERT(minlen != 0);
705 2154 : ASSERT(maxlen != 0);
706 :
707 : /*
708 : * Loop over sizes, from maxlen down to minlen.
709 : * This time, when we do the allocations, allow smaller ones
710 : * to succeed.
711 : */
712 2875 : for (l = xfs_highbit32(maxlen); l >= xfs_highbit32(minlen); l--) {
713 : /*
714 : * Loop over all the bitmap blocks, try an allocation
715 : * starting in that block.
716 : */
717 12374 : for (i = 0; i < mp->m_sb.sb_rbmblocks; i++) {
718 : /*
719 : * Get the summary information for this level/block.
720 : */
721 11653 : error = xfs_rtget_summary(mp, tp, l, i, rbpp, rsb,
722 : &sum);
723 11653 : if (error) {
724 0 : return error;
725 : }
726 : /*
727 : * If nothing there, go on to next.
728 : */
729 11653 : if (!sum)
730 9496 : continue;
731 : /*
732 : * Try the allocation. Make sure the specified
733 : * minlen/maxlen are in the possible range for
734 : * this summary level.
735 : */
736 3417 : error = xfs_rtallocate_extent_block(mp, tp, i,
737 2157 : XFS_RTMAX(minlen, 1 << l),
738 2157 : XFS_RTMIN(maxlen, (1 << (l + 1)) - 1),
739 : len, &n, rbpp, rsb, prod, &r);
740 2157 : if (error) {
741 0 : return error;
742 : }
743 : /*
744 : * If it worked, return that extent.
745 : */
746 2157 : if (r != NULLRTBLOCK) {
747 2154 : *rtblock = r;
748 2154 : return 0;
749 : }
750 : /*
751 : * If the "next block to try" returned from the
752 : * allocator is beyond the next bitmap block,
753 : * skip to that bitmap block.
754 : */
755 3 : if (XFS_BITTOBLOCK(mp, n) > i + 1)
756 0 : i = XFS_BITTOBLOCK(mp, n) - 1;
757 : }
758 : }
759 : /*
760 : * Got nothing, return failure.
761 : */
762 0 : *rtblock = NULLRTBLOCK;
763 0 : return 0;
764 : }
765 :
766 : /*
767 : * Allocate space to the bitmap or summary file, and zero it, for growfs.
768 : */
769 : STATIC int
770 73 : xfs_growfs_rt_alloc(
771 : struct xfs_mount *mp, /* file system mount point */
772 : xfs_extlen_t oblocks, /* old count of blocks */
773 : xfs_extlen_t nblocks, /* new count of blocks */
774 : struct xfs_inode *ip) /* inode (bitmap/summary) */
775 : {
776 73 : xfs_fileoff_t bno; /* block number in file */
777 73 : struct xfs_buf *bp; /* temporary buffer for zeroing */
778 73 : xfs_daddr_t d; /* disk block address */
779 73 : int error; /* error return value */
780 73 : xfs_fsblock_t fsbno; /* filesystem block for bno */
781 73 : struct xfs_bmbt_irec map; /* block map output */
782 73 : int nmap; /* number of block maps */
783 73 : int resblks; /* space reservation */
784 73 : enum xfs_blft buf_type;
785 73 : struct xfs_trans *tp;
786 :
787 73 : if (ip == mp->m_rsumip)
788 : buf_type = XFS_BLFT_RTSUMMARY_BUF;
789 : else
790 42 : buf_type = XFS_BLFT_RTBITMAP_BUF;
791 :
792 : /*
793 : * Allocate space to the file, as necessary.
794 : */
795 154 : while (oblocks < nblocks) {
796 92 : resblks = XFS_GROWFSRT_SPACE_RES(mp, nblocks - oblocks);
797 : /*
798 : * Reserve space & log for one extent added to the file.
799 : */
800 92 : error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtalloc, resblks,
801 : 0, 0, &tp);
802 92 : if (error)
803 0 : return error;
804 : /*
805 : * Lock the inode.
806 : */
807 92 : xfs_ilock(ip, XFS_ILOCK_EXCL);
808 92 : xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
809 :
810 92 : error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK,
811 : XFS_IEXT_ADD_NOSPLIT_CNT);
812 92 : if (error == -EFBIG)
813 8 : error = xfs_iext_count_upgrade(tp, ip,
814 : XFS_IEXT_ADD_NOSPLIT_CNT);
815 92 : if (error)
816 8 : goto out_trans_cancel;
817 :
818 : /*
819 : * Allocate blocks to the bitmap file.
820 : */
821 84 : nmap = 1;
822 84 : error = xfs_bmapi_write(tp, ip, oblocks, nblocks - oblocks,
823 : XFS_BMAPI_METADATA, 0, &map, &nmap);
824 84 : if (!error && nmap < 1)
825 : error = -ENOSPC;
826 81 : if (error)
827 3 : goto out_trans_cancel;
828 : /*
829 : * Free any blocks freed up in the transaction, then commit.
830 : */
831 81 : error = xfs_trans_commit(tp);
832 81 : if (error)
833 0 : return error;
834 : /*
835 : * Now we need to clear the allocated blocks.
836 : * Do this one block per transaction, to keep it simple.
837 : */
838 81 : for (bno = map.br_startoff, fsbno = map.br_startblock;
839 180 : bno < map.br_startoff + map.br_blockcount;
840 99 : bno++, fsbno++) {
841 : /*
842 : * Reserve log for one block zeroing.
843 : */
844 99 : error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtzero,
845 : 0, 0, 0, &tp);
846 99 : if (error)
847 0 : return error;
848 : /*
849 : * Lock the bitmap inode.
850 : */
851 99 : xfs_ilock(ip, XFS_ILOCK_EXCL);
852 99 : xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
853 : /*
854 : * Get a buffer for the block.
855 : */
856 99 : d = XFS_FSB_TO_DADDR(mp, fsbno);
857 99 : error = xfs_trans_get_buf(tp, mp->m_ddev_targp, d,
858 : mp->m_bsize, 0, &bp);
859 99 : if (error)
860 0 : goto out_trans_cancel;
861 :
862 99 : xfs_trans_buf_set_type(tp, bp, buf_type);
863 99 : bp->b_ops = &xfs_rtbuf_ops;
864 99 : memset(bp->b_addr, 0, mp->m_sb.sb_blocksize);
865 99 : xfs_trans_log_buf(tp, bp, 0, mp->m_sb.sb_blocksize - 1);
866 : /*
867 : * Commit the transaction.
868 : */
869 99 : error = xfs_trans_commit(tp);
870 99 : if (error)
871 0 : return error;
872 : }
873 : /*
874 : * Go on to the next extent, if any.
875 : */
876 81 : oblocks = map.br_startoff + map.br_blockcount;
877 : }
878 :
879 : return 0;
880 :
881 11 : out_trans_cancel:
882 11 : xfs_trans_cancel(tp);
883 11 : return error;
884 : }
885 :
886 : static void
887 60711 : xfs_alloc_rsum_cache(
888 : xfs_mount_t *mp, /* file system mount structure */
889 : xfs_extlen_t rbmblocks) /* number of rt bitmap blocks */
890 : {
891 : /*
892 : * The rsum cache is initialized to all zeroes, which is trivially a
893 : * lower bound on the minimum level with any free extents. We can
894 : * continue without the cache if it couldn't be allocated.
895 : */
896 60711 : mp->m_rsum_cache = kvzalloc(rbmblocks, GFP_KERNEL);
897 60711 : if (!mp->m_rsum_cache)
898 0 : xfs_warn(mp, "could not allocate realtime summary cache");
899 60711 : }
900 :
901 : /*
902 : * Visible (exported) functions.
903 : */
904 :
905 : /*
906 : * Grow the realtime area of the filesystem.
907 : */
908 : int
909 47 : xfs_growfs_rt(
910 : xfs_mount_t *mp, /* mount point for filesystem */
911 : xfs_growfs_rt_t *in) /* growfs rt input struct */
912 : {
913 47 : xfs_rtblock_t bmbno; /* bitmap block number */
914 47 : struct xfs_buf *bp; /* temporary buffer */
915 47 : int error; /* error return value */
916 47 : xfs_mount_t *nmp; /* new (fake) mount structure */
917 47 : xfs_rfsblock_t nrblocks; /* new number of realtime blocks */
918 47 : xfs_extlen_t nrbmblocks; /* new number of rt bitmap blocks */
919 47 : xfs_rtblock_t nrextents; /* new number of realtime extents */
920 47 : uint8_t nrextslog; /* new log2 of sb_rextents */
921 47 : xfs_extlen_t nrsumblocks; /* new number of summary blocks */
922 47 : uint nrsumlevels; /* new rt summary levels */
923 47 : uint nrsumsize; /* new size of rt summary, bytes */
924 47 : xfs_sb_t *nsbp; /* new superblock */
925 47 : xfs_extlen_t rbmblocks; /* current number of rt bitmap blocks */
926 47 : xfs_extlen_t rsumblocks; /* current number of rt summary blks */
927 47 : xfs_sb_t *sbp; /* old superblock */
928 47 : xfs_fsblock_t sumbno; /* summary block number */
929 47 : uint8_t *rsum_cache; /* old summary cache */
930 :
931 47 : sbp = &mp->m_sb;
932 :
933 47 : if (!capable(CAP_SYS_ADMIN))
934 : return -EPERM;
935 :
936 : /* Needs to have been mounted with an rt device. */
937 47 : if (!XFS_IS_REALTIME_MOUNT(mp))
938 : return -EINVAL;
939 : /*
940 : * Mount should fail if the rt bitmap/summary files don't load, but
941 : * we'll check anyway.
942 : */
943 47 : if (!mp->m_rbmip || !mp->m_rsumip)
944 : return -EINVAL;
945 :
946 : /* Shrink not supported. */
947 47 : if (in->newblocks <= sbp->sb_rblocks)
948 : return -EINVAL;
949 :
950 : /* Can only change rt extent size when adding rt volume. */
951 47 : if (sbp->sb_rblocks > 0 && in->extsize != sbp->sb_rextsize)
952 : return -EINVAL;
953 :
954 : /* Range check the extent size. */
955 47 : if (XFS_FSB_TO_B(mp, in->extsize) > XFS_MAX_RTEXTSIZE ||
956 47 : XFS_FSB_TO_B(mp, in->extsize) < XFS_MIN_RTEXTSIZE)
957 0 : return -EINVAL;
958 :
959 : /* Unsupported realtime features. */
960 47 : if (xfs_has_rmapbt(mp) || xfs_has_reflink(mp))
961 : return -EOPNOTSUPP;
962 :
963 42 : nrblocks = in->newblocks;
964 42 : error = xfs_sb_validate_fsb_count(sbp, nrblocks);
965 42 : if (error)
966 : return error;
967 : /*
968 : * Read in the last block of the device, make sure it exists.
969 : */
970 126 : error = xfs_buf_read_uncached(mp->m_rtdev_targp,
971 42 : XFS_FSB_TO_BB(mp, nrblocks - 1),
972 42 : XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);
973 42 : if (error)
974 : return error;
975 42 : xfs_buf_relse(bp);
976 :
977 : /*
978 : * Calculate new parameters. These are the final values to be reached.
979 : */
980 42 : nrextents = nrblocks;
981 42 : do_div(nrextents, in->extsize);
982 42 : nrbmblocks = howmany_64(nrextents, NBBY * sbp->sb_blocksize);
983 42 : nrextslog = xfs_highbit32(nrextents);
984 42 : nrsumlevels = nrextslog + 1;
985 42 : nrsumsize = (uint)sizeof(xfs_suminfo_t) * nrsumlevels * nrbmblocks;
986 42 : nrsumblocks = XFS_B_TO_FSB(mp, nrsumsize);
987 42 : nrsumsize = XFS_FSB_TO_B(mp, nrsumblocks);
988 : /*
989 : * New summary size can't be more than half the size of
990 : * the log. This prevents us from getting a log overflow,
991 : * since we'll log basically the whole summary file at once.
992 : */
993 42 : if (nrsumblocks > (mp->m_sb.sb_logblocks >> 1))
994 : return -EINVAL;
995 : /*
996 : * Get the old block counts for bitmap and summary inodes.
997 : * These can't change since other growfs callers are locked out.
998 : */
999 42 : rbmblocks = XFS_B_TO_FSB(mp, mp->m_rbmip->i_disk_size);
1000 42 : rsumblocks = XFS_B_TO_FSB(mp, mp->m_rsumip->i_disk_size);
1001 : /*
1002 : * Allocate space to the bitmap and summary files, as necessary.
1003 : */
1004 42 : error = xfs_growfs_rt_alloc(mp, rbmblocks, nrbmblocks, mp->m_rbmip);
1005 42 : if (error)
1006 : return error;
1007 31 : error = xfs_growfs_rt_alloc(mp, rsumblocks, nrsumblocks, mp->m_rsumip);
1008 31 : if (error)
1009 : return error;
1010 :
1011 31 : rsum_cache = mp->m_rsum_cache;
1012 31 : if (nrbmblocks != sbp->sb_rbmblocks)
1013 9 : xfs_alloc_rsum_cache(mp, nrbmblocks);
1014 :
1015 : /*
1016 : * Allocate a new (fake) mount/sb.
1017 : */
1018 31 : nmp = kmem_alloc(sizeof(*nmp), 0);
1019 : /*
1020 : * Loop over the bitmap blocks.
1021 : * We will do everything one bitmap block at a time.
1022 : * Skip the current block if it is exactly full.
1023 : * This also deals with the case where there were no rtextents before.
1024 : */
1025 62 : for (bmbno = sbp->sb_rbmblocks -
1026 31 : ((sbp->sb_rextents & ((1 << mp->m_blkbit_log) - 1)) != 0);
1027 89 : bmbno < nrbmblocks;
1028 58 : bmbno++) {
1029 58 : struct xfs_trans *tp;
1030 58 : xfs_rfsblock_t nrblocks_step;
1031 :
1032 58 : *nmp = *mp;
1033 58 : nsbp = &nmp->m_sb;
1034 : /*
1035 : * Calculate new sb and mount fields for this round.
1036 : */
1037 58 : nsbp->sb_rextsize = in->extsize;
1038 58 : nsbp->sb_rbmblocks = bmbno + 1;
1039 58 : nrblocks_step = (bmbno + 1) * NBBY * nsbp->sb_blocksize *
1040 58 : nsbp->sb_rextsize;
1041 58 : nsbp->sb_rblocks = min(nrblocks, nrblocks_step);
1042 58 : nsbp->sb_rextents = nsbp->sb_rblocks;
1043 58 : do_div(nsbp->sb_rextents, nsbp->sb_rextsize);
1044 58 : ASSERT(nsbp->sb_rextents != 0);
1045 58 : nsbp->sb_rextslog = xfs_highbit32(nsbp->sb_rextents);
1046 58 : nrsumlevels = nmp->m_rsumlevels = nsbp->sb_rextslog + 1;
1047 58 : nrsumsize =
1048 58 : (uint)sizeof(xfs_suminfo_t) * nrsumlevels *
1049 58 : nsbp->sb_rbmblocks;
1050 58 : nrsumblocks = XFS_B_TO_FSB(mp, nrsumsize);
1051 58 : nmp->m_rsumsize = nrsumsize = XFS_FSB_TO_B(mp, nrsumblocks);
1052 : /*
1053 : * Start a transaction, get the log reservation.
1054 : */
1055 58 : error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtfree, 0, 0, 0,
1056 : &tp);
1057 58 : if (error)
1058 : break;
1059 : /*
1060 : * Lock out other callers by grabbing the bitmap inode lock.
1061 : */
1062 58 : xfs_ilock(mp->m_rbmip, XFS_ILOCK_EXCL | XFS_ILOCK_RTBITMAP);
1063 58 : xfs_trans_ijoin(tp, mp->m_rbmip, XFS_ILOCK_EXCL);
1064 : /*
1065 : * Update the bitmap inode's size ondisk and incore. We need
1066 : * to update the incore size so that inode inactivation won't
1067 : * punch what it thinks are "posteof" blocks.
1068 : */
1069 58 : mp->m_rbmip->i_disk_size =
1070 58 : nsbp->sb_rbmblocks * nsbp->sb_blocksize;
1071 58 : i_size_write(VFS_I(mp->m_rbmip), mp->m_rbmip->i_disk_size);
1072 58 : xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);
1073 : /*
1074 : * Get the summary inode into the transaction.
1075 : */
1076 58 : xfs_ilock(mp->m_rsumip, XFS_ILOCK_EXCL | XFS_ILOCK_RTSUM);
1077 58 : xfs_trans_ijoin(tp, mp->m_rsumip, XFS_ILOCK_EXCL);
1078 : /*
1079 : * Update the summary inode's size. We need to update the
1080 : * incore size so that inode inactivation won't punch what it
1081 : * thinks are "posteof" blocks.
1082 : */
1083 58 : mp->m_rsumip->i_disk_size = nmp->m_rsumsize;
1084 58 : i_size_write(VFS_I(mp->m_rsumip), mp->m_rsumip->i_disk_size);
1085 58 : xfs_trans_log_inode(tp, mp->m_rsumip, XFS_ILOG_CORE);
1086 : /*
1087 : * Copy summary data from old to new sizes.
1088 : * Do this when the real size (not block-aligned) changes.
1089 : */
1090 58 : if (sbp->sb_rbmblocks != nsbp->sb_rbmblocks ||
1091 31 : mp->m_rsumlevels != nmp->m_rsumlevels) {
1092 43 : error = xfs_rtcopy_summary(mp, nmp, tp);
1093 43 : if (error)
1094 0 : goto error_cancel;
1095 : }
1096 : /*
1097 : * Update superblock fields.
1098 : */
1099 58 : if (nsbp->sb_rextsize != sbp->sb_rextsize)
1100 0 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSIZE,
1101 0 : nsbp->sb_rextsize - sbp->sb_rextsize);
1102 58 : if (nsbp->sb_rbmblocks != sbp->sb_rbmblocks)
1103 27 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBMBLOCKS,
1104 27 : nsbp->sb_rbmblocks - sbp->sb_rbmblocks);
1105 58 : if (nsbp->sb_rblocks != sbp->sb_rblocks)
1106 58 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBLOCKS,
1107 58 : nsbp->sb_rblocks - sbp->sb_rblocks);
1108 58 : if (nsbp->sb_rextents != sbp->sb_rextents)
1109 58 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTENTS,
1110 58 : nsbp->sb_rextents - sbp->sb_rextents);
1111 58 : if (nsbp->sb_rextslog != sbp->sb_rextslog)
1112 25 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSLOG,
1113 25 : nsbp->sb_rextslog - sbp->sb_rextslog);
1114 : /*
1115 : * Free new extent.
1116 : */
1117 58 : bp = NULL;
1118 58 : error = xfs_rtfree_range(nmp, tp, sbp->sb_rextents,
1119 58 : nsbp->sb_rextents - sbp->sb_rextents, &bp, &sumbno);
1120 58 : if (error) {
1121 0 : error_cancel:
1122 0 : xfs_trans_cancel(tp);
1123 0 : break;
1124 : }
1125 : /*
1126 : * Mark more blocks free in the superblock.
1127 : */
1128 58 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS,
1129 58 : nsbp->sb_rextents - sbp->sb_rextents);
1130 : /*
1131 : * Update mp values into the real mp structure.
1132 : */
1133 58 : mp->m_rsumlevels = nrsumlevels;
1134 58 : mp->m_rsumsize = nrsumsize;
1135 :
1136 58 : error = xfs_trans_commit(tp);
1137 58 : if (error)
1138 : break;
1139 :
1140 : /* Ensure the mount RT feature flag is now set. */
1141 58 : mp->m_features |= XFS_FEAT_REALTIME;
1142 : }
1143 31 : if (error)
1144 0 : goto out_free;
1145 :
1146 : /* Update secondary superblocks now the physical grow has completed */
1147 31 : error = xfs_update_secondary_sbs(mp);
1148 :
1149 31 : out_free:
1150 : /*
1151 : * Free the fake mp structure.
1152 : */
1153 31 : kmem_free(nmp);
1154 :
1155 : /*
1156 : * If we had to allocate a new rsum_cache, we either need to free the
1157 : * old one (if we succeeded) or free the new one and restore the old one
1158 : * (if there was an error).
1159 : */
1160 31 : if (rsum_cache != mp->m_rsum_cache) {
1161 9 : if (error) {
1162 0 : kmem_free(mp->m_rsum_cache);
1163 0 : mp->m_rsum_cache = rsum_cache;
1164 : } else {
1165 9 : kmem_free(rsum_cache);
1166 : }
1167 : }
1168 :
1169 : return error;
1170 : }
1171 :
1172 : /*
1173 : * Allocate an extent in the realtime subvolume, with the usual allocation
1174 : * parameters. The length units are all in realtime extents, as is the
1175 : * result block number.
1176 : */
1177 : int /* error */
1178 57355602 : xfs_rtallocate_extent(
1179 : xfs_trans_t *tp, /* transaction pointer */
1180 : xfs_rtblock_t bno, /* starting block number to allocate */
1181 : xfs_extlen_t minlen, /* minimum length to allocate */
1182 : xfs_extlen_t maxlen, /* maximum length to allocate */
1183 : xfs_extlen_t *len, /* out: actual length allocated */
1184 : int wasdel, /* was a delayed allocation extent */
1185 : xfs_extlen_t prod, /* extent product factor */
1186 : xfs_rtblock_t *rtblock) /* out: start block allocated */
1187 : {
1188 57355602 : xfs_mount_t *mp = tp->t_mountp;
1189 57355602 : int error; /* error value */
1190 57355602 : xfs_rtblock_t r; /* result allocated block */
1191 57355602 : xfs_fsblock_t sb; /* summary file block number */
1192 57355602 : struct xfs_buf *sumbp; /* summary file block buffer */
1193 :
1194 57355602 : ASSERT(xfs_isilocked(mp->m_rbmip, XFS_ILOCK_EXCL));
1195 57355602 : ASSERT(minlen > 0 && minlen <= maxlen);
1196 :
1197 : /*
1198 : * If prod is set then figure out what to do to minlen and maxlen.
1199 : */
1200 57355602 : if (prod > 1) {
1201 40 : xfs_extlen_t i;
1202 :
1203 40 : if ((i = maxlen % prod))
1204 0 : maxlen -= i;
1205 40 : if ((i = minlen % prod))
1206 40 : minlen += prod - i;
1207 40 : if (maxlen < minlen) {
1208 0 : *rtblock = NULLRTBLOCK;
1209 0 : return 0;
1210 : }
1211 : }
1212 :
1213 57355602 : retry:
1214 57355612 : sumbp = NULL;
1215 57355612 : if (bno == 0) {
1216 1707313 : error = xfs_rtallocate_extent_size(mp, tp, minlen, maxlen, len,
1217 : &sumbp, &sb, prod, &r);
1218 : } else {
1219 55648299 : error = xfs_rtallocate_extent_near(mp, tp, bno, minlen, maxlen,
1220 : len, &sumbp, &sb, prod, &r);
1221 : }
1222 :
1223 57355612 : if (error)
1224 12 : return error;
1225 :
1226 : /*
1227 : * If it worked, update the superblock.
1228 : */
1229 57355600 : if (r != NULLRTBLOCK) {
1230 57355570 : long slen = (long)*len;
1231 :
1232 57355570 : ASSERT(*len >= minlen && *len <= maxlen);
1233 57355570 : if (wasdel)
1234 0 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_RES_FREXTENTS, -slen);
1235 : else
1236 57355570 : xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS, -slen);
1237 30 : } else if (prod > 1) {
1238 10 : prod = 1;
1239 10 : goto retry;
1240 : }
1241 :
1242 57355590 : *rtblock = r;
1243 57355590 : return 0;
1244 : }
1245 :
1246 : /*
1247 : * Initialize realtime fields in the mount structure.
1248 : */
1249 : int /* error */
1250 60864 : xfs_rtmount_init(
1251 : struct xfs_mount *mp) /* file system mount structure */
1252 : {
1253 60864 : struct xfs_buf *bp; /* buffer for last block of subvolume */
1254 60864 : struct xfs_sb *sbp; /* filesystem superblock copy in mount */
1255 60864 : xfs_daddr_t d; /* address of last block of subvolume */
1256 60864 : int error;
1257 :
1258 60864 : sbp = &mp->m_sb;
1259 60864 : if (sbp->sb_rblocks == 0)
1260 : return 0;
1261 16350 : if (mp->m_rtdev_targp == NULL) {
1262 100 : xfs_warn(mp,
1263 : "Filesystem has a realtime volume, use rtdev=device option");
1264 100 : return -ENODEV;
1265 : }
1266 16250 : mp->m_rsumlevels = sbp->sb_rextslog + 1;
1267 16250 : mp->m_rsumsize =
1268 16250 : (uint)sizeof(xfs_suminfo_t) * mp->m_rsumlevels *
1269 16250 : sbp->sb_rbmblocks;
1270 16250 : mp->m_rsumsize = roundup(mp->m_rsumsize, sbp->sb_blocksize);
1271 16250 : mp->m_rbmip = mp->m_rsumip = NULL;
1272 : /*
1273 : * Check that the realtime section is an ok size.
1274 : */
1275 16250 : d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);
1276 16250 : if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_rblocks) {
1277 0 : xfs_warn(mp, "realtime mount -- %llu != %llu",
1278 : (unsigned long long) XFS_BB_TO_FSB(mp, d),
1279 : (unsigned long long) mp->m_sb.sb_rblocks);
1280 0 : return -EFBIG;
1281 : }
1282 48750 : error = xfs_buf_read_uncached(mp->m_rtdev_targp,
1283 16250 : d - XFS_FSB_TO_BB(mp, 1),
1284 16250 : XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);
1285 16250 : if (error) {
1286 0 : xfs_warn(mp, "realtime device size check failed");
1287 0 : return error;
1288 : }
1289 16250 : xfs_buf_relse(bp);
1290 16250 : return 0;
1291 : }
1292 :
1293 : static int
1294 58455 : xfs_rtalloc_count_frextent(
1295 : struct xfs_mount *mp,
1296 : struct xfs_trans *tp,
1297 : const struct xfs_rtalloc_rec *rec,
1298 : void *priv)
1299 : {
1300 58455 : uint64_t *valp = priv;
1301 :
1302 58455 : *valp += rec->ar_extcount;
1303 58455 : return 0;
1304 : }
1305 :
1306 : /*
1307 : * Reinitialize the number of free realtime extents from the realtime bitmap.
1308 : * Callers must ensure that there is no other activity in the filesystem.
1309 : */
1310 : int
1311 1621 : xfs_rtalloc_reinit_frextents(
1312 : struct xfs_mount *mp)
1313 : {
1314 1621 : uint64_t val = 0;
1315 1621 : int error;
1316 :
1317 1621 : xfs_ilock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP);
1318 1621 : error = xfs_rtalloc_query_all(mp, NULL, xfs_rtalloc_count_frextent,
1319 : &val);
1320 1621 : xfs_iunlock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP);
1321 1621 : if (error)
1322 : return error;
1323 :
1324 1621 : spin_lock(&mp->m_sb_lock);
1325 1621 : mp->m_sb.sb_frextents = val;
1326 1621 : spin_unlock(&mp->m_sb_lock);
1327 1621 : percpu_counter_set(&mp->m_frextents, mp->m_sb.sb_frextents);
1328 1621 : return 0;
1329 : }
1330 :
1331 : /*
1332 : * Read in the bmbt of an rt metadata inode so that we never have to load them
1333 : * at runtime. This enables the use of shared ILOCKs for rtbitmap scans. Use
1334 : * an empty transaction to avoid deadlocking on loops in the bmbt.
1335 : */
1336 : static inline int
1337 121404 : xfs_rtmount_iread_extents(
1338 : struct xfs_inode *ip,
1339 : unsigned int lock_class)
1340 : {
1341 121404 : struct xfs_trans *tp;
1342 121404 : int error;
1343 :
1344 121404 : error = xfs_trans_alloc_empty(ip->i_mount, &tp);
1345 121404 : if (error)
1346 : return error;
1347 :
1348 121404 : xfs_ilock(ip, XFS_ILOCK_EXCL | lock_class);
1349 :
1350 121404 : error = xfs_iread_extents(tp, ip, XFS_DATA_FORK);
1351 121404 : if (error)
1352 0 : goto out_unlock;
1353 :
1354 121404 : if (xfs_inode_has_attr_fork(ip)) {
1355 110198 : error = xfs_iread_extents(tp, ip, XFS_ATTR_FORK);
1356 110198 : if (error)
1357 0 : goto out_unlock;
1358 : }
1359 :
1360 121404 : out_unlock:
1361 121404 : xfs_iunlock(ip, XFS_ILOCK_EXCL | lock_class);
1362 121404 : xfs_trans_cancel(tp);
1363 121404 : return error;
1364 : }
1365 :
1366 : /*
1367 : * Get the bitmap and summary inodes and the summary cache into the mount
1368 : * structure at mount time.
1369 : */
1370 : int /* error */
1371 60702 : xfs_rtmount_inodes(
1372 : xfs_mount_t *mp) /* file system mount structure */
1373 : {
1374 60702 : int error; /* error return value */
1375 60702 : xfs_sb_t *sbp;
1376 :
1377 60702 : sbp = &mp->m_sb;
1378 60702 : error = xfs_iget(mp, NULL, sbp->sb_rbmino, 0, 0, &mp->m_rbmip);
1379 60702 : if (xfs_metadata_is_sick(error))
1380 0 : xfs_rt_mark_sick(mp, XFS_SICK_RT_BITMAP);
1381 60702 : if (error)
1382 : return error;
1383 60702 : ASSERT(mp->m_rbmip != NULL);
1384 :
1385 60702 : error = xfs_rtmount_iread_extents(mp->m_rbmip, XFS_ILOCK_RTBITMAP);
1386 60702 : if (error)
1387 0 : goto out_rele_bitmap;
1388 :
1389 60702 : error = xfs_iget(mp, NULL, sbp->sb_rsumino, 0, 0, &mp->m_rsumip);
1390 60702 : if (xfs_metadata_is_sick(error))
1391 0 : xfs_rt_mark_sick(mp, XFS_SICK_RT_SUMMARY);
1392 60702 : if (error)
1393 0 : goto out_rele_bitmap;
1394 60702 : ASSERT(mp->m_rsumip != NULL);
1395 :
1396 60702 : error = xfs_rtmount_iread_extents(mp->m_rsumip, XFS_ILOCK_RTSUM);
1397 60702 : if (error)
1398 0 : goto out_rele_summary;
1399 :
1400 60702 : xfs_alloc_rsum_cache(mp, sbp->sb_rbmblocks);
1401 60702 : return 0;
1402 :
1403 : out_rele_summary:
1404 0 : xfs_irele(mp->m_rsumip);
1405 0 : out_rele_bitmap:
1406 0 : xfs_irele(mp->m_rbmip);
1407 0 : return error;
1408 : }
1409 :
1410 : void
1411 60712 : xfs_rtunmount_inodes(
1412 : struct xfs_mount *mp)
1413 : {
1414 60712 : kmem_free(mp->m_rsum_cache);
1415 60712 : if (mp->m_rbmip)
1416 60712 : xfs_irele(mp->m_rbmip);
1417 60712 : if (mp->m_rsumip)
1418 60712 : xfs_irele(mp->m_rsumip);
1419 60712 : }
1420 :
1421 : /*
1422 : * Pick an extent for allocation at the start of a new realtime file.
1423 : * Use the sequence number stored in the atime field of the bitmap inode.
1424 : * Translate this to a fraction of the rtextents, and return the product
1425 : * of rtextents and the fraction.
1426 : * The fraction sequence is 0, 1/2, 1/4, 3/4, 1/8, ..., 7/8, 1/16, ...
1427 : */
1428 : int /* error */
1429 6259690 : xfs_rtpick_extent(
1430 : xfs_mount_t *mp, /* file system mount point */
1431 : xfs_trans_t *tp, /* transaction pointer */
1432 : xfs_extlen_t len, /* allocation length (rtextents) */
1433 : xfs_rtblock_t *pick) /* result rt extent */
1434 : {
1435 6259690 : xfs_rtblock_t b; /* result block */
1436 6259690 : int log2; /* log of sequence number */
1437 6259690 : uint64_t resid; /* residual after log removed */
1438 6259690 : uint64_t seq; /* sequence number of file creation */
1439 6259690 : uint64_t *seqp; /* pointer to seqno in inode */
1440 :
1441 6259690 : ASSERT(xfs_isilocked(mp->m_rbmip, XFS_ILOCK_EXCL));
1442 :
1443 6259690 : seqp = (uint64_t *)&VFS_I(mp->m_rbmip)->i_atime;
1444 6259690 : if (!(mp->m_rbmip->i_diflags & XFS_DIFLAG_NEWRTBM)) {
1445 0 : mp->m_rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM;
1446 0 : *seqp = 0;
1447 : }
1448 6259690 : seq = *seqp;
1449 6259690 : if ((log2 = xfs_highbit64(seq)) == -1)
1450 9206 : b = 0;
1451 : else {
1452 6250484 : resid = seq - (1ULL << log2);
1453 0 : b = (mp->m_sb.sb_rextents * ((resid << 1) + 1ULL)) >>
1454 6250484 : (log2 + 1);
1455 6250484 : if (b >= mp->m_sb.sb_rextents)
1456 0 : div64_u64_rem(b, mp->m_sb.sb_rextents, &b);
1457 6250484 : if (b + len > mp->m_sb.sb_rextents)
1458 111 : b = mp->m_sb.sb_rextents - len;
1459 : }
1460 6259690 : *seqp = seq + 1;
1461 6259690 : xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);
1462 6259690 : *pick = b;
1463 6259690 : return 0;
1464 : }
1465 :
1466 : /*
1467 : * Decide if this is an unwritten extent that isn't aligned to a rt extent
1468 : * boundary. If it is, shorten the mapping so that we're ready to convert
1469 : * everything up to the next rt extent to a zeroed written extent. If not,
1470 : * return false.
1471 : */
1472 : static inline bool
1473 0 : xfs_rtfile_want_conversion(
1474 : struct xfs_mount *mp,
1475 : struct xfs_bmbt_irec *irec)
1476 : {
1477 0 : xfs_fileoff_t rext_next;
1478 0 : uint32_t modoff, modcnt;
1479 :
1480 0 : if (irec->br_state != XFS_EXT_UNWRITTEN)
1481 : return false;
1482 :
1483 0 : div_u64_rem(irec->br_startoff, mp->m_sb.sb_rextsize, &modoff);
1484 0 : if (modoff == 0) {
1485 0 : uint64_t rexts = div_u64_rem(irec->br_blockcount,
1486 : mp->m_sb.sb_rextsize, &modcnt);
1487 :
1488 0 : if (rexts > 0) {
1489 : /*
1490 : * Unwritten mapping starts at an rt extent boundary
1491 : * and is longer than one rt extent. Round the length
1492 : * down to the nearest extent but don't select it for
1493 : * conversion.
1494 : */
1495 0 : irec->br_blockcount -= modcnt;
1496 0 : modcnt = 0;
1497 : }
1498 :
1499 : /* Unwritten mapping is perfectly aligned, do not convert. */
1500 0 : if (modcnt == 0)
1501 : return false;
1502 : }
1503 :
1504 : /*
1505 : * Unaligned and unwritten; trim to the current rt extent and select it
1506 : * for conversion.
1507 : */
1508 0 : rext_next = (irec->br_startoff - modoff) + mp->m_sb.sb_rextsize;
1509 0 : xfs_trim_extent(irec, irec->br_startoff, rext_next - irec->br_startoff);
1510 0 : return true;
1511 : }
1512 :
1513 : /*
1514 : * Find an unwritten extent in the given file range, zero it, and convert the
1515 : * mapping to written. Adjust the scan cursor on the way out.
1516 : */
1517 : STATIC int
1518 0 : xfs_rtfile_convert_one(
1519 : struct xfs_inode *ip,
1520 : xfs_fileoff_t *offp,
1521 : xfs_fileoff_t endoff)
1522 : {
1523 0 : struct xfs_bmbt_irec irec;
1524 0 : struct xfs_mount *mp = ip->i_mount;
1525 0 : struct xfs_trans *tp;
1526 0 : unsigned int resblks;
1527 0 : int nmap;
1528 0 : int error;
1529 :
1530 0 : resblks = XFS_DIOSTRAT_SPACE_RES(mp, 1);
1531 0 : error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp);
1532 0 : if (error)
1533 : return error;
1534 :
1535 0 : xfs_ilock(ip, XFS_ILOCK_EXCL);
1536 0 : xfs_trans_ijoin(tp, ip, 0);
1537 :
1538 : /*
1539 : * Read the mapping. If we find an unwritten extent that isn't aligned
1540 : * to an rt extent boundary...
1541 : */
1542 0 : retry:
1543 0 : nmap = 1;
1544 0 : error = xfs_bmapi_read(ip, *offp, endoff - *offp, &irec, &nmap, 0);
1545 0 : if (error)
1546 0 : goto out_cancel;
1547 0 : ASSERT(nmap == 1);
1548 0 : ASSERT(irec.br_startoff == *offp);
1549 0 : if (!xfs_rtfile_want_conversion(mp, &irec)) {
1550 0 : *offp = irec.br_startoff + irec.br_blockcount;
1551 0 : if (*offp >= endoff)
1552 0 : goto out_cancel;
1553 0 : goto retry;
1554 : }
1555 :
1556 : /*
1557 : * ...make sure this partially unwritten rt extent gets converted to a
1558 : * zeroed written extent that we can remap.
1559 : */
1560 0 : nmap = 1;
1561 0 : error = xfs_bmapi_write(tp, ip, irec.br_startoff, irec.br_blockcount,
1562 : XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO, 0, &irec, &nmap);
1563 0 : if (error)
1564 0 : goto out_cancel;
1565 0 : ASSERT(nmap == 1);
1566 0 : if (irec.br_state != XFS_EXT_NORM) {
1567 0 : ASSERT(0);
1568 0 : error = -EIO;
1569 0 : goto out_cancel;
1570 : }
1571 0 : error = xfs_trans_commit(tp);
1572 0 : if (error)
1573 0 : goto out_unlock;
1574 :
1575 0 : xfs_iunlock(ip, XFS_ILOCK_EXCL);
1576 0 : *offp = irec.br_startoff + irec.br_blockcount;
1577 0 : return 0;
1578 :
1579 0 : out_cancel:
1580 0 : xfs_trans_cancel(tp);
1581 0 : out_unlock:
1582 0 : xfs_iunlock(ip, XFS_ILOCK_EXCL);
1583 0 : return error;
1584 : }
1585 :
1586 : /*
1587 : * For all realtime extents backing the given range of a file, search for
1588 : * unwritten mappings that do not cover a full rt extent and convert them
1589 : * to zeroed written mappings. The goal is to end up with one mapping per rt
1590 : * extent so that we can perform a remapping operation. Callers must ensure
1591 : * that there are no dirty pages in the given range.
1592 : */
1593 : int
1594 0 : xfs_rtfile_convert_unwritten(
1595 : struct xfs_inode *ip,
1596 : loff_t pos,
1597 : uint64_t len)
1598 : {
1599 0 : struct xfs_mount *mp = ip->i_mount;
1600 0 : xfs_fileoff_t off;
1601 0 : xfs_fileoff_t endoff;
1602 0 : int error;
1603 :
1604 0 : if (mp->m_sb.sb_rextsize == 1)
1605 : return 0;
1606 :
1607 0 : off = rounddown_64(XFS_B_TO_FSBT(mp, pos), mp->m_sb.sb_rextsize);
1608 0 : endoff = roundup_64(XFS_B_TO_FSB(mp, pos + len), mp->m_sb.sb_rextsize);
1609 :
1610 0 : trace_xfs_rtfile_convert_unwritten(ip, pos, len);
1611 :
1612 0 : while (off < endoff) {
1613 0 : if (fatal_signal_pending(current))
1614 : return -EINTR;
1615 :
1616 0 : error = xfs_rtfile_convert_one(ip, &off, endoff);
1617 0 : if (error)
1618 0 : return error;
1619 : }
1620 :
1621 : return 0;
1622 : }
|
