Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2014 Red Hat, Inc.
4 : * All Rights Reserved.
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_bit.h"
13 : #include "xfs_mount.h"
14 : #include "xfs_sb.h"
15 : #include "xfs_defer.h"
16 : #include "xfs_btree.h"
17 : #include "xfs_trans.h"
18 : #include "xfs_alloc.h"
19 : #include "xfs_rmap.h"
20 : #include "xfs_rmap_btree.h"
21 : #include "xfs_trace.h"
22 : #include "xfs_errortag.h"
23 : #include "xfs_error.h"
24 : #include "xfs_inode.h"
25 : #include "xfs_ag.h"
26 : #include "xfs_health.h"
27 :
28 : struct kmem_cache *xfs_rmap_intent_cache;
29 :
30 : /*
31 : * Lookup the first record less than or equal to [bno, len, owner, offset]
32 : * in the btree given by cur.
33 : */
34 : int
35 430531461 : xfs_rmap_lookup_le(
36 : struct xfs_btree_cur *cur,
37 : xfs_agblock_t bno,
38 : uint64_t owner,
39 : uint64_t offset,
40 : unsigned int flags,
41 : struct xfs_rmap_irec *irec,
42 : int *stat)
43 : {
44 430531461 : int get_stat = 0;
45 430531461 : int error;
46 :
47 430531461 : cur->bc_rec.r.rm_startblock = bno;
48 430531461 : cur->bc_rec.r.rm_blockcount = 0;
49 430531461 : cur->bc_rec.r.rm_owner = owner;
50 430531461 : cur->bc_rec.r.rm_offset = offset;
51 430531461 : cur->bc_rec.r.rm_flags = flags;
52 :
53 430531461 : error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat);
54 430532927 : if (error || !(*stat) || !irec)
55 : return error;
56 :
57 424540252 : error = xfs_rmap_get_rec(cur, irec, &get_stat);
58 424543281 : if (error)
59 : return error;
60 424543281 : if (!get_stat) {
61 0 : xfs_btree_mark_sick(cur);
62 0 : return -EFSCORRUPTED;
63 : }
64 :
65 : return 0;
66 : }
67 :
68 : /*
69 : * Lookup the record exactly matching [bno, len, owner, offset]
70 : * in the btree given by cur.
71 : */
72 : int
73 0 : xfs_rmap_lookup_eq(
74 : struct xfs_btree_cur *cur,
75 : xfs_agblock_t bno,
76 : xfs_extlen_t len,
77 : uint64_t owner,
78 : uint64_t offset,
79 : unsigned int flags,
80 : int *stat)
81 : {
82 247243694 : cur->bc_rec.r.rm_startblock = bno;
83 247243694 : cur->bc_rec.r.rm_blockcount = len;
84 247243694 : cur->bc_rec.r.rm_owner = owner;
85 247243694 : cur->bc_rec.r.rm_offset = offset;
86 247243694 : cur->bc_rec.r.rm_flags = flags;
87 0 : return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat);
88 : }
89 :
90 : /*
91 : * Update the record referred to by cur to the value given
92 : * by [bno, len, owner, offset].
93 : * This either works (return 0) or gets an EFSCORRUPTED error.
94 : */
95 : STATIC int
96 32151720 : xfs_rmap_update(
97 : struct xfs_btree_cur *cur,
98 : struct xfs_rmap_irec *irec)
99 : {
100 32151720 : union xfs_btree_rec rec;
101 32151720 : int error;
102 :
103 32151720 : trace_xfs_rmap_update(cur->bc_mp, cur->bc_ag.pag->pag_agno,
104 : irec->rm_startblock, irec->rm_blockcount,
105 : irec->rm_owner, irec->rm_offset, irec->rm_flags);
106 :
107 32151710 : rec.rmap.rm_startblock = cpu_to_be32(irec->rm_startblock);
108 32151710 : rec.rmap.rm_blockcount = cpu_to_be32(irec->rm_blockcount);
109 32151710 : rec.rmap.rm_owner = cpu_to_be64(irec->rm_owner);
110 32151710 : rec.rmap.rm_offset = cpu_to_be64(
111 : xfs_rmap_irec_offset_pack(irec));
112 32151710 : error = xfs_btree_update(cur, &rec);
113 32151721 : if (error)
114 0 : trace_xfs_rmap_update_error(cur->bc_mp,
115 0 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
116 32151721 : return error;
117 : }
118 :
119 : int
120 85892310 : xfs_rmap_insert(
121 : struct xfs_btree_cur *rcur,
122 : xfs_agblock_t agbno,
123 : xfs_extlen_t len,
124 : uint64_t owner,
125 : uint64_t offset,
126 : unsigned int flags)
127 : {
128 85892310 : int i;
129 85892310 : int error;
130 :
131 85892310 : trace_xfs_rmap_insert(rcur->bc_mp, rcur->bc_ag.pag->pag_agno, agbno,
132 : len, owner, offset, flags);
133 :
134 85892431 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
135 85892510 : if (error)
136 0 : goto done;
137 85892510 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 0)) {
138 0 : xfs_btree_mark_sick(rcur);
139 0 : error = -EFSCORRUPTED;
140 0 : goto done;
141 : }
142 :
143 85892510 : rcur->bc_rec.r.rm_startblock = agbno;
144 85892510 : rcur->bc_rec.r.rm_blockcount = len;
145 85892510 : rcur->bc_rec.r.rm_owner = owner;
146 85892510 : rcur->bc_rec.r.rm_offset = offset;
147 85892510 : rcur->bc_rec.r.rm_flags = flags;
148 85892510 : error = xfs_btree_insert(rcur, &i);
149 85892523 : if (error)
150 158 : goto done;
151 85892365 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
152 0 : xfs_btree_mark_sick(rcur);
153 0 : error = -EFSCORRUPTED;
154 0 : goto done;
155 : }
156 85892523 : done:
157 85892523 : if (error)
158 158 : trace_xfs_rmap_insert_error(rcur->bc_mp,
159 158 : rcur->bc_ag.pag->pag_agno, error, _RET_IP_);
160 85892523 : return error;
161 : }
162 :
163 : STATIC int
164 55209268 : xfs_rmap_delete(
165 : struct xfs_btree_cur *rcur,
166 : xfs_agblock_t agbno,
167 : xfs_extlen_t len,
168 : uint64_t owner,
169 : uint64_t offset,
170 : unsigned int flags)
171 : {
172 55209268 : int i;
173 55209268 : int error;
174 :
175 55209268 : trace_xfs_rmap_delete(rcur->bc_mp, rcur->bc_ag.pag->pag_agno, agbno,
176 : len, owner, offset, flags);
177 :
178 55209282 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
179 55209342 : if (error)
180 0 : goto done;
181 55209342 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
182 0 : xfs_btree_mark_sick(rcur);
183 0 : error = -EFSCORRUPTED;
184 0 : goto done;
185 : }
186 :
187 55209342 : error = xfs_btree_delete(rcur, &i);
188 55209310 : if (error)
189 0 : goto done;
190 55209310 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
191 0 : xfs_btree_mark_sick(rcur);
192 0 : error = -EFSCORRUPTED;
193 0 : goto done;
194 : }
195 55209310 : done:
196 55209310 : if (error)
197 0 : trace_xfs_rmap_delete_error(rcur->bc_mp,
198 0 : rcur->bc_ag.pag->pag_agno, error, _RET_IP_);
199 55209310 : return error;
200 : }
201 :
202 : /* Convert an internal btree record to an rmap record. */
203 : xfs_failaddr_t
204 50906426272 : xfs_rmap_btrec_to_irec(
205 : const union xfs_btree_rec *rec,
206 : struct xfs_rmap_irec *irec)
207 : {
208 50906426272 : irec->rm_startblock = be32_to_cpu(rec->rmap.rm_startblock);
209 50906426272 : irec->rm_blockcount = be32_to_cpu(rec->rmap.rm_blockcount);
210 50906426272 : irec->rm_owner = be64_to_cpu(rec->rmap.rm_owner);
211 50906426272 : return xfs_rmap_irec_offset_unpack(be64_to_cpu(rec->rmap.rm_offset),
212 : irec);
213 : }
214 :
215 : inline xfs_failaddr_t
216 51208080245 : xfs_rmap_check_perag_irec(
217 : struct xfs_perag *pag,
218 : const struct xfs_rmap_irec *irec)
219 : {
220 51208080245 : struct xfs_mount *mp = pag->pag_mount;
221 51208080245 : bool is_inode;
222 51208080245 : bool is_unwritten;
223 51208080245 : bool is_bmbt;
224 51208080245 : bool is_attr;
225 :
226 51208080245 : if (irec->rm_blockcount == 0)
227 0 : return __this_address;
228 51208080245 : if (irec->rm_startblock <= XFS_AGFL_BLOCK(mp)) {
229 2977342 : if (irec->rm_owner != XFS_RMAP_OWN_FS)
230 0 : return __this_address;
231 2977342 : if (irec->rm_blockcount != XFS_AGFL_BLOCK(mp) + 1)
232 0 : return __this_address;
233 : } else {
234 : /* check for valid extent range, including overflow */
235 51205102903 : if (!xfs_verify_agbext(pag, irec->rm_startblock,
236 : irec->rm_blockcount))
237 0 : return __this_address;
238 : }
239 :
240 51208080245 : if (!(xfs_verify_ino(mp, irec->rm_owner) ||
241 1208152622 : (irec->rm_owner <= XFS_RMAP_OWN_FS &&
242 : irec->rm_owner >= XFS_RMAP_OWN_MIN)))
243 0 : return __this_address;
244 :
245 : /* Check flags. */
246 51235374653 : is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner);
247 51235374653 : is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK;
248 51235374653 : is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK;
249 51235374653 : is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN;
250 :
251 51235374653 : if (is_bmbt && irec->rm_offset != 0)
252 0 : return __this_address;
253 :
254 51235374653 : if (!is_inode && irec->rm_offset != 0)
255 0 : return __this_address;
256 :
257 51235374653 : if (is_unwritten && (is_bmbt || !is_inode || is_attr))
258 0 : return __this_address;
259 :
260 51235374653 : if (!is_inode && (is_bmbt || is_unwritten || is_attr))
261 0 : return __this_address;
262 :
263 : /* Check for a valid fork offset, if applicable. */
264 >10038*10^7 : if (is_inode && !is_bmbt &&
265 49117316181 : !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
266 0 : return __this_address;
267 :
268 : return NULL;
269 : }
270 :
271 : /* Simple checks for rmap records. */
272 : xfs_failaddr_t
273 51237062572 : xfs_rmap_check_irec(
274 : struct xfs_btree_cur *cur,
275 : const struct xfs_rmap_irec *irec)
276 : {
277 51237062572 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
278 46959925 : return xfs_rmap_check_perag_irec(cur->bc_mem.pag, irec);
279 51190102647 : return xfs_rmap_check_perag_irec(cur->bc_ag.pag, irec);
280 : }
281 :
282 : static inline int
283 0 : xfs_rmap_complain_bad_rec(
284 : struct xfs_btree_cur *cur,
285 : xfs_failaddr_t fa,
286 : const struct xfs_rmap_irec *irec)
287 : {
288 0 : struct xfs_mount *mp = cur->bc_mp;
289 :
290 0 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
291 0 : xfs_warn(mp,
292 : "In-Memory Reverse Mapping BTree record corruption detected at %pS!", fa);
293 : else
294 0 : xfs_warn(mp,
295 : "Reverse Mapping BTree record corruption in AG %d detected at %pS!",
296 : cur->bc_ag.pag->pag_agno, fa);
297 0 : xfs_warn(mp,
298 : "Owner 0x%llx, flags 0x%x, start block 0x%x block count 0x%x",
299 : irec->rm_owner, irec->rm_flags, irec->rm_startblock,
300 : irec->rm_blockcount);
301 0 : xfs_btree_mark_sick(cur);
302 0 : return -EFSCORRUPTED;
303 : }
304 :
305 : /*
306 : * Get the data from the pointed-to record.
307 : */
308 : int
309 1012687897 : xfs_rmap_get_rec(
310 : struct xfs_btree_cur *cur,
311 : struct xfs_rmap_irec *irec,
312 : int *stat)
313 : {
314 1012687897 : union xfs_btree_rec *rec;
315 1012687897 : xfs_failaddr_t fa;
316 1012687897 : int error;
317 :
318 1012687897 : error = xfs_btree_get_rec(cur, &rec, stat);
319 1012690731 : if (error || !*stat)
320 : return error;
321 :
322 1012692430 : fa = xfs_rmap_btrec_to_irec(rec, irec);
323 1012694981 : if (!fa)
324 1012692792 : fa = xfs_rmap_check_irec(cur, irec);
325 1012696569 : if (fa)
326 0 : return xfs_rmap_complain_bad_rec(cur, fa, irec);
327 :
328 : return 0;
329 : }
330 :
331 : struct xfs_find_left_neighbor_info {
332 : struct xfs_rmap_irec high;
333 : struct xfs_rmap_irec *irec;
334 : };
335 :
336 : /* For each rmap given, figure out if it matches the key we want. */
337 : STATIC int
338 104703641 : xfs_rmap_find_left_neighbor_helper(
339 : struct xfs_btree_cur *cur,
340 : const struct xfs_rmap_irec *rec,
341 : void *priv)
342 : {
343 104703641 : struct xfs_find_left_neighbor_info *info = priv;
344 :
345 104703641 : trace_xfs_rmap_find_left_neighbor_candidate(cur->bc_mp,
346 104703641 : cur->bc_ag.pag->pag_agno, rec->rm_startblock,
347 104703641 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
348 104703641 : rec->rm_flags);
349 :
350 104703898 : if (rec->rm_owner != info->high.rm_owner)
351 : return 0;
352 26973266 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
353 26973266 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
354 26954507 : rec->rm_offset + rec->rm_blockcount - 1 != info->high.rm_offset)
355 : return 0;
356 :
357 9812275 : *info->irec = *rec;
358 9812275 : return -ECANCELED;
359 : }
360 :
361 : /*
362 : * Find the record to the left of the given extent, being careful only to
363 : * return a match with the same owner and adjacent physical and logical
364 : * block ranges.
365 : */
366 : STATIC int
367 89110501 : xfs_rmap_find_left_neighbor(
368 : struct xfs_btree_cur *cur,
369 : xfs_agblock_t bno,
370 : uint64_t owner,
371 : uint64_t offset,
372 : unsigned int flags,
373 : struct xfs_rmap_irec *irec,
374 : int *stat)
375 : {
376 89110501 : struct xfs_find_left_neighbor_info info;
377 89110501 : int found = 0;
378 89110501 : int error;
379 :
380 89110501 : *stat = 0;
381 89110501 : if (bno == 0)
382 : return 0;
383 89110501 : info.high.rm_startblock = bno - 1;
384 89110501 : info.high.rm_owner = owner;
385 89110501 : if (!XFS_RMAP_NON_INODE_OWNER(owner) &&
386 89110389 : !(flags & XFS_RMAP_BMBT_BLOCK)) {
387 89110501 : if (offset == 0)
388 : return 0;
389 88632948 : info.high.rm_offset = offset - 1;
390 : } else
391 0 : info.high.rm_offset = 0;
392 88632948 : info.high.rm_flags = flags;
393 88632948 : info.high.rm_blockcount = 0;
394 88632948 : info.irec = irec;
395 :
396 88632948 : trace_xfs_rmap_find_left_neighbor_query(cur->bc_mp,
397 88632948 : cur->bc_ag.pag->pag_agno, bno, 0, owner, offset, flags);
398 :
399 : /*
400 : * Historically, we always used the range query to walk every reverse
401 : * mapping that could possibly overlap the key that the caller asked
402 : * for, and filter out the ones that don't. That is very slow when
403 : * there are a lot of records.
404 : *
405 : * However, there are two scenarios where the classic btree search can
406 : * produce correct results -- if the index contains a record that is an
407 : * exact match for the lookup key; and if there are no other records
408 : * between the record we want and the key we supplied.
409 : *
410 : * As an optimization, try a non-overlapped lookup first. This makes
411 : * extent conversion and remap operations run a bit faster if the
412 : * physical extents aren't being shared. If we don't find what we
413 : * want, we fall back to the overlapped query.
414 : */
415 88632939 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
416 : &found);
417 88633057 : if (error)
418 : return error;
419 88632843 : if (found)
420 88633058 : error = xfs_rmap_find_left_neighbor_helper(cur, irec, &info);
421 88632840 : if (!error)
422 79183409 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
423 : xfs_rmap_find_left_neighbor_helper, &info);
424 88632854 : if (error != -ECANCELED)
425 : return error;
426 :
427 9812275 : *stat = 1;
428 9812275 : trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
429 9812275 : cur->bc_ag.pag->pag_agno, irec->rm_startblock,
430 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
431 : irec->rm_flags);
432 9812275 : return 0;
433 : }
434 :
435 : /* For each rmap given, figure out if it matches the key we want. */
436 : STATIC int
437 112999935 : xfs_rmap_lookup_le_range_helper(
438 : struct xfs_btree_cur *cur,
439 : const struct xfs_rmap_irec *rec,
440 : void *priv)
441 : {
442 112999935 : struct xfs_find_left_neighbor_info *info = priv;
443 :
444 112999935 : trace_xfs_rmap_lookup_le_range_candidate(cur->bc_mp,
445 112999935 : cur->bc_ag.pag->pag_agno, rec->rm_startblock,
446 112999935 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
447 112999935 : rec->rm_flags);
448 :
449 113000077 : if (rec->rm_owner != info->high.rm_owner)
450 : return 0;
451 105904340 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
452 105904340 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
453 105904340 : (rec->rm_offset > info->high.rm_offset ||
454 105887723 : rec->rm_offset + rec->rm_blockcount <= info->high.rm_offset))
455 : return 0;
456 :
457 105881907 : *info->irec = *rec;
458 105881907 : return -ECANCELED;
459 : }
460 :
461 : /*
462 : * Find the record to the left of the given extent, being careful only to
463 : * return a match with the same owner and overlapping physical and logical
464 : * block ranges. This is the overlapping-interval version of
465 : * xfs_rmap_lookup_le.
466 : */
467 : int
468 105881662 : xfs_rmap_lookup_le_range(
469 : struct xfs_btree_cur *cur,
470 : xfs_agblock_t bno,
471 : uint64_t owner,
472 : uint64_t offset,
473 : unsigned int flags,
474 : struct xfs_rmap_irec *irec,
475 : int *stat)
476 : {
477 105881662 : struct xfs_find_left_neighbor_info info;
478 105881662 : int found = 0;
479 105881662 : int error;
480 :
481 105881662 : info.high.rm_startblock = bno;
482 105881662 : info.high.rm_owner = owner;
483 105881662 : if (!XFS_RMAP_NON_INODE_OWNER(owner) && !(flags & XFS_RMAP_BMBT_BLOCK))
484 105881662 : info.high.rm_offset = offset;
485 : else
486 0 : info.high.rm_offset = 0;
487 105881662 : info.high.rm_flags = flags;
488 105881662 : info.high.rm_blockcount = 0;
489 105881662 : *stat = 0;
490 105881662 : info.irec = irec;
491 :
492 105881662 : trace_xfs_rmap_lookup_le_range(cur->bc_mp, cur->bc_ag.pag->pag_agno,
493 : bno, 0, owner, offset, flags);
494 :
495 : /*
496 : * Historically, we always used the range query to walk every reverse
497 : * mapping that could possibly overlap the key that the caller asked
498 : * for, and filter out the ones that don't. That is very slow when
499 : * there are a lot of records.
500 : *
501 : * However, there are two scenarios where the classic btree search can
502 : * produce correct results -- if the index contains a record that is an
503 : * exact match for the lookup key; and if there are no other records
504 : * between the record we want and the key we supplied.
505 : *
506 : * As an optimization, try a non-overlapped lookup first. This makes
507 : * scrub run much faster on most filesystems because bmbt records are
508 : * usually an exact match for rmap records. If we don't find what we
509 : * want, we fall back to the overlapped query.
510 : */
511 105881573 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
512 : &found);
513 105881954 : if (error)
514 : return error;
515 105881667 : if (found)
516 105881773 : error = xfs_rmap_lookup_le_range_helper(cur, irec, &info);
517 105881864 : if (!error)
518 84944 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
519 : xfs_rmap_lookup_le_range_helper, &info);
520 105881864 : if (error != -ECANCELED)
521 : return error;
522 :
523 105881870 : *stat = 1;
524 105881870 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
525 105881870 : cur->bc_ag.pag->pag_agno, irec->rm_startblock,
526 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
527 : irec->rm_flags);
528 105881870 : return 0;
529 : }
530 :
531 : /*
532 : * Perform all the relevant owner checks for a removal op. If we're doing an
533 : * unknown-owner removal then we have no owner information to check.
534 : */
535 : static int
536 43547519 : xfs_rmap_free_check_owner(
537 : struct xfs_btree_cur *cur,
538 : uint64_t ltoff,
539 : struct xfs_rmap_irec *rec,
540 : xfs_filblks_t len,
541 : uint64_t owner,
542 : uint64_t offset,
543 : unsigned int flags)
544 : {
545 43547519 : struct xfs_mount *mp = cur->bc_mp;
546 43547519 : int error = 0;
547 :
548 43547519 : if (owner == XFS_RMAP_OWN_UNKNOWN)
549 : return 0;
550 :
551 : /* Make sure the unwritten flag matches. */
552 43547091 : if (XFS_IS_CORRUPT(mp,
553 : (flags & XFS_RMAP_UNWRITTEN) !=
554 : (rec->rm_flags & XFS_RMAP_UNWRITTEN))) {
555 0 : xfs_btree_mark_sick(cur);
556 0 : error = -EFSCORRUPTED;
557 0 : goto out;
558 : }
559 :
560 : /* Make sure the owner matches what we expect to find in the tree. */
561 43547091 : if (XFS_IS_CORRUPT(mp, owner != rec->rm_owner)) {
562 0 : xfs_btree_mark_sick(cur);
563 0 : error = -EFSCORRUPTED;
564 0 : goto out;
565 : }
566 :
567 : /* Check the offset, if necessary. */
568 43547091 : if (XFS_RMAP_NON_INODE_OWNER(owner))
569 2640944 : goto out;
570 :
571 40906147 : if (flags & XFS_RMAP_BMBT_BLOCK) {
572 511194 : if (XFS_IS_CORRUPT(mp,
573 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK))) {
574 0 : xfs_btree_mark_sick(cur);
575 0 : error = -EFSCORRUPTED;
576 0 : goto out;
577 : }
578 : } else {
579 40394953 : if (XFS_IS_CORRUPT(mp, rec->rm_offset > offset)) {
580 0 : xfs_btree_mark_sick(cur);
581 0 : error = -EFSCORRUPTED;
582 0 : goto out;
583 : }
584 40394953 : if (XFS_IS_CORRUPT(mp,
585 : offset + len > ltoff + rec->rm_blockcount)) {
586 0 : xfs_btree_mark_sick(cur);
587 0 : error = -EFSCORRUPTED;
588 0 : goto out;
589 : }
590 : }
591 :
592 : out:
593 : return error;
594 : }
595 :
596 : /*
597 : * Find the extent in the rmap btree and remove it.
598 : *
599 : * The record we find should always be an exact match for the extent that we're
600 : * looking for, since we insert them into the btree without modification.
601 : *
602 : * Special Case #1: when growing the filesystem, we "free" an extent when
603 : * growing the last AG. This extent is new space and so it is not tracked as
604 : * used space in the btree. The growfs code will pass in an owner of
605 : * XFS_RMAP_OWN_NULL to indicate that it expected that there is no owner of this
606 : * extent. We verify that - the extent lookup result in a record that does not
607 : * overlap.
608 : *
609 : * Special Case #2: EFIs do not record the owner of the extent, so when
610 : * recovering EFIs from the log we pass in XFS_RMAP_OWN_UNKNOWN to tell the rmap
611 : * btree to ignore the owner (i.e. wildcard match) so we don't trigger
612 : * corruption checks during log recovery.
613 : */
614 : STATIC int
615 43556246 : xfs_rmap_unmap(
616 : struct xfs_btree_cur *cur,
617 : xfs_agblock_t bno,
618 : xfs_extlen_t len,
619 : bool unwritten,
620 : const struct xfs_owner_info *oinfo)
621 : {
622 43556246 : struct xfs_mount *mp = cur->bc_mp;
623 43556246 : struct xfs_rmap_irec ltrec;
624 43556246 : uint64_t ltoff;
625 43556246 : int error = 0;
626 43556246 : int i;
627 43556246 : uint64_t owner;
628 43556246 : uint64_t offset;
629 43556246 : unsigned int flags;
630 43556246 : bool ignore_off;
631 :
632 43556246 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
633 43556246 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
634 40906744 : (flags & XFS_RMAP_BMBT_BLOCK);
635 43556246 : if (unwritten)
636 1425178 : flags |= XFS_RMAP_UNWRITTEN;
637 43556246 : trace_xfs_rmap_unmap(mp, cur->bc_ag.pag->pag_agno, bno, len,
638 : unwritten, oinfo);
639 :
640 : /*
641 : * We should always have a left record because there's a static record
642 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
643 : * will not ever be removed from the tree.
644 : */
645 43556204 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec, &i);
646 43555814 : if (error)
647 72 : goto out_error;
648 43555742 : if (XFS_IS_CORRUPT(mp, i != 1)) {
649 0 : xfs_btree_mark_sick(cur);
650 0 : error = -EFSCORRUPTED;
651 0 : goto out_error;
652 : }
653 :
654 43555742 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
655 43555742 : cur->bc_ag.pag->pag_agno, ltrec.rm_startblock,
656 : ltrec.rm_blockcount, ltrec.rm_owner,
657 : ltrec.rm_offset, ltrec.rm_flags);
658 43555737 : ltoff = ltrec.rm_offset;
659 :
660 : /*
661 : * For growfs, the incoming extent must be beyond the left record we
662 : * just found as it is new space and won't be used by anyone. This is
663 : * just a corruption check as we don't actually do anything with this
664 : * extent. Note that we need to use >= instead of > because it might
665 : * be the case that the "left" extent goes all the way to EOFS.
666 : */
667 43555737 : if (owner == XFS_RMAP_OWN_NULL) {
668 28 : if (XFS_IS_CORRUPT(mp,
669 : bno <
670 : ltrec.rm_startblock + ltrec.rm_blockcount)) {
671 0 : xfs_btree_mark_sick(cur);
672 0 : error = -EFSCORRUPTED;
673 0 : goto out_error;
674 : }
675 28 : goto out_done;
676 : }
677 :
678 : /*
679 : * If we're doing an unknown-owner removal for EFI recovery, we expect
680 : * to find the full range in the rmapbt or nothing at all. If we
681 : * don't find any rmaps overlapping either end of the range, we're
682 : * done. Hopefully this means that the EFI creator already queued
683 : * (and finished) a RUI to remove the rmap.
684 : */
685 43555709 : if (owner == XFS_RMAP_OWN_UNKNOWN &&
686 8511 : ltrec.rm_startblock + ltrec.rm_blockcount <= bno) {
687 8083 : struct xfs_rmap_irec rtrec;
688 :
689 8083 : error = xfs_btree_increment(cur, 0, &i);
690 8083 : if (error)
691 0 : goto out_error;
692 8083 : if (i == 0)
693 8081 : goto out_done;
694 8070 : error = xfs_rmap_get_rec(cur, &rtrec, &i);
695 8068 : if (error)
696 0 : goto out_error;
697 8068 : if (XFS_IS_CORRUPT(mp, i != 1)) {
698 0 : xfs_btree_mark_sick(cur);
699 0 : error = -EFSCORRUPTED;
700 0 : goto out_error;
701 : }
702 8068 : if (rtrec.rm_startblock >= bno + len)
703 8068 : goto out_done;
704 : }
705 :
706 : /* Make sure the extent we found covers the entire freeing range. */
707 43547626 : if (XFS_IS_CORRUPT(mp,
708 : ltrec.rm_startblock > bno ||
709 : ltrec.rm_startblock + ltrec.rm_blockcount <
710 : bno + len)) {
711 0 : xfs_btree_mark_sick(cur);
712 0 : error = -EFSCORRUPTED;
713 0 : goto out_error;
714 : }
715 :
716 : /* Check owner information. */
717 43547626 : error = xfs_rmap_free_check_owner(cur, ltoff, <rec, len, owner,
718 : offset, flags);
719 43547551 : if (error)
720 0 : goto out_error;
721 :
722 43547551 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
723 : /* exact match, simply remove the record from rmap tree */
724 38223638 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
725 : ltrec.rm_startblock, ltrec.rm_blockcount,
726 : ltrec.rm_owner, ltrec.rm_offset,
727 : ltrec.rm_flags);
728 38223597 : error = xfs_btree_delete(cur, &i);
729 38223682 : if (error)
730 0 : goto out_error;
731 38223682 : if (XFS_IS_CORRUPT(mp, i != 1)) {
732 0 : xfs_btree_mark_sick(cur);
733 0 : error = -EFSCORRUPTED;
734 0 : goto out_error;
735 : }
736 5323913 : } else if (ltrec.rm_startblock == bno) {
737 : /*
738 : * overlap left hand side of extent: move the start, trim the
739 : * length and update the current record.
740 : *
741 : * ltbno ltlen
742 : * Orig: |oooooooooooooooooooo|
743 : * Freeing: |fffffffff|
744 : * Result: |rrrrrrrrrr|
745 : * bno len
746 : */
747 957078 : ltrec.rm_startblock += len;
748 957078 : ltrec.rm_blockcount -= len;
749 957078 : if (!ignore_off)
750 129033 : ltrec.rm_offset += len;
751 957078 : error = xfs_rmap_update(cur, <rec);
752 957068 : if (error)
753 0 : goto out_error;
754 4366835 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
755 : /*
756 : * overlap right hand side of extent: trim the length and update
757 : * the current record.
758 : *
759 : * ltbno ltlen
760 : * Orig: |oooooooooooooooooooo|
761 : * Freeing: |fffffffff|
762 : * Result: |rrrrrrrrrr|
763 : * bno len
764 : */
765 442297 : ltrec.rm_blockcount -= len;
766 442297 : error = xfs_rmap_update(cur, <rec);
767 442294 : if (error)
768 0 : goto out_error;
769 : } else {
770 :
771 : /*
772 : * overlap middle of extent: trim the length of the existing
773 : * record to the length of the new left-extent size, increment
774 : * the insertion position so we can insert a new record
775 : * containing the remaining right-extent space.
776 : *
777 : * ltbno ltlen
778 : * Orig: |oooooooooooooooooooo|
779 : * Freeing: |fffffffff|
780 : * Result: |rrrrr| |rrrr|
781 : * bno len
782 : */
783 3924538 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
784 :
785 3924538 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
786 3924538 : error = xfs_rmap_update(cur, <rec);
787 3924539 : if (error)
788 0 : goto out_error;
789 :
790 3924539 : error = xfs_btree_increment(cur, 0, &i);
791 3924538 : if (error)
792 0 : goto out_error;
793 :
794 3924538 : cur->bc_rec.r.rm_startblock = bno + len;
795 3924538 : cur->bc_rec.r.rm_blockcount = orig_len - len -
796 3924538 : ltrec.rm_blockcount;
797 3924538 : cur->bc_rec.r.rm_owner = ltrec.rm_owner;
798 3924538 : if (ignore_off)
799 616323 : cur->bc_rec.r.rm_offset = 0;
800 : else
801 3308215 : cur->bc_rec.r.rm_offset = offset + len;
802 3924538 : cur->bc_rec.r.rm_flags = flags;
803 3924538 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno,
804 : cur->bc_rec.r.rm_startblock,
805 : cur->bc_rec.r.rm_blockcount,
806 : cur->bc_rec.r.rm_owner,
807 : cur->bc_rec.r.rm_offset,
808 : cur->bc_rec.r.rm_flags);
809 3924537 : error = xfs_btree_insert(cur, &i);
810 3924537 : if (error)
811 0 : goto out_error;
812 : }
813 :
814 3924537 : out_done:
815 43555690 : trace_xfs_rmap_unmap_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
816 : unwritten, oinfo);
817 43555557 : out_error:
818 43555557 : if (error)
819 72 : trace_xfs_rmap_unmap_error(mp, cur->bc_ag.pag->pag_agno,
820 72 : error, _RET_IP_);
821 43555557 : return error;
822 : }
823 :
824 : #ifdef CONFIG_XFS_LIVE_HOOKS
825 : /*
826 : * Use a static key here to reduce the overhead of rmapbt live updates. If
827 : * the compiler supports jump labels, the static branch will be replaced by a
828 : * nop sled when there are no hook users. Online fsck is currently the only
829 : * caller, so this is a reasonable tradeoff.
830 : *
831 : * Note: Patching the kernel code requires taking the cpu hotplug lock. Other
832 : * parts of the kernel allocate memory with that lock held, which means that
833 : * XFS callers cannot hold any locks that might be used by memory reclaim or
834 : * writeback when calling the static_branch_{inc,dec} functions.
835 : */
836 : DEFINE_STATIC_XFS_HOOK_SWITCH(xfs_rmap_hooks_switch);
837 :
838 : void
839 4493 : xfs_rmap_hook_disable(void)
840 : {
841 4493 : xfs_hooks_switch_off(&xfs_rmap_hooks_switch);
842 4497 : }
843 :
844 : void
845 4497 : xfs_rmap_hook_enable(void)
846 : {
847 4497 : xfs_hooks_switch_on(&xfs_rmap_hooks_switch);
848 4496 : }
849 :
850 : /* Call downstream hooks for a reverse mapping update. */
851 : static inline void
852 236962612 : xfs_rmap_update_hook(
853 : struct xfs_trans *tp,
854 : struct xfs_perag *pag,
855 : enum xfs_rmap_intent_type op,
856 : xfs_agblock_t startblock,
857 : xfs_extlen_t blockcount,
858 : bool unwritten,
859 : const struct xfs_owner_info *oinfo)
860 : {
861 237867131 : if (xfs_hooks_switched_on(&xfs_rmap_hooks_switch)) {
862 904512 : struct xfs_rmap_update_params p = {
863 : .startblock = startblock,
864 : .blockcount = blockcount,
865 : .unwritten = unwritten,
866 : .oinfo = *oinfo, /* struct copy */
867 : };
868 :
869 904512 : if (pag)
870 904512 : xfs_hooks_call(&pag->pag_rmap_update_hooks, op, &p);
871 : }
872 236964071 : }
873 :
874 : /* Call the specified function during a reverse mapping update. */
875 : int
876 4468 : xfs_rmap_hook_add(
877 : struct xfs_perag *pag,
878 : struct xfs_rmap_hook *hook)
879 : {
880 4468 : return xfs_hooks_add(&pag->pag_rmap_update_hooks, &hook->update_hook);
881 : }
882 :
883 : /* Stop calling the specified function during a reverse mapping update. */
884 : void
885 4470 : xfs_rmap_hook_del(
886 : struct xfs_perag *pag,
887 : struct xfs_rmap_hook *hook)
888 : {
889 4470 : xfs_hooks_del(&pag->pag_rmap_update_hooks, &hook->update_hook);
890 4470 : }
891 : #else
892 : # define xfs_rmap_update_hook(t, p, o, s, b, u, oi) do { } while(0)
893 : #endif /* CONFIG_XFS_LIVE_HOOKS */
894 :
895 : /*
896 : * Remove a reference to an extent in the rmap btree.
897 : */
898 : int
899 1048158 : xfs_rmap_free(
900 : struct xfs_trans *tp,
901 : struct xfs_buf *agbp,
902 : struct xfs_perag *pag,
903 : xfs_agblock_t bno,
904 : xfs_extlen_t len,
905 : const struct xfs_owner_info *oinfo)
906 : {
907 1048158 : struct xfs_mount *mp = tp->t_mountp;
908 1048158 : struct xfs_btree_cur *cur;
909 1048158 : int error;
910 :
911 1048158 : if (!xfs_has_rmapbt(mp))
912 : return 0;
913 :
914 929163 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
915 929152 : xfs_rmap_update_hook(tp, pag, XFS_RMAP_UNMAP, bno, len, false, oinfo);
916 929152 : error = xfs_rmap_unmap(cur, bno, len, false, oinfo);
917 :
918 929139 : xfs_btree_del_cursor(cur, error);
919 929139 : return error;
920 : }
921 :
922 : /*
923 : * A mergeable rmap must have the same owner and the same values for
924 : * the unwritten, attr_fork, and bmbt flags. The startblock and
925 : * offset are checked separately.
926 : */
927 : static bool
928 109678426 : xfs_rmap_is_mergeable(
929 : struct xfs_rmap_irec *irec,
930 : uint64_t owner,
931 : unsigned int flags)
932 : {
933 109678426 : if (irec->rm_owner == XFS_RMAP_OWN_NULL)
934 : return false;
935 109678426 : if (irec->rm_owner != owner)
936 : return false;
937 56242904 : if ((flags & XFS_RMAP_UNWRITTEN) ^
938 56242904 : (irec->rm_flags & XFS_RMAP_UNWRITTEN))
939 : return false;
940 54036045 : if ((flags & XFS_RMAP_ATTR_FORK) ^
941 : (irec->rm_flags & XFS_RMAP_ATTR_FORK))
942 : return false;
943 54031767 : if ((flags & XFS_RMAP_BMBT_BLOCK) ^
944 : (irec->rm_flags & XFS_RMAP_BMBT_BLOCK))
945 161377 : return false;
946 : return true;
947 : }
948 :
949 : /*
950 : * When we allocate a new block, the first thing we do is add a reference to
951 : * the extent in the rmap btree. This takes the form of a [agbno, length,
952 : * owner, offset] record. Flags are encoded in the high bits of the offset
953 : * field.
954 : */
955 : STATIC int
956 52067970 : xfs_rmap_map(
957 : struct xfs_btree_cur *cur,
958 : xfs_agblock_t bno,
959 : xfs_extlen_t len,
960 : bool unwritten,
961 : const struct xfs_owner_info *oinfo)
962 : {
963 52067970 : struct xfs_mount *mp = cur->bc_mp;
964 52067970 : struct xfs_rmap_irec ltrec;
965 52067970 : struct xfs_rmap_irec gtrec;
966 52067970 : int have_gt;
967 52067970 : int have_lt;
968 52067970 : int error = 0;
969 52067970 : int i;
970 52067970 : uint64_t owner;
971 52067970 : uint64_t offset;
972 52067970 : unsigned int flags = 0;
973 52067970 : bool ignore_off;
974 :
975 52067970 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
976 52067970 : ASSERT(owner != 0);
977 52067970 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
978 50670269 : (flags & XFS_RMAP_BMBT_BLOCK);
979 52067970 : if (unwritten)
980 12441111 : flags |= XFS_RMAP_UNWRITTEN;
981 52067970 : trace_xfs_rmap_map(mp, cur->bc_ag.pag->pag_agno, bno, len,
982 : unwritten, oinfo);
983 52068286 : ASSERT(!xfs_rmap_should_skip_owner_update(oinfo));
984 :
985 : /*
986 : * For the initial lookup, look for an exact match or the left-adjacent
987 : * record for our insertion point. This will also give us the record for
988 : * start block contiguity tests.
989 : */
990 52068286 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec,
991 : &have_lt);
992 52068296 : if (error)
993 405 : goto out_error;
994 52067891 : if (have_lt) {
995 52058308 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
996 52058308 : cur->bc_ag.pag->pag_agno, ltrec.rm_startblock,
997 : ltrec.rm_blockcount, ltrec.rm_owner,
998 : ltrec.rm_offset, ltrec.rm_flags);
999 :
1000 52058476 : if (!xfs_rmap_is_mergeable(<rec, owner, flags))
1001 28539349 : have_lt = 0;
1002 : }
1003 :
1004 52068059 : if (XFS_IS_CORRUPT(mp,
1005 : have_lt != 0 &&
1006 : ltrec.rm_startblock + ltrec.rm_blockcount > bno)) {
1007 0 : xfs_btree_mark_sick(cur);
1008 0 : error = -EFSCORRUPTED;
1009 0 : goto out_error;
1010 : }
1011 :
1012 : /*
1013 : * Increment the cursor to see if we have a right-adjacent record to our
1014 : * insertion point. This will give us the record for end block
1015 : * contiguity tests.
1016 : */
1017 52068059 : error = xfs_btree_increment(cur, 0, &have_gt);
1018 52067034 : if (error)
1019 2 : goto out_error;
1020 52067032 : if (have_gt) {
1021 46610730 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
1022 46611685 : if (error)
1023 0 : goto out_error;
1024 46611685 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
1025 0 : xfs_btree_mark_sick(cur);
1026 0 : error = -EFSCORRUPTED;
1027 0 : goto out_error;
1028 : }
1029 46611685 : if (XFS_IS_CORRUPT(mp, bno + len > gtrec.rm_startblock)) {
1030 0 : xfs_btree_mark_sick(cur);
1031 0 : error = -EFSCORRUPTED;
1032 0 : goto out_error;
1033 : }
1034 46611685 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1035 46611685 : cur->bc_ag.pag->pag_agno, gtrec.rm_startblock,
1036 : gtrec.rm_blockcount, gtrec.rm_owner,
1037 : gtrec.rm_offset, gtrec.rm_flags);
1038 46611585 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
1039 26779906 : have_gt = 0;
1040 : }
1041 :
1042 : /*
1043 : * Note: cursor currently points one record to the right of ltrec, even
1044 : * if there is no record in the tree to the right.
1045 : */
1046 52067887 : if (have_lt &&
1047 23519381 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
1048 4635736 : (ignore_off || ltrec.rm_offset + ltrec.rm_blockcount == offset)) {
1049 : /*
1050 : * left edge contiguous, merge into left record.
1051 : *
1052 : * ltbno ltlen
1053 : * orig: |ooooooooo|
1054 : * adding: |aaaaaaaaa|
1055 : * result: |rrrrrrrrrrrrrrrrrrr|
1056 : * bno len
1057 : */
1058 3767167 : ltrec.rm_blockcount += len;
1059 3767167 : if (have_gt &&
1060 1331402 : bno + len == gtrec.rm_startblock &&
1061 6000 : (ignore_off || offset + len == gtrec.rm_offset) &&
1062 37531 : (unsigned long)ltrec.rm_blockcount + len +
1063 37531 : gtrec.rm_blockcount <= XFS_RMAP_LEN_MAX) {
1064 : /*
1065 : * right edge also contiguous, delete right record
1066 : * and merge into left record.
1067 : *
1068 : * ltbno ltlen gtbno gtlen
1069 : * orig: |ooooooooo| |ooooooooo|
1070 : * adding: |aaaaaaaaa|
1071 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
1072 : */
1073 37531 : ltrec.rm_blockcount += gtrec.rm_blockcount;
1074 37531 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1075 : gtrec.rm_startblock,
1076 : gtrec.rm_blockcount,
1077 : gtrec.rm_owner,
1078 : gtrec.rm_offset,
1079 : gtrec.rm_flags);
1080 37531 : error = xfs_btree_delete(cur, &i);
1081 37531 : if (error)
1082 0 : goto out_error;
1083 37531 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1084 0 : xfs_btree_mark_sick(cur);
1085 0 : error = -EFSCORRUPTED;
1086 0 : goto out_error;
1087 : }
1088 : }
1089 :
1090 : /* point the cursor back to the left record and update */
1091 3767167 : error = xfs_btree_decrement(cur, 0, &have_gt);
1092 3767161 : if (error)
1093 0 : goto out_error;
1094 3767161 : error = xfs_rmap_update(cur, <rec);
1095 3767170 : if (error)
1096 0 : goto out_error;
1097 48300720 : } else if (have_gt &&
1098 18500447 : bno + len == gtrec.rm_startblock &&
1099 675023 : (ignore_off || offset + len == gtrec.rm_offset)) {
1100 : /*
1101 : * right edge contiguous, merge into right record.
1102 : *
1103 : * gtbno gtlen
1104 : * Orig: |ooooooooo|
1105 : * adding: |aaaaaaaaa|
1106 : * Result: |rrrrrrrrrrrrrrrrrrr|
1107 : * bno len
1108 : */
1109 89358 : gtrec.rm_startblock = bno;
1110 89358 : gtrec.rm_blockcount += len;
1111 89358 : if (!ignore_off)
1112 4469 : gtrec.rm_offset = offset;
1113 89358 : error = xfs_rmap_update(cur, >rec);
1114 89359 : if (error)
1115 0 : goto out_error;
1116 : } else {
1117 : /*
1118 : * no contiguous edge with identical owner, insert
1119 : * new record at current cursor position.
1120 : */
1121 48211362 : cur->bc_rec.r.rm_startblock = bno;
1122 48211362 : cur->bc_rec.r.rm_blockcount = len;
1123 48211362 : cur->bc_rec.r.rm_owner = owner;
1124 48211362 : cur->bc_rec.r.rm_offset = offset;
1125 48211362 : cur->bc_rec.r.rm_flags = flags;
1126 48211362 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1127 : owner, offset, flags);
1128 48211019 : error = xfs_btree_insert(cur, &i);
1129 48211423 : if (error)
1130 28 : goto out_error;
1131 48211395 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1132 0 : xfs_btree_mark_sick(cur);
1133 0 : error = -EFSCORRUPTED;
1134 0 : goto out_error;
1135 : }
1136 : }
1137 :
1138 52067924 : trace_xfs_rmap_map_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1139 : unwritten, oinfo);
1140 52068254 : out_error:
1141 52068254 : if (error)
1142 435 : trace_xfs_rmap_map_error(mp, cur->bc_ag.pag->pag_agno,
1143 435 : error, _RET_IP_);
1144 52068254 : return error;
1145 : }
1146 :
1147 : /*
1148 : * Add a reference to an extent in the rmap btree.
1149 : */
1150 : int
1151 2152599 : xfs_rmap_alloc(
1152 : struct xfs_trans *tp,
1153 : struct xfs_buf *agbp,
1154 : struct xfs_perag *pag,
1155 : xfs_agblock_t bno,
1156 : xfs_extlen_t len,
1157 : const struct xfs_owner_info *oinfo)
1158 : {
1159 2152599 : struct xfs_mount *mp = tp->t_mountp;
1160 2152599 : struct xfs_btree_cur *cur;
1161 2152599 : int error;
1162 :
1163 2152599 : if (!xfs_has_rmapbt(mp))
1164 : return 0;
1165 :
1166 1745813 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
1167 1745790 : xfs_rmap_update_hook(tp, pag, XFS_RMAP_MAP, bno, len, false, oinfo);
1168 1745790 : error = xfs_rmap_map(cur, bno, len, false, oinfo);
1169 :
1170 1745812 : xfs_btree_del_cursor(cur, error);
1171 1745812 : return error;
1172 : }
1173 :
1174 : #define RMAP_LEFT_CONTIG (1 << 0)
1175 : #define RMAP_RIGHT_CONTIG (1 << 1)
1176 : #define RMAP_LEFT_FILLING (1 << 2)
1177 : #define RMAP_RIGHT_FILLING (1 << 3)
1178 : #define RMAP_LEFT_VALID (1 << 6)
1179 : #define RMAP_RIGHT_VALID (1 << 7)
1180 :
1181 : #define LEFT r[0]
1182 : #define RIGHT r[1]
1183 : #define PREV r[2]
1184 : #define NEW r[3]
1185 :
1186 : /*
1187 : * Convert an unwritten extent to a real extent or vice versa.
1188 : * Does not handle overlapping extents.
1189 : */
1190 : STATIC int
1191 5982549 : xfs_rmap_convert(
1192 : struct xfs_btree_cur *cur,
1193 : xfs_agblock_t bno,
1194 : xfs_extlen_t len,
1195 : bool unwritten,
1196 : const struct xfs_owner_info *oinfo)
1197 : {
1198 5982549 : struct xfs_mount *mp = cur->bc_mp;
1199 5982549 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1200 : /* left is 0, right is 1, */
1201 : /* prev is 2, new is 3 */
1202 5982549 : uint64_t owner;
1203 5982549 : uint64_t offset;
1204 5982549 : uint64_t new_endoff;
1205 5982549 : unsigned int oldext;
1206 5982549 : unsigned int newext;
1207 5982549 : unsigned int flags = 0;
1208 5982549 : int i;
1209 5982549 : int state = 0;
1210 5982549 : int error;
1211 :
1212 5982549 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1213 5982549 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1214 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1215 5982549 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1216 5982549 : new_endoff = offset + len;
1217 5982549 : trace_xfs_rmap_convert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1218 : unwritten, oinfo);
1219 :
1220 : /*
1221 : * For the initial lookup, look for an exact match or the left-adjacent
1222 : * record for our insertion point. This will also give us the record for
1223 : * start block contiguity tests.
1224 : */
1225 5982545 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, &PREV, &i);
1226 5982512 : if (error)
1227 0 : goto done;
1228 5982512 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1229 0 : xfs_btree_mark_sick(cur);
1230 0 : error = -EFSCORRUPTED;
1231 0 : goto done;
1232 : }
1233 :
1234 5982512 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
1235 5982512 : cur->bc_ag.pag->pag_agno, PREV.rm_startblock,
1236 : PREV.rm_blockcount, PREV.rm_owner,
1237 : PREV.rm_offset, PREV.rm_flags);
1238 :
1239 5982506 : ASSERT(PREV.rm_offset <= offset);
1240 5982506 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1241 5982506 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1242 5982506 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1243 :
1244 : /*
1245 : * Set flags determining what part of the previous oldext allocation
1246 : * extent is being replaced by a newext allocation.
1247 : */
1248 5982506 : if (PREV.rm_offset == offset)
1249 5771597 : state |= RMAP_LEFT_FILLING;
1250 5982506 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1251 5548841 : state |= RMAP_RIGHT_FILLING;
1252 :
1253 : /*
1254 : * Decrement the cursor to see if we have a left-adjacent record to our
1255 : * insertion point. This will give us the record for end block
1256 : * contiguity tests.
1257 : */
1258 5982506 : error = xfs_btree_decrement(cur, 0, &i);
1259 5982536 : if (error)
1260 0 : goto done;
1261 5982536 : if (i) {
1262 5982536 : state |= RMAP_LEFT_VALID;
1263 5982536 : error = xfs_rmap_get_rec(cur, &LEFT, &i);
1264 5982548 : if (error)
1265 0 : goto done;
1266 5982548 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1267 0 : xfs_btree_mark_sick(cur);
1268 0 : error = -EFSCORRUPTED;
1269 0 : goto done;
1270 : }
1271 5982548 : if (XFS_IS_CORRUPT(mp,
1272 : LEFT.rm_startblock + LEFT.rm_blockcount >
1273 : bno)) {
1274 0 : xfs_btree_mark_sick(cur);
1275 0 : error = -EFSCORRUPTED;
1276 0 : goto done;
1277 : }
1278 5982548 : trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
1279 5982548 : cur->bc_ag.pag->pag_agno, LEFT.rm_startblock,
1280 : LEFT.rm_blockcount, LEFT.rm_owner,
1281 : LEFT.rm_offset, LEFT.rm_flags);
1282 5982545 : if (LEFT.rm_startblock + LEFT.rm_blockcount == bno &&
1283 5567412 : LEFT.rm_offset + LEFT.rm_blockcount == offset &&
1284 377525 : xfs_rmap_is_mergeable(&LEFT, owner, newext))
1285 373564 : state |= RMAP_LEFT_CONTIG;
1286 : }
1287 :
1288 : /*
1289 : * Increment the cursor to see if we have a right-adjacent record to our
1290 : * insertion point. This will give us the record for end block
1291 : * contiguity tests.
1292 : */
1293 5982545 : error = xfs_btree_increment(cur, 0, &i);
1294 5982545 : if (error)
1295 0 : goto done;
1296 5982545 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1297 0 : xfs_btree_mark_sick(cur);
1298 0 : error = -EFSCORRUPTED;
1299 0 : goto done;
1300 : }
1301 5982545 : error = xfs_btree_increment(cur, 0, &i);
1302 5982552 : if (error)
1303 0 : goto done;
1304 5982552 : if (i) {
1305 5542480 : state |= RMAP_RIGHT_VALID;
1306 5542480 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1307 5542522 : if (error)
1308 0 : goto done;
1309 5542522 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1310 0 : xfs_btree_mark_sick(cur);
1311 0 : error = -EFSCORRUPTED;
1312 0 : goto done;
1313 : }
1314 5542522 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1315 0 : xfs_btree_mark_sick(cur);
1316 0 : error = -EFSCORRUPTED;
1317 0 : goto done;
1318 : }
1319 5542522 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1320 5542522 : cur->bc_ag.pag->pag_agno, RIGHT.rm_startblock,
1321 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1322 : RIGHT.rm_offset, RIGHT.rm_flags);
1323 5542517 : if (bno + len == RIGHT.rm_startblock &&
1324 3761454 : offset + len == RIGHT.rm_offset &&
1325 73134 : xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1326 70560 : state |= RMAP_RIGHT_CONTIG;
1327 : }
1328 :
1329 : /* check that left + prev + right is not too long */
1330 5982589 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1331 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1332 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1333 24640 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1334 24640 : (unsigned long)LEFT.rm_blockcount + len +
1335 24640 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1336 0 : state &= ~RMAP_RIGHT_CONTIG;
1337 :
1338 5982542 : trace_xfs_rmap_convert_state(mp, cur->bc_ag.pag->pag_agno, state,
1339 5982589 : _RET_IP_);
1340 :
1341 : /* reset the cursor back to PREV */
1342 5982547 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, NULL, &i);
1343 5982547 : if (error)
1344 0 : goto done;
1345 5982547 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1346 0 : xfs_btree_mark_sick(cur);
1347 0 : error = -EFSCORRUPTED;
1348 0 : goto done;
1349 : }
1350 :
1351 : /*
1352 : * Switch out based on the FILLING and CONTIG state bits.
1353 : */
1354 5982547 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1355 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1356 24640 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1357 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1358 : /*
1359 : * Setting all of a previous oldext extent to newext.
1360 : * The left and right neighbors are both contiguous with new.
1361 : */
1362 24640 : error = xfs_btree_increment(cur, 0, &i);
1363 24640 : if (error)
1364 0 : goto done;
1365 24640 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1366 0 : xfs_btree_mark_sick(cur);
1367 0 : error = -EFSCORRUPTED;
1368 0 : goto done;
1369 : }
1370 24640 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1371 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1372 : RIGHT.rm_owner, RIGHT.rm_offset,
1373 : RIGHT.rm_flags);
1374 24640 : error = xfs_btree_delete(cur, &i);
1375 24640 : if (error)
1376 0 : goto done;
1377 24640 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1378 0 : xfs_btree_mark_sick(cur);
1379 0 : error = -EFSCORRUPTED;
1380 0 : goto done;
1381 : }
1382 24640 : error = xfs_btree_decrement(cur, 0, &i);
1383 24640 : if (error)
1384 0 : goto done;
1385 24640 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1386 0 : xfs_btree_mark_sick(cur);
1387 0 : error = -EFSCORRUPTED;
1388 0 : goto done;
1389 : }
1390 24640 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1391 : PREV.rm_startblock, PREV.rm_blockcount,
1392 : PREV.rm_owner, PREV.rm_offset,
1393 : PREV.rm_flags);
1394 24640 : error = xfs_btree_delete(cur, &i);
1395 24640 : if (error)
1396 0 : goto done;
1397 24640 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1398 0 : xfs_btree_mark_sick(cur);
1399 0 : error = -EFSCORRUPTED;
1400 0 : goto done;
1401 : }
1402 24640 : error = xfs_btree_decrement(cur, 0, &i);
1403 24640 : if (error)
1404 0 : goto done;
1405 24640 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1406 0 : xfs_btree_mark_sick(cur);
1407 0 : error = -EFSCORRUPTED;
1408 0 : goto done;
1409 : }
1410 24640 : NEW = LEFT;
1411 24640 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1412 24640 : error = xfs_rmap_update(cur, &NEW);
1413 24640 : if (error)
1414 0 : goto done;
1415 : break;
1416 :
1417 228081 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1418 : /*
1419 : * Setting all of a previous oldext extent to newext.
1420 : * The left neighbor is contiguous, the right is not.
1421 : */
1422 228081 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1423 : PREV.rm_startblock, PREV.rm_blockcount,
1424 : PREV.rm_owner, PREV.rm_offset,
1425 : PREV.rm_flags);
1426 228081 : error = xfs_btree_delete(cur, &i);
1427 228081 : if (error)
1428 0 : goto done;
1429 228081 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1430 0 : xfs_btree_mark_sick(cur);
1431 0 : error = -EFSCORRUPTED;
1432 0 : goto done;
1433 : }
1434 228081 : error = xfs_btree_decrement(cur, 0, &i);
1435 228081 : if (error)
1436 0 : goto done;
1437 228081 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1438 0 : xfs_btree_mark_sick(cur);
1439 0 : error = -EFSCORRUPTED;
1440 0 : goto done;
1441 : }
1442 228081 : NEW = LEFT;
1443 228081 : NEW.rm_blockcount += PREV.rm_blockcount;
1444 228081 : error = xfs_rmap_update(cur, &NEW);
1445 228081 : if (error)
1446 0 : goto done;
1447 : break;
1448 :
1449 22137 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1450 : /*
1451 : * Setting all of a previous oldext extent to newext.
1452 : * The right neighbor is contiguous, the left is not.
1453 : */
1454 22137 : error = xfs_btree_increment(cur, 0, &i);
1455 22137 : if (error)
1456 0 : goto done;
1457 22137 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1458 0 : xfs_btree_mark_sick(cur);
1459 0 : error = -EFSCORRUPTED;
1460 0 : goto done;
1461 : }
1462 22137 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1463 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1464 : RIGHT.rm_owner, RIGHT.rm_offset,
1465 : RIGHT.rm_flags);
1466 22137 : error = xfs_btree_delete(cur, &i);
1467 22137 : if (error)
1468 0 : goto done;
1469 22137 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1470 0 : xfs_btree_mark_sick(cur);
1471 0 : error = -EFSCORRUPTED;
1472 0 : goto done;
1473 : }
1474 22137 : error = xfs_btree_decrement(cur, 0, &i);
1475 22137 : if (error)
1476 0 : goto done;
1477 22137 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1478 0 : xfs_btree_mark_sick(cur);
1479 0 : error = -EFSCORRUPTED;
1480 0 : goto done;
1481 : }
1482 22137 : NEW = PREV;
1483 22137 : NEW.rm_blockcount = len + RIGHT.rm_blockcount;
1484 22137 : NEW.rm_flags = newext;
1485 22137 : error = xfs_rmap_update(cur, &NEW);
1486 22137 : if (error)
1487 0 : goto done;
1488 : break;
1489 :
1490 5197218 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1491 : /*
1492 : * Setting all of a previous oldext extent to newext.
1493 : * Neither the left nor right neighbors are contiguous with
1494 : * the new one.
1495 : */
1496 5197218 : NEW = PREV;
1497 5197218 : NEW.rm_flags = newext;
1498 5197218 : error = xfs_rmap_update(cur, &NEW);
1499 5197202 : if (error)
1500 0 : goto done;
1501 : break;
1502 :
1503 120843 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1504 : /*
1505 : * Setting the first part of a previous oldext extent to newext.
1506 : * The left neighbor is contiguous.
1507 : */
1508 120843 : NEW = PREV;
1509 120843 : NEW.rm_offset += len;
1510 120843 : NEW.rm_startblock += len;
1511 120843 : NEW.rm_blockcount -= len;
1512 120843 : error = xfs_rmap_update(cur, &NEW);
1513 120843 : if (error)
1514 0 : goto done;
1515 120843 : error = xfs_btree_decrement(cur, 0, &i);
1516 120843 : if (error)
1517 0 : goto done;
1518 120843 : NEW = LEFT;
1519 120843 : NEW.rm_blockcount += len;
1520 120843 : error = xfs_rmap_update(cur, &NEW);
1521 120843 : if (error)
1522 0 : goto done;
1523 : break;
1524 :
1525 178728 : case RMAP_LEFT_FILLING:
1526 : /*
1527 : * Setting the first part of a previous oldext extent to newext.
1528 : * The left neighbor is not contiguous.
1529 : */
1530 178728 : NEW = PREV;
1531 178728 : NEW.rm_startblock += len;
1532 178728 : NEW.rm_offset += len;
1533 178728 : NEW.rm_blockcount -= len;
1534 178728 : error = xfs_rmap_update(cur, &NEW);
1535 178728 : if (error)
1536 0 : goto done;
1537 178728 : NEW.rm_startblock = bno;
1538 178728 : NEW.rm_owner = owner;
1539 178728 : NEW.rm_offset = offset;
1540 178728 : NEW.rm_blockcount = len;
1541 178728 : NEW.rm_flags = newext;
1542 178728 : cur->bc_rec.r = NEW;
1543 178728 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno,
1544 : len, owner, offset, newext);
1545 178728 : error = xfs_btree_insert(cur, &i);
1546 178727 : if (error)
1547 0 : goto done;
1548 178727 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1549 0 : xfs_btree_mark_sick(cur);
1550 0 : error = -EFSCORRUPTED;
1551 0 : goto done;
1552 : }
1553 : break;
1554 :
1555 23783 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1556 : /*
1557 : * Setting the last part of a previous oldext extent to newext.
1558 : * The right neighbor is contiguous with the new allocation.
1559 : */
1560 23783 : NEW = PREV;
1561 23783 : NEW.rm_blockcount -= len;
1562 23783 : error = xfs_rmap_update(cur, &NEW);
1563 23783 : if (error)
1564 0 : goto done;
1565 23783 : error = xfs_btree_increment(cur, 0, &i);
1566 23783 : if (error)
1567 0 : goto done;
1568 23783 : NEW = RIGHT;
1569 23783 : NEW.rm_offset = offset;
1570 23783 : NEW.rm_startblock = bno;
1571 23783 : NEW.rm_blockcount += len;
1572 23783 : error = xfs_rmap_update(cur, &NEW);
1573 23783 : if (error)
1574 0 : goto done;
1575 : break;
1576 :
1577 53019 : case RMAP_RIGHT_FILLING:
1578 : /*
1579 : * Setting the last part of a previous oldext extent to newext.
1580 : * The right neighbor is not contiguous.
1581 : */
1582 53019 : NEW = PREV;
1583 53019 : NEW.rm_blockcount -= len;
1584 53019 : error = xfs_rmap_update(cur, &NEW);
1585 53019 : if (error)
1586 0 : goto done;
1587 53019 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1588 : oldext, &i);
1589 53019 : if (error)
1590 0 : goto done;
1591 53019 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1592 0 : xfs_btree_mark_sick(cur);
1593 0 : error = -EFSCORRUPTED;
1594 0 : goto done;
1595 : }
1596 53019 : NEW.rm_startblock = bno;
1597 53019 : NEW.rm_owner = owner;
1598 53019 : NEW.rm_offset = offset;
1599 53019 : NEW.rm_blockcount = len;
1600 53019 : NEW.rm_flags = newext;
1601 53019 : cur->bc_rec.r = NEW;
1602 53019 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno,
1603 : len, owner, offset, newext);
1604 53019 : error = xfs_btree_insert(cur, &i);
1605 53019 : if (error)
1606 0 : goto done;
1607 53019 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1608 0 : xfs_btree_mark_sick(cur);
1609 0 : error = -EFSCORRUPTED;
1610 0 : goto done;
1611 : }
1612 : break;
1613 :
1614 134098 : case 0:
1615 : /*
1616 : * Setting the middle part of a previous oldext extent to
1617 : * newext. Contiguity is impossible here.
1618 : * One extent becomes three extents.
1619 : */
1620 : /* new right extent - oldext */
1621 134098 : NEW.rm_startblock = bno + len;
1622 134098 : NEW.rm_owner = owner;
1623 134098 : NEW.rm_offset = new_endoff;
1624 134098 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1625 : new_endoff;
1626 134098 : NEW.rm_flags = PREV.rm_flags;
1627 134098 : error = xfs_rmap_update(cur, &NEW);
1628 134098 : if (error)
1629 0 : goto done;
1630 : /* new left extent - oldext */
1631 134098 : NEW = PREV;
1632 134098 : NEW.rm_blockcount = offset - PREV.rm_offset;
1633 134098 : cur->bc_rec.r = NEW;
1634 134098 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno,
1635 : NEW.rm_startblock, NEW.rm_blockcount,
1636 : NEW.rm_owner, NEW.rm_offset,
1637 : NEW.rm_flags);
1638 134098 : error = xfs_btree_insert(cur, &i);
1639 134098 : if (error)
1640 0 : goto done;
1641 134098 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1642 0 : xfs_btree_mark_sick(cur);
1643 0 : error = -EFSCORRUPTED;
1644 0 : goto done;
1645 : }
1646 : /*
1647 : * Reset the cursor to the position of the new extent
1648 : * we are about to insert as we can't trust it after
1649 : * the previous insert.
1650 : */
1651 134098 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1652 : oldext, &i);
1653 134098 : if (error)
1654 0 : goto done;
1655 134098 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1656 0 : xfs_btree_mark_sick(cur);
1657 0 : error = -EFSCORRUPTED;
1658 0 : goto done;
1659 : }
1660 : /* new middle extent - newext */
1661 134098 : cur->bc_rec.r.rm_flags &= ~XFS_RMAP_UNWRITTEN;
1662 134098 : cur->bc_rec.r.rm_flags |= newext;
1663 134098 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1664 : owner, offset, newext);
1665 134098 : error = xfs_btree_insert(cur, &i);
1666 134098 : if (error)
1667 0 : goto done;
1668 134098 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1669 0 : xfs_btree_mark_sick(cur);
1670 0 : error = -EFSCORRUPTED;
1671 0 : goto done;
1672 : }
1673 : break;
1674 :
1675 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1676 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1677 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1678 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1679 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1680 : case RMAP_LEFT_CONTIG:
1681 : case RMAP_RIGHT_CONTIG:
1682 : /*
1683 : * These cases are all impossible.
1684 : */
1685 0 : ASSERT(0);
1686 : }
1687 :
1688 5982530 : trace_xfs_rmap_convert_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1689 : unwritten, oinfo);
1690 5982521 : done:
1691 5982521 : if (error)
1692 0 : trace_xfs_rmap_convert_error(cur->bc_mp,
1693 0 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
1694 5982521 : return error;
1695 : }
1696 :
1697 : /*
1698 : * Convert an unwritten extent to a real extent or vice versa. If there is no
1699 : * possibility of overlapping extents, delegate to the simpler convert
1700 : * function.
1701 : */
1702 : STATIC int
1703 12368541 : xfs_rmap_convert_shared(
1704 : struct xfs_btree_cur *cur,
1705 : xfs_agblock_t bno,
1706 : xfs_extlen_t len,
1707 : bool unwritten,
1708 : const struct xfs_owner_info *oinfo)
1709 : {
1710 12368541 : struct xfs_mount *mp = cur->bc_mp;
1711 12368541 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1712 : /* left is 0, right is 1, */
1713 : /* prev is 2, new is 3 */
1714 12368541 : uint64_t owner;
1715 12368541 : uint64_t offset;
1716 12368541 : uint64_t new_endoff;
1717 12368541 : unsigned int oldext;
1718 12368541 : unsigned int newext;
1719 12368541 : unsigned int flags = 0;
1720 12368541 : int i;
1721 12368541 : int state = 0;
1722 12368541 : int error;
1723 :
1724 12368541 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1725 12368541 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1726 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1727 12368541 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1728 12368541 : new_endoff = offset + len;
1729 12368541 : trace_xfs_rmap_convert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1730 : unwritten, oinfo);
1731 :
1732 : /*
1733 : * For the initial lookup, look for and exact match or the left-adjacent
1734 : * record for our insertion point. This will also give us the record for
1735 : * start block contiguity tests.
1736 : */
1737 12368541 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, oldext,
1738 : &PREV, &i);
1739 12368534 : if (error)
1740 2 : goto done;
1741 12368532 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1742 0 : xfs_btree_mark_sick(cur);
1743 0 : error = -EFSCORRUPTED;
1744 0 : goto done;
1745 : }
1746 :
1747 12368532 : ASSERT(PREV.rm_offset <= offset);
1748 12368532 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1749 12368532 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1750 12368532 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1751 :
1752 : /*
1753 : * Set flags determining what part of the previous oldext allocation
1754 : * extent is being replaced by a newext allocation.
1755 : */
1756 12368532 : if (PREV.rm_offset == offset)
1757 12014317 : state |= RMAP_LEFT_FILLING;
1758 12368532 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1759 10034426 : state |= RMAP_RIGHT_FILLING;
1760 :
1761 : /* Is there a left record that abuts our range? */
1762 12368532 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, newext,
1763 : &LEFT, &i);
1764 12368540 : if (error)
1765 0 : goto done;
1766 12368540 : if (i) {
1767 249249 : state |= RMAP_LEFT_VALID;
1768 249249 : if (XFS_IS_CORRUPT(mp,
1769 : LEFT.rm_startblock + LEFT.rm_blockcount >
1770 : bno)) {
1771 0 : xfs_btree_mark_sick(cur);
1772 0 : error = -EFSCORRUPTED;
1773 0 : goto done;
1774 : }
1775 249249 : if (xfs_rmap_is_mergeable(&LEFT, owner, newext))
1776 249248 : state |= RMAP_LEFT_CONTIG;
1777 : }
1778 :
1779 : /* Is there a right record that abuts our range? */
1780 12368540 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
1781 : newext, &i);
1782 12368540 : if (error)
1783 0 : goto done;
1784 12368540 : if (i) {
1785 126831 : state |= RMAP_RIGHT_VALID;
1786 126831 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1787 126831 : if (error)
1788 0 : goto done;
1789 126831 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1790 0 : xfs_btree_mark_sick(cur);
1791 0 : error = -EFSCORRUPTED;
1792 0 : goto done;
1793 : }
1794 126831 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1795 0 : xfs_btree_mark_sick(cur);
1796 0 : error = -EFSCORRUPTED;
1797 0 : goto done;
1798 : }
1799 126831 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1800 126831 : cur->bc_ag.pag->pag_agno, RIGHT.rm_startblock,
1801 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1802 : RIGHT.rm_offset, RIGHT.rm_flags);
1803 126831 : if (xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1804 126831 : state |= RMAP_RIGHT_CONTIG;
1805 : }
1806 :
1807 : /* check that left + prev + right is not too long */
1808 12368540 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1809 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1810 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1811 26057 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1812 26057 : (unsigned long)LEFT.rm_blockcount + len +
1813 26057 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1814 0 : state &= ~RMAP_RIGHT_CONTIG;
1815 :
1816 12368540 : trace_xfs_rmap_convert_state(mp, cur->bc_ag.pag->pag_agno, state,
1817 12368540 : _RET_IP_);
1818 : /*
1819 : * Switch out based on the FILLING and CONTIG state bits.
1820 : */
1821 12368540 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1822 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1823 26057 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1824 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1825 : /*
1826 : * Setting all of a previous oldext extent to newext.
1827 : * The left and right neighbors are both contiguous with new.
1828 : */
1829 26057 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1830 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1831 : RIGHT.rm_offset, RIGHT.rm_flags);
1832 26057 : if (error)
1833 0 : goto done;
1834 26057 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1835 : PREV.rm_blockcount, PREV.rm_owner,
1836 : PREV.rm_offset, PREV.rm_flags);
1837 26057 : if (error)
1838 0 : goto done;
1839 26057 : NEW = LEFT;
1840 26057 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1841 : NEW.rm_blockcount, NEW.rm_owner,
1842 : NEW.rm_offset, NEW.rm_flags, &i);
1843 26057 : if (error)
1844 0 : goto done;
1845 26057 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1846 0 : xfs_btree_mark_sick(cur);
1847 0 : error = -EFSCORRUPTED;
1848 0 : goto done;
1849 : }
1850 26057 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1851 26057 : error = xfs_rmap_update(cur, &NEW);
1852 26057 : if (error)
1853 0 : goto done;
1854 : break;
1855 :
1856 171369 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1857 : /*
1858 : * Setting all of a previous oldext extent to newext.
1859 : * The left neighbor is contiguous, the right is not.
1860 : */
1861 171369 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1862 : PREV.rm_blockcount, PREV.rm_owner,
1863 : PREV.rm_offset, PREV.rm_flags);
1864 171369 : if (error)
1865 0 : goto done;
1866 171369 : NEW = LEFT;
1867 171369 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1868 : NEW.rm_blockcount, NEW.rm_owner,
1869 : NEW.rm_offset, NEW.rm_flags, &i);
1870 171369 : if (error)
1871 0 : goto done;
1872 171369 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1873 0 : xfs_btree_mark_sick(cur);
1874 0 : error = -EFSCORRUPTED;
1875 0 : goto done;
1876 : }
1877 171369 : NEW.rm_blockcount += PREV.rm_blockcount;
1878 171369 : error = xfs_rmap_update(cur, &NEW);
1879 171369 : if (error)
1880 0 : goto done;
1881 : break;
1882 :
1883 89397 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1884 : /*
1885 : * Setting all of a previous oldext extent to newext.
1886 : * The right neighbor is contiguous, the left is not.
1887 : */
1888 89397 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1889 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1890 : RIGHT.rm_offset, RIGHT.rm_flags);
1891 89397 : if (error)
1892 0 : goto done;
1893 89397 : NEW = PREV;
1894 89397 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1895 : NEW.rm_blockcount, NEW.rm_owner,
1896 : NEW.rm_offset, NEW.rm_flags, &i);
1897 89397 : if (error)
1898 0 : goto done;
1899 89397 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1900 0 : xfs_btree_mark_sick(cur);
1901 0 : error = -EFSCORRUPTED;
1902 0 : goto done;
1903 : }
1904 89397 : NEW.rm_blockcount += RIGHT.rm_blockcount;
1905 89397 : NEW.rm_flags = RIGHT.rm_flags;
1906 89397 : error = xfs_rmap_update(cur, &NEW);
1907 89397 : if (error)
1908 0 : goto done;
1909 : break;
1910 :
1911 9561049 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1912 : /*
1913 : * Setting all of a previous oldext extent to newext.
1914 : * Neither the left nor right neighbors are contiguous with
1915 : * the new one.
1916 : */
1917 9561049 : NEW = PREV;
1918 9561049 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1919 : NEW.rm_blockcount, NEW.rm_owner,
1920 : NEW.rm_offset, NEW.rm_flags, &i);
1921 9561049 : if (error)
1922 0 : goto done;
1923 9561049 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1924 0 : xfs_btree_mark_sick(cur);
1925 0 : error = -EFSCORRUPTED;
1926 0 : goto done;
1927 : }
1928 9561049 : NEW.rm_flags = newext;
1929 9561049 : error = xfs_rmap_update(cur, &NEW);
1930 9561048 : if (error)
1931 0 : goto done;
1932 : break;
1933 :
1934 51822 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1935 : /*
1936 : * Setting the first part of a previous oldext extent to newext.
1937 : * The left neighbor is contiguous.
1938 : */
1939 51822 : NEW = PREV;
1940 51822 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1941 : NEW.rm_blockcount, NEW.rm_owner,
1942 : NEW.rm_offset, NEW.rm_flags);
1943 51822 : if (error)
1944 0 : goto done;
1945 51822 : NEW.rm_offset += len;
1946 51822 : NEW.rm_startblock += len;
1947 51822 : NEW.rm_blockcount -= len;
1948 51822 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1949 : NEW.rm_blockcount, NEW.rm_owner,
1950 : NEW.rm_offset, NEW.rm_flags);
1951 51822 : if (error)
1952 0 : goto done;
1953 51822 : NEW = LEFT;
1954 51822 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1955 : NEW.rm_blockcount, NEW.rm_owner,
1956 : NEW.rm_offset, NEW.rm_flags, &i);
1957 51822 : if (error)
1958 0 : goto done;
1959 51822 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1960 0 : xfs_btree_mark_sick(cur);
1961 0 : error = -EFSCORRUPTED;
1962 0 : goto done;
1963 : }
1964 51822 : NEW.rm_blockcount += len;
1965 51822 : error = xfs_rmap_update(cur, &NEW);
1966 51822 : if (error)
1967 0 : goto done;
1968 : break;
1969 :
1970 2114624 : case RMAP_LEFT_FILLING:
1971 : /*
1972 : * Setting the first part of a previous oldext extent to newext.
1973 : * The left neighbor is not contiguous.
1974 : */
1975 2114624 : NEW = PREV;
1976 2114624 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1977 : NEW.rm_blockcount, NEW.rm_owner,
1978 : NEW.rm_offset, NEW.rm_flags);
1979 2114624 : if (error)
1980 0 : goto done;
1981 2114624 : NEW.rm_offset += len;
1982 2114624 : NEW.rm_startblock += len;
1983 2114624 : NEW.rm_blockcount -= len;
1984 2114624 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1985 : NEW.rm_blockcount, NEW.rm_owner,
1986 : NEW.rm_offset, NEW.rm_flags);
1987 2114624 : if (error)
1988 0 : goto done;
1989 2114624 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1990 2114624 : if (error)
1991 0 : goto done;
1992 : break;
1993 :
1994 11377 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1995 : /*
1996 : * Setting the last part of a previous oldext extent to newext.
1997 : * The right neighbor is contiguous with the new allocation.
1998 : */
1999 11377 : NEW = PREV;
2000 11377 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2001 : NEW.rm_blockcount, NEW.rm_owner,
2002 : NEW.rm_offset, NEW.rm_flags, &i);
2003 11377 : if (error)
2004 0 : goto done;
2005 11377 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2006 0 : xfs_btree_mark_sick(cur);
2007 0 : error = -EFSCORRUPTED;
2008 0 : goto done;
2009 : }
2010 11377 : NEW.rm_blockcount = offset - NEW.rm_offset;
2011 11377 : error = xfs_rmap_update(cur, &NEW);
2012 11377 : if (error)
2013 0 : goto done;
2014 11377 : NEW = RIGHT;
2015 11377 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
2016 : NEW.rm_blockcount, NEW.rm_owner,
2017 : NEW.rm_offset, NEW.rm_flags);
2018 11377 : if (error)
2019 0 : goto done;
2020 11377 : NEW.rm_offset = offset;
2021 11377 : NEW.rm_startblock = bno;
2022 11377 : NEW.rm_blockcount += len;
2023 11377 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2024 : NEW.rm_blockcount, NEW.rm_owner,
2025 : NEW.rm_offset, NEW.rm_flags);
2026 11377 : if (error)
2027 0 : goto done;
2028 : break;
2029 :
2030 175177 : case RMAP_RIGHT_FILLING:
2031 : /*
2032 : * Setting the last part of a previous oldext extent to newext.
2033 : * The right neighbor is not contiguous.
2034 : */
2035 175177 : NEW = PREV;
2036 175177 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2037 : NEW.rm_blockcount, NEW.rm_owner,
2038 : NEW.rm_offset, NEW.rm_flags, &i);
2039 175177 : if (error)
2040 0 : goto done;
2041 175177 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2042 0 : xfs_btree_mark_sick(cur);
2043 0 : error = -EFSCORRUPTED;
2044 0 : goto done;
2045 : }
2046 175177 : NEW.rm_blockcount -= len;
2047 175177 : error = xfs_rmap_update(cur, &NEW);
2048 175177 : if (error)
2049 0 : goto done;
2050 175177 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
2051 175177 : if (error)
2052 0 : goto done;
2053 : break;
2054 :
2055 167668 : case 0:
2056 : /*
2057 : * Setting the middle part of a previous oldext extent to
2058 : * newext. Contiguity is impossible here.
2059 : * One extent becomes three extents.
2060 : */
2061 : /* new right extent - oldext */
2062 167668 : NEW.rm_startblock = bno + len;
2063 167668 : NEW.rm_owner = owner;
2064 167668 : NEW.rm_offset = new_endoff;
2065 167668 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
2066 : new_endoff;
2067 167668 : NEW.rm_flags = PREV.rm_flags;
2068 167668 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2069 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
2070 : NEW.rm_flags);
2071 167668 : if (error)
2072 0 : goto done;
2073 : /* new left extent - oldext */
2074 167668 : NEW = PREV;
2075 167668 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2076 : NEW.rm_blockcount, NEW.rm_owner,
2077 : NEW.rm_offset, NEW.rm_flags, &i);
2078 167668 : if (error)
2079 0 : goto done;
2080 167668 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2081 0 : xfs_btree_mark_sick(cur);
2082 0 : error = -EFSCORRUPTED;
2083 0 : goto done;
2084 : }
2085 167668 : NEW.rm_blockcount = offset - NEW.rm_offset;
2086 167668 : error = xfs_rmap_update(cur, &NEW);
2087 167668 : if (error)
2088 0 : goto done;
2089 : /* new middle extent - newext */
2090 167668 : NEW.rm_startblock = bno;
2091 167668 : NEW.rm_blockcount = len;
2092 167668 : NEW.rm_owner = owner;
2093 167668 : NEW.rm_offset = offset;
2094 167668 : NEW.rm_flags = newext;
2095 167668 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2096 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
2097 : NEW.rm_flags);
2098 167668 : if (error)
2099 0 : goto done;
2100 : break;
2101 :
2102 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2103 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2104 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
2105 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
2106 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2107 : case RMAP_LEFT_CONTIG:
2108 : case RMAP_RIGHT_CONTIG:
2109 : /*
2110 : * These cases are all impossible.
2111 : */
2112 0 : ASSERT(0);
2113 : }
2114 :
2115 12368539 : trace_xfs_rmap_convert_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2116 : unwritten, oinfo);
2117 12368541 : done:
2118 12368541 : if (error)
2119 2 : trace_xfs_rmap_convert_error(cur->bc_mp,
2120 2 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2121 12368541 : return error;
2122 : }
2123 :
2124 : #undef NEW
2125 : #undef LEFT
2126 : #undef RIGHT
2127 : #undef PREV
2128 :
2129 : /*
2130 : * Find an extent in the rmap btree and unmap it. For rmap extent types that
2131 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2132 : * that the prev/next records in the btree might belong to another owner.
2133 : * Therefore we must use delete+insert to alter any of the key fields.
2134 : *
2135 : * For every other situation there can only be one owner for a given extent,
2136 : * so we can call the regular _free function.
2137 : */
2138 : STATIC int
2139 57341005 : xfs_rmap_unmap_shared(
2140 : struct xfs_btree_cur *cur,
2141 : xfs_agblock_t bno,
2142 : xfs_extlen_t len,
2143 : bool unwritten,
2144 : const struct xfs_owner_info *oinfo)
2145 : {
2146 57341005 : struct xfs_mount *mp = cur->bc_mp;
2147 57341005 : struct xfs_rmap_irec ltrec;
2148 57341005 : uint64_t ltoff;
2149 57341005 : int error = 0;
2150 57341005 : int i;
2151 57341005 : uint64_t owner;
2152 57341005 : uint64_t offset;
2153 57341005 : unsigned int flags;
2154 :
2155 57341005 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2156 57341005 : if (unwritten)
2157 6062565 : flags |= XFS_RMAP_UNWRITTEN;
2158 57341005 : trace_xfs_rmap_unmap(mp, cur->bc_ag.pag->pag_agno, bno, len,
2159 : unwritten, oinfo);
2160 :
2161 : /*
2162 : * We should always have a left record because there's a static record
2163 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
2164 : * will not ever be removed from the tree.
2165 : */
2166 57341062 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, flags,
2167 : <rec, &i);
2168 57341051 : if (error)
2169 285 : goto out_error;
2170 57340766 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2171 0 : xfs_btree_mark_sick(cur);
2172 0 : error = -EFSCORRUPTED;
2173 0 : goto out_error;
2174 : }
2175 57340766 : ltoff = ltrec.rm_offset;
2176 :
2177 : /* Make sure the extent we found covers the entire freeing range. */
2178 57340766 : if (XFS_IS_CORRUPT(mp,
2179 : ltrec.rm_startblock > bno ||
2180 : ltrec.rm_startblock + ltrec.rm_blockcount <
2181 : bno + len)) {
2182 0 : xfs_btree_mark_sick(cur);
2183 0 : error = -EFSCORRUPTED;
2184 0 : goto out_error;
2185 : }
2186 :
2187 : /* Make sure the owner matches what we expect to find in the tree. */
2188 57340766 : if (XFS_IS_CORRUPT(mp, owner != ltrec.rm_owner)) {
2189 0 : xfs_btree_mark_sick(cur);
2190 0 : error = -EFSCORRUPTED;
2191 0 : goto out_error;
2192 : }
2193 :
2194 : /* Make sure the unwritten flag matches. */
2195 57340766 : if (XFS_IS_CORRUPT(mp,
2196 : (flags & XFS_RMAP_UNWRITTEN) !=
2197 : (ltrec.rm_flags & XFS_RMAP_UNWRITTEN))) {
2198 0 : xfs_btree_mark_sick(cur);
2199 0 : error = -EFSCORRUPTED;
2200 0 : goto out_error;
2201 : }
2202 :
2203 : /* Check the offset. */
2204 57340766 : if (XFS_IS_CORRUPT(mp, ltrec.rm_offset > offset)) {
2205 0 : xfs_btree_mark_sick(cur);
2206 0 : error = -EFSCORRUPTED;
2207 0 : goto out_error;
2208 : }
2209 57340766 : if (XFS_IS_CORRUPT(mp, offset > ltoff + ltrec.rm_blockcount)) {
2210 0 : xfs_btree_mark_sick(cur);
2211 0 : error = -EFSCORRUPTED;
2212 0 : goto out_error;
2213 : }
2214 :
2215 57340766 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
2216 : /* Exact match, simply remove the record from rmap tree. */
2217 50425565 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2218 : ltrec.rm_blockcount, ltrec.rm_owner,
2219 : ltrec.rm_offset, ltrec.rm_flags);
2220 50425562 : if (error)
2221 0 : goto out_error;
2222 6915201 : } else if (ltrec.rm_startblock == bno) {
2223 : /*
2224 : * Overlap left hand side of extent: move the start, trim the
2225 : * length and update the current record.
2226 : *
2227 : * ltbno ltlen
2228 : * Orig: |oooooooooooooooooooo|
2229 : * Freeing: |fffffffff|
2230 : * Result: |rrrrrrrrrr|
2231 : * bno len
2232 : */
2233 :
2234 : /* Delete prev rmap. */
2235 1782170 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2236 : ltrec.rm_blockcount, ltrec.rm_owner,
2237 : ltrec.rm_offset, ltrec.rm_flags);
2238 1782171 : if (error)
2239 0 : goto out_error;
2240 :
2241 : /* Add an rmap at the new offset. */
2242 1782171 : ltrec.rm_startblock += len;
2243 1782171 : ltrec.rm_blockcount -= len;
2244 1782171 : ltrec.rm_offset += len;
2245 1782171 : error = xfs_rmap_insert(cur, ltrec.rm_startblock,
2246 : ltrec.rm_blockcount, ltrec.rm_owner,
2247 : ltrec.rm_offset, ltrec.rm_flags);
2248 1782170 : if (error)
2249 0 : goto out_error;
2250 5133031 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
2251 : /*
2252 : * Overlap right hand side of extent: trim the length and
2253 : * update the current record.
2254 : *
2255 : * ltbno ltlen
2256 : * Orig: |oooooooooooooooooooo|
2257 : * Freeing: |fffffffff|
2258 : * Result: |rrrrrrrrrr|
2259 : * bno len
2260 : */
2261 1110564 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2262 : ltrec.rm_blockcount, ltrec.rm_owner,
2263 : ltrec.rm_offset, ltrec.rm_flags, &i);
2264 1110565 : if (error)
2265 0 : goto out_error;
2266 1110565 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2267 0 : xfs_btree_mark_sick(cur);
2268 0 : error = -EFSCORRUPTED;
2269 0 : goto out_error;
2270 : }
2271 1110565 : ltrec.rm_blockcount -= len;
2272 1110565 : error = xfs_rmap_update(cur, <rec);
2273 1110565 : if (error)
2274 0 : goto out_error;
2275 : } else {
2276 : /*
2277 : * Overlap middle of extent: trim the length of the existing
2278 : * record to the length of the new left-extent size, increment
2279 : * the insertion position so we can insert a new record
2280 : * containing the remaining right-extent space.
2281 : *
2282 : * ltbno ltlen
2283 : * Orig: |oooooooooooooooooooo|
2284 : * Freeing: |fffffffff|
2285 : * Result: |rrrrr| |rrrr|
2286 : * bno len
2287 : */
2288 4022467 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
2289 :
2290 : /* Shrink the left side of the rmap */
2291 4022467 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2292 : ltrec.rm_blockcount, ltrec.rm_owner,
2293 : ltrec.rm_offset, ltrec.rm_flags, &i);
2294 4022471 : if (error)
2295 0 : goto out_error;
2296 4022471 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2297 0 : xfs_btree_mark_sick(cur);
2298 0 : error = -EFSCORRUPTED;
2299 0 : goto out_error;
2300 : }
2301 4022471 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
2302 4022471 : error = xfs_rmap_update(cur, <rec);
2303 4022467 : if (error)
2304 0 : goto out_error;
2305 :
2306 : /* Add an rmap at the new offset */
2307 4022467 : error = xfs_rmap_insert(cur, bno + len,
2308 4022467 : orig_len - len - ltrec.rm_blockcount,
2309 : ltrec.rm_owner, offset + len,
2310 : ltrec.rm_flags);
2311 4022470 : if (error)
2312 0 : goto out_error;
2313 : }
2314 :
2315 57340767 : trace_xfs_rmap_unmap_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2316 : unwritten, oinfo);
2317 57341019 : out_error:
2318 57341019 : if (error)
2319 285 : trace_xfs_rmap_unmap_error(cur->bc_mp,
2320 285 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2321 57341019 : return error;
2322 : }
2323 :
2324 : /*
2325 : * Find an extent in the rmap btree and map it. For rmap extent types that
2326 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2327 : * that the prev/next records in the btree might belong to another owner.
2328 : * Therefore we must use delete+insert to alter any of the key fields.
2329 : *
2330 : * For every other situation there can only be one owner for a given extent,
2331 : * so we can call the regular _alloc function.
2332 : */
2333 : STATIC int
2334 76742293 : xfs_rmap_map_shared(
2335 : struct xfs_btree_cur *cur,
2336 : xfs_agblock_t bno,
2337 : xfs_extlen_t len,
2338 : bool unwritten,
2339 : const struct xfs_owner_info *oinfo)
2340 : {
2341 76742293 : struct xfs_mount *mp = cur->bc_mp;
2342 76742293 : struct xfs_rmap_irec ltrec;
2343 76742293 : struct xfs_rmap_irec gtrec;
2344 76742293 : int have_gt;
2345 76742293 : int have_lt;
2346 76742293 : int error = 0;
2347 76742293 : int i;
2348 76742293 : uint64_t owner;
2349 76742293 : uint64_t offset;
2350 76742293 : unsigned int flags = 0;
2351 :
2352 76742293 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2353 76742293 : if (unwritten)
2354 14837909 : flags |= XFS_RMAP_UNWRITTEN;
2355 76742293 : trace_xfs_rmap_map(mp, cur->bc_ag.pag->pag_agno, bno, len,
2356 : unwritten, oinfo);
2357 :
2358 : /* Is there a left record that abuts our range? */
2359 76742348 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, flags,
2360 : <rec, &have_lt);
2361 76742386 : if (error)
2362 214 : goto out_error;
2363 76742172 : if (have_lt &&
2364 9563026 : !xfs_rmap_is_mergeable(<rec, owner, flags))
2365 374944 : have_lt = 0;
2366 :
2367 : /* Is there a right record that abuts our range? */
2368 76742172 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
2369 : flags, &have_gt);
2370 76742189 : if (error)
2371 1 : goto out_error;
2372 76742188 : if (have_gt) {
2373 619018 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
2374 619018 : if (error)
2375 0 : goto out_error;
2376 619018 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
2377 0 : xfs_btree_mark_sick(cur);
2378 0 : error = -EFSCORRUPTED;
2379 0 : goto out_error;
2380 : }
2381 619018 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
2382 619018 : cur->bc_ag.pag->pag_agno, gtrec.rm_startblock,
2383 : gtrec.rm_blockcount, gtrec.rm_owner,
2384 : gtrec.rm_offset, gtrec.rm_flags);
2385 :
2386 619018 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
2387 108138 : have_gt = 0;
2388 : }
2389 :
2390 76742188 : if (have_lt &&
2391 9188082 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
2392 1457205 : ltrec.rm_offset + ltrec.rm_blockcount == offset) {
2393 : /*
2394 : * Left edge contiguous, merge into left record.
2395 : *
2396 : * ltbno ltlen
2397 : * orig: |ooooooooo|
2398 : * adding: |aaaaaaaaa|
2399 : * result: |rrrrrrrrrrrrrrrrrrr|
2400 : * bno len
2401 : */
2402 1457205 : ltrec.rm_blockcount += len;
2403 1457205 : if (have_gt &&
2404 333331 : bno + len == gtrec.rm_startblock &&
2405 333331 : offset + len == gtrec.rm_offset) {
2406 : /*
2407 : * Right edge also contiguous, delete right record
2408 : * and merge into left record.
2409 : *
2410 : * ltbno ltlen gtbno gtlen
2411 : * orig: |ooooooooo| |ooooooooo|
2412 : * adding: |aaaaaaaaa|
2413 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
2414 : */
2415 333331 : ltrec.rm_blockcount += gtrec.rm_blockcount;
2416 333331 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2417 : gtrec.rm_blockcount, gtrec.rm_owner,
2418 : gtrec.rm_offset, gtrec.rm_flags);
2419 333331 : if (error)
2420 0 : goto out_error;
2421 : }
2422 :
2423 : /* Point the cursor back to the left record and update. */
2424 1457205 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2425 : ltrec.rm_blockcount, ltrec.rm_owner,
2426 : ltrec.rm_offset, ltrec.rm_flags, &i);
2427 1457205 : if (error)
2428 0 : goto out_error;
2429 1457205 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2430 0 : xfs_btree_mark_sick(cur);
2431 0 : error = -EFSCORRUPTED;
2432 0 : goto out_error;
2433 : }
2434 :
2435 1457205 : error = xfs_rmap_update(cur, <rec);
2436 1457205 : if (error)
2437 0 : goto out_error;
2438 75284983 : } else if (have_gt &&
2439 177549 : bno + len == gtrec.rm_startblock &&
2440 177549 : offset + len == gtrec.rm_offset) {
2441 : /*
2442 : * Right edge contiguous, merge into right record.
2443 : *
2444 : * gtbno gtlen
2445 : * Orig: |ooooooooo|
2446 : * adding: |aaaaaaaaa|
2447 : * Result: |rrrrrrrrrrrrrrrrrrr|
2448 : * bno len
2449 : */
2450 : /* Delete the old record. */
2451 177549 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2452 : gtrec.rm_blockcount, gtrec.rm_owner,
2453 : gtrec.rm_offset, gtrec.rm_flags);
2454 177549 : if (error)
2455 0 : goto out_error;
2456 :
2457 : /* Move the start and re-add it. */
2458 177549 : gtrec.rm_startblock = bno;
2459 177549 : gtrec.rm_blockcount += len;
2460 177549 : gtrec.rm_offset = offset;
2461 177549 : error = xfs_rmap_insert(cur, gtrec.rm_startblock,
2462 : gtrec.rm_blockcount, gtrec.rm_owner,
2463 : gtrec.rm_offset, gtrec.rm_flags);
2464 177549 : if (error)
2465 0 : goto out_error;
2466 : } else {
2467 : /*
2468 : * No contiguous edge with identical owner, insert
2469 : * new record at current cursor position.
2470 : */
2471 75107434 : error = xfs_rmap_insert(cur, bno, len, owner, offset, flags);
2472 75107417 : if (error)
2473 158 : goto out_error;
2474 : }
2475 :
2476 76742013 : trace_xfs_rmap_map_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2477 : unwritten, oinfo);
2478 76742390 : out_error:
2479 76742390 : if (error)
2480 373 : trace_xfs_rmap_map_error(cur->bc_mp,
2481 373 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2482 76742390 : return error;
2483 : }
2484 :
2485 : /* Insert a raw rmap into the rmapbt. */
2486 : int
2487 10996062 : xfs_rmap_map_raw(
2488 : struct xfs_btree_cur *cur,
2489 : struct xfs_rmap_irec *rmap)
2490 : {
2491 10996062 : struct xfs_owner_info oinfo;
2492 :
2493 10996062 : xfs_owner_info_pack(&oinfo, rmap->rm_owner, rmap->rm_offset,
2494 : rmap->rm_flags);
2495 :
2496 10996062 : if ((rmap->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK |
2497 8106343 : XFS_RMAP_UNWRITTEN)) ||
2498 8106343 : XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2499 3022902 : return xfs_rmap_map(cur, rmap->rm_startblock,
2500 : rmap->rm_blockcount,
2501 3022902 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2502 : &oinfo);
2503 :
2504 7973160 : return xfs_rmap_map_shared(cur, rmap->rm_startblock,
2505 : rmap->rm_blockcount,
2506 7973160 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2507 : &oinfo);
2508 : }
2509 :
2510 : struct xfs_rmap_query_range_info {
2511 : xfs_rmap_query_range_fn fn;
2512 : void *priv;
2513 : };
2514 :
2515 : /* Format btree record and pass to our callback. */
2516 : STATIC int
2517 49242179020 : xfs_rmap_query_range_helper(
2518 : struct xfs_btree_cur *cur,
2519 : const union xfs_btree_rec *rec,
2520 : void *priv)
2521 : {
2522 49242179020 : struct xfs_rmap_query_range_info *query = priv;
2523 49242179020 : struct xfs_rmap_irec irec;
2524 49242179020 : xfs_failaddr_t fa;
2525 :
2526 49242179020 : fa = xfs_rmap_btrec_to_irec(rec, &irec);
2527 49430083802 : if (!fa)
2528 49423828110 : fa = xfs_rmap_check_irec(cur, &irec);
2529 49527449054 : if (fa)
2530 0 : return xfs_rmap_complain_bad_rec(cur, fa, &irec);
2531 :
2532 49527449054 : return query->fn(cur, &irec, query->priv);
2533 : }
2534 :
2535 : /* Find all rmaps between two keys. */
2536 : int
2537 505984600 : xfs_rmap_query_range(
2538 : struct xfs_btree_cur *cur,
2539 : const struct xfs_rmap_irec *low_rec,
2540 : const struct xfs_rmap_irec *high_rec,
2541 : xfs_rmap_query_range_fn fn,
2542 : void *priv)
2543 : {
2544 505984600 : union xfs_btree_irec low_brec = { .r = *low_rec };
2545 505984600 : union xfs_btree_irec high_brec = { .r = *high_rec };
2546 505984600 : struct xfs_rmap_query_range_info query = { .priv = priv, .fn = fn };
2547 :
2548 505984600 : return xfs_btree_query_range(cur, &low_brec, &high_brec,
2549 : xfs_rmap_query_range_helper, &query);
2550 : }
2551 :
2552 : /* Find all rmaps. */
2553 : int
2554 1679473 : xfs_rmap_query_all(
2555 : struct xfs_btree_cur *cur,
2556 : xfs_rmap_query_range_fn fn,
2557 : void *priv)
2558 : {
2559 1679473 : struct xfs_rmap_query_range_info query;
2560 :
2561 1679473 : query.priv = priv;
2562 1679473 : query.fn = fn;
2563 1679473 : return xfs_btree_query_all(cur, xfs_rmap_query_range_helper, &query);
2564 : }
2565 :
2566 : /* Clean up after calling xfs_rmap_finish_one. */
2567 : void
2568 214694853 : xfs_rmap_finish_one_cleanup(
2569 : struct xfs_trans *tp,
2570 : struct xfs_btree_cur *rcur,
2571 : int error)
2572 : {
2573 214696118 : struct xfs_buf *agbp;
2574 :
2575 214694853 : if (rcur == NULL)
2576 : return;
2577 214695081 : agbp = rcur->bc_ag.agbp;
2578 214693816 : xfs_btree_del_cursor(rcur, error);
2579 214696428 : if (error)
2580 1080 : xfs_trans_brelse(tp, agbp);
2581 : }
2582 :
2583 : /* Commit an rmap operation into the ondisk tree. */
2584 : int
2585 234387015 : __xfs_rmap_finish_intent(
2586 : struct xfs_btree_cur *rcur,
2587 : enum xfs_rmap_intent_type op,
2588 : xfs_agblock_t bno,
2589 : xfs_extlen_t len,
2590 : const struct xfs_owner_info *oinfo,
2591 : bool unwritten)
2592 : {
2593 234387015 : switch (op) {
2594 47298676 : case XFS_RMAP_ALLOC:
2595 : case XFS_RMAP_MAP:
2596 47298676 : return xfs_rmap_map(rcur, bno, len, unwritten, oinfo);
2597 68769149 : case XFS_RMAP_MAP_SHARED:
2598 68769149 : return xfs_rmap_map_shared(rcur, bno, len, unwritten, oinfo);
2599 42627057 : case XFS_RMAP_FREE:
2600 : case XFS_RMAP_UNMAP:
2601 42627057 : return xfs_rmap_unmap(rcur, bno, len, unwritten, oinfo);
2602 57341045 : case XFS_RMAP_UNMAP_SHARED:
2603 57341045 : return xfs_rmap_unmap_shared(rcur, bno, len, unwritten, oinfo);
2604 5982550 : case XFS_RMAP_CONVERT:
2605 5982550 : return xfs_rmap_convert(rcur, bno, len, !unwritten, oinfo);
2606 12368538 : case XFS_RMAP_CONVERT_SHARED:
2607 12368538 : return xfs_rmap_convert_shared(rcur, bno, len, !unwritten,
2608 12368538 : oinfo);
2609 0 : default:
2610 0 : ASSERT(0);
2611 0 : return -EFSCORRUPTED;
2612 : }
2613 : }
2614 :
2615 : /*
2616 : * Process one of the deferred rmap operations. We pass back the
2617 : * btree cursor to maintain our lock on the rmapbt between calls.
2618 : * This saves time and eliminates a buffer deadlock between the
2619 : * superblock and the AGF because we'll always grab them in the same
2620 : * order.
2621 : */
2622 : int
2623 234291065 : xfs_rmap_finish_one(
2624 : struct xfs_trans *tp,
2625 : struct xfs_rmap_intent *ri,
2626 : struct xfs_btree_cur **pcur)
2627 : {
2628 234291065 : struct xfs_mount *mp = tp->t_mountp;
2629 234291065 : struct xfs_btree_cur *rcur;
2630 234291065 : struct xfs_buf *agbp = NULL;
2631 234291065 : int error = 0;
2632 234291065 : struct xfs_owner_info oinfo;
2633 234291065 : xfs_agblock_t bno;
2634 234291065 : bool unwritten;
2635 :
2636 234291065 : bno = XFS_FSB_TO_AGBNO(mp, ri->ri_bmap.br_startblock);
2637 :
2638 234291065 : trace_xfs_rmap_deferred(mp, ri->ri_pag->pag_agno, ri->ri_type, bno,
2639 : ri->ri_owner, ri->ri_whichfork,
2640 : ri->ri_bmap.br_startoff, ri->ri_bmap.br_blockcount,
2641 : ri->ri_bmap.br_state);
2642 :
2643 234293230 : if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_RMAP_FINISH_ONE))
2644 : return -EIO;
2645 :
2646 : /*
2647 : * If we haven't gotten a cursor or the cursor AG doesn't match
2648 : * the startblock, get one now.
2649 : */
2650 234291315 : rcur = *pcur;
2651 234291315 : if (rcur != NULL && rcur->bc_ag.pag != ri->ri_pag) {
2652 1265 : xfs_rmap_finish_one_cleanup(tp, rcur, 0);
2653 1265 : rcur = NULL;
2654 1265 : *pcur = NULL;
2655 : }
2656 234291315 : if (rcur == NULL) {
2657 : /*
2658 : * Refresh the freelist before we start changing the
2659 : * rmapbt, because a shape change could cause us to
2660 : * allocate blocks.
2661 : */
2662 214697574 : error = xfs_free_extent_fix_freelist(tp, ri->ri_pag, &agbp);
2663 214698855 : if (error) {
2664 1031 : xfs_ag_mark_sick(ri->ri_pag, XFS_SICK_AG_AGFL);
2665 1031 : return error;
2666 : }
2667 214697824 : if (XFS_IS_CORRUPT(tp->t_mountp, !agbp)) {
2668 0 : xfs_ag_mark_sick(ri->ri_pag, XFS_SICK_AG_AGFL);
2669 0 : return -EFSCORRUPTED;
2670 : }
2671 :
2672 214697824 : rcur = xfs_rmapbt_init_cursor(mp, tp, agbp, ri->ri_pag);
2673 : }
2674 234291343 : *pcur = rcur;
2675 :
2676 234291343 : xfs_rmap_ino_owner(&oinfo, ri->ri_owner, ri->ri_whichfork,
2677 : ri->ri_bmap.br_startoff);
2678 234291343 : unwritten = ri->ri_bmap.br_state == XFS_EXT_UNWRITTEN;
2679 234291343 : bno = XFS_FSB_TO_AGBNO(rcur->bc_mp, ri->ri_bmap.br_startblock);
2680 :
2681 234291343 : error = __xfs_rmap_finish_intent(rcur, ri->ri_type, bno,
2682 234291343 : ri->ri_bmap.br_blockcount, &oinfo, unwritten);
2683 234290398 : if (error)
2684 : return error;
2685 :
2686 234287612 : xfs_rmap_update_hook(tp, ri->ri_pag, ri->ri_type, bno,
2687 234287612 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2688 234287612 : return 0;
2689 : }
2690 :
2691 : /*
2692 : * Don't defer an rmap if we aren't an rmap filesystem.
2693 : */
2694 : static bool
2695 : xfs_rmap_update_is_needed(
2696 : struct xfs_mount *mp,
2697 : int whichfork)
2698 : {
2699 233692170 : return xfs_has_rmapbt(mp) && whichfork != XFS_COW_FORK;
2700 : }
2701 :
2702 : /*
2703 : * Record a rmap intent; the list is kept sorted first by AG and then by
2704 : * increasing age.
2705 : */
2706 : static void
2707 234281278 : __xfs_rmap_add(
2708 : struct xfs_trans *tp,
2709 : enum xfs_rmap_intent_type type,
2710 : uint64_t owner,
2711 : int whichfork,
2712 : struct xfs_bmbt_irec *bmap)
2713 : {
2714 234281278 : struct xfs_rmap_intent *ri;
2715 :
2716 234281278 : trace_xfs_rmap_defer(tp->t_mountp,
2717 234281278 : XFS_FSB_TO_AGNO(tp->t_mountp, bmap->br_startblock),
2718 : type,
2719 : XFS_FSB_TO_AGBNO(tp->t_mountp, bmap->br_startblock),
2720 : owner, whichfork,
2721 : bmap->br_startoff,
2722 : bmap->br_blockcount,
2723 : bmap->br_state);
2724 :
2725 234280223 : ri = kmem_cache_alloc(xfs_rmap_intent_cache, GFP_NOFS | __GFP_NOFAIL);
2726 234277156 : INIT_LIST_HEAD(&ri->ri_list);
2727 234277156 : ri->ri_type = type;
2728 234277156 : ri->ri_owner = owner;
2729 234277156 : ri->ri_whichfork = whichfork;
2730 234277156 : ri->ri_bmap = *bmap;
2731 :
2732 234277156 : xfs_rmap_update_get_group(tp->t_mountp, ri);
2733 234277115 : xfs_defer_add(tp, XFS_DEFER_OPS_TYPE_RMAP, &ri->ri_list);
2734 234281616 : }
2735 :
2736 : /* Map an extent into a file. */
2737 : void
2738 138003614 : xfs_rmap_map_extent(
2739 : struct xfs_trans *tp,
2740 : struct xfs_inode *ip,
2741 : int whichfork,
2742 : struct xfs_bmbt_irec *PREV)
2743 : {
2744 138003614 : enum xfs_rmap_intent_type type = XFS_RMAP_MAP;
2745 :
2746 276007228 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2747 : return;
2748 :
2749 115660943 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2750 68738879 : type = XFS_RMAP_MAP_SHARED;
2751 :
2752 115660943 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2753 : }
2754 :
2755 : /* Unmap an extent out of a file. */
2756 : void
2757 114255000 : xfs_rmap_unmap_extent(
2758 : struct xfs_trans *tp,
2759 : struct xfs_inode *ip,
2760 : int whichfork,
2761 : struct xfs_bmbt_irec *PREV)
2762 : {
2763 114255000 : enum xfs_rmap_intent_type type = XFS_RMAP_UNMAP;
2764 :
2765 228510000 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2766 : return;
2767 :
2768 97708837 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2769 57317434 : type = XFS_RMAP_UNMAP_SHARED;
2770 :
2771 97708837 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2772 : }
2773 :
2774 : /*
2775 : * Convert a data fork extent from unwritten to real or vice versa.
2776 : *
2777 : * Note that tp can be NULL here as no transaction is used for COW fork
2778 : * unwritten conversion.
2779 : */
2780 : void
2781 26536244 : xfs_rmap_convert_extent(
2782 : struct xfs_mount *mp,
2783 : struct xfs_trans *tp,
2784 : struct xfs_inode *ip,
2785 : int whichfork,
2786 : struct xfs_bmbt_irec *PREV)
2787 : {
2788 26536244 : enum xfs_rmap_intent_type type = XFS_RMAP_CONVERT;
2789 :
2790 53072488 : if (!xfs_rmap_update_is_needed(mp, whichfork))
2791 : return;
2792 :
2793 18325387 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2794 12345973 : type = XFS_RMAP_CONVERT_SHARED;
2795 :
2796 18325387 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2797 : }
2798 :
2799 : /* Schedule the creation of an rmap for non-file data. */
2800 : void
2801 359509 : xfs_rmap_alloc_extent(
2802 : struct xfs_trans *tp,
2803 : xfs_agnumber_t agno,
2804 : xfs_agblock_t bno,
2805 : xfs_extlen_t len,
2806 : uint64_t owner)
2807 : {
2808 359509 : struct xfs_bmbt_irec bmap;
2809 :
2810 359509 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2811 0 : return;
2812 :
2813 359509 : bmap.br_startblock = XFS_AGB_TO_FSB(tp->t_mountp, agno, bno);
2814 359509 : bmap.br_blockcount = len;
2815 359509 : bmap.br_startoff = 0;
2816 359509 : bmap.br_state = XFS_EXT_NORM;
2817 :
2818 359509 : __xfs_rmap_add(tp, XFS_RMAP_ALLOC, owner, XFS_DATA_FORK, &bmap);
2819 : }
2820 :
2821 : /* Schedule the deletion of an rmap for non-file data. */
2822 : void
2823 2230826 : xfs_rmap_free_extent(
2824 : struct xfs_trans *tp,
2825 : xfs_agnumber_t agno,
2826 : xfs_agblock_t bno,
2827 : xfs_extlen_t len,
2828 : uint64_t owner)
2829 : {
2830 2230826 : struct xfs_bmbt_irec bmap;
2831 :
2832 2230826 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2833 0 : return;
2834 :
2835 2230826 : bmap.br_startblock = XFS_AGB_TO_FSB(tp->t_mountp, agno, bno);
2836 2230826 : bmap.br_blockcount = len;
2837 2230826 : bmap.br_startoff = 0;
2838 2230826 : bmap.br_state = XFS_EXT_NORM;
2839 :
2840 2230826 : __xfs_rmap_add(tp, XFS_RMAP_FREE, owner, XFS_DATA_FORK, &bmap);
2841 : }
2842 :
2843 : /* Compare rmap records. Returns -1 if a < b, 1 if a > b, and 0 if equal. */
2844 : int
2845 47555525 : xfs_rmap_compare(
2846 : const struct xfs_rmap_irec *a,
2847 : const struct xfs_rmap_irec *b)
2848 : {
2849 47555525 : __u64 oa;
2850 47555525 : __u64 ob;
2851 :
2852 47555525 : oa = xfs_rmap_irec_offset_pack(a);
2853 47555525 : ob = xfs_rmap_irec_offset_pack(b);
2854 :
2855 47555525 : if (a->rm_startblock < b->rm_startblock)
2856 : return -1;
2857 47552508 : else if (a->rm_startblock > b->rm_startblock)
2858 : return 1;
2859 7842385 : else if (a->rm_owner < b->rm_owner)
2860 : return -1;
2861 7763476 : else if (a->rm_owner > b->rm_owner)
2862 : return 1;
2863 7658428 : else if (oa < ob)
2864 : return -1;
2865 7605525 : else if (oa > ob)
2866 : return 1;
2867 : else
2868 3714 : return 0;
2869 : }
2870 :
2871 : /*
2872 : * Scan the physical storage part of the keyspace of the reverse mapping index
2873 : * and tell us if the area has no records, is fully mapped by records, or is
2874 : * partially filled.
2875 : */
2876 : int
2877 382396471 : xfs_rmap_has_records(
2878 : struct xfs_btree_cur *cur,
2879 : xfs_agblock_t bno,
2880 : xfs_extlen_t len,
2881 : enum xbtree_recpacking *outcome)
2882 : {
2883 382396471 : union xfs_btree_key mask = {
2884 : .rmap.rm_startblock = cpu_to_be32(-1U),
2885 : };
2886 382396471 : union xfs_btree_irec low;
2887 382396471 : union xfs_btree_irec high;
2888 :
2889 382396471 : memset(&low, 0, sizeof(low));
2890 382396471 : low.r.rm_startblock = bno;
2891 382396471 : memset(&high, 0xFF, sizeof(high));
2892 382396471 : high.r.rm_startblock = bno + len - 1;
2893 :
2894 382396471 : return xfs_btree_has_records(cur, &low, &high, &mask, outcome);
2895 : }
2896 :
2897 : struct xfs_rmap_ownercount {
2898 : /* Owner that we're looking for. */
2899 : struct xfs_rmap_irec good;
2900 :
2901 : /* rmap search keys */
2902 : struct xfs_rmap_irec low;
2903 : struct xfs_rmap_irec high;
2904 :
2905 : struct xfs_rmap_matches *results;
2906 :
2907 : /* Stop early if we find a nonmatch? */
2908 : bool stop_on_nonmatch;
2909 : };
2910 :
2911 : /* Does this rmap represent space that can have multiple owners? */
2912 : static inline bool
2913 : xfs_rmap_shareable(
2914 : struct xfs_mount *mp,
2915 : const struct xfs_rmap_irec *rmap)
2916 : {
2917 45751786 : if (!xfs_has_reflink(mp))
2918 : return false;
2919 91503497 : if (XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2920 : return false;
2921 45106961 : if (rmap->rm_flags & (XFS_RMAP_ATTR_FORK |
2922 : XFS_RMAP_BMBT_BLOCK))
2923 878193 : return false;
2924 : return true;
2925 : }
2926 :
2927 : static inline void
2928 401385577 : xfs_rmap_ownercount_init(
2929 : struct xfs_rmap_ownercount *roc,
2930 : xfs_agblock_t bno,
2931 : xfs_extlen_t len,
2932 : const struct xfs_owner_info *oinfo,
2933 : struct xfs_rmap_matches *results)
2934 : {
2935 401385577 : memset(roc, 0, sizeof(*roc));
2936 401385577 : roc->results = results;
2937 :
2938 401385577 : roc->low.rm_startblock = bno;
2939 401385577 : memset(&roc->high, 0xFF, sizeof(roc->high));
2940 401385577 : roc->high.rm_startblock = bno + len - 1;
2941 :
2942 401385577 : memset(results, 0, sizeof(*results));
2943 401385577 : roc->good.rm_startblock = bno;
2944 401385577 : roc->good.rm_blockcount = len;
2945 401385577 : roc->good.rm_owner = oinfo->oi_owner;
2946 401385577 : roc->good.rm_offset = oinfo->oi_offset;
2947 401385577 : if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK)
2948 906843 : roc->good.rm_flags |= XFS_RMAP_ATTR_FORK;
2949 401385577 : if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK)
2950 1138709 : roc->good.rm_flags |= XFS_RMAP_BMBT_BLOCK;
2951 401385577 : }
2952 :
2953 : /* Figure out if this is a match for the owner. */
2954 : STATIC int
2955 431165206 : xfs_rmap_count_owners_helper(
2956 : struct xfs_btree_cur *cur,
2957 : const struct xfs_rmap_irec *rec,
2958 : void *priv)
2959 : {
2960 431165206 : struct xfs_rmap_ownercount *roc = priv;
2961 431165206 : struct xfs_rmap_irec check = *rec;
2962 431165206 : unsigned int keyflags;
2963 431165206 : bool filedata;
2964 431165206 : int64_t delta;
2965 :
2966 431165206 : filedata = !XFS_RMAP_NON_INODE_OWNER(check.rm_owner) &&
2967 174675118 : !(check.rm_flags & XFS_RMAP_BMBT_BLOCK);
2968 :
2969 : /* Trim the part of check that comes before the comparison range. */
2970 431165206 : delta = (int64_t)roc->good.rm_startblock - check.rm_startblock;
2971 431165206 : if (delta > 0) {
2972 174069136 : check.rm_startblock += delta;
2973 174069136 : check.rm_blockcount -= delta;
2974 174069136 : if (filedata)
2975 1386539 : check.rm_offset += delta;
2976 : }
2977 :
2978 : /* Trim the part of check that comes after the comparison range. */
2979 431165206 : delta = (check.rm_startblock + check.rm_blockcount) -
2980 431165206 : (roc->good.rm_startblock + roc->good.rm_blockcount);
2981 431165206 : if (delta > 0)
2982 209112820 : check.rm_blockcount -= delta;
2983 :
2984 : /* Don't care about unwritten status for establishing ownership. */
2985 431165206 : keyflags = check.rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK);
2986 :
2987 431165206 : if (check.rm_startblock == roc->good.rm_startblock &&
2988 399773270 : check.rm_blockcount == roc->good.rm_blockcount &&
2989 399773270 : check.rm_owner == roc->good.rm_owner &&
2990 385413412 : check.rm_offset == roc->good.rm_offset &&
2991 385413461 : keyflags == roc->good.rm_flags) {
2992 385413420 : roc->results->matches++;
2993 : } else {
2994 45751786 : roc->results->non_owner_matches++;
2995 91503572 : if (xfs_rmap_shareable(cur->bc_mp, &roc->good) ^
2996 : xfs_rmap_shareable(cur->bc_mp, &check))
2997 44228772 : roc->results->bad_non_owner_matches++;
2998 : }
2999 :
3000 431165206 : if (roc->results->non_owner_matches && roc->stop_on_nonmatch)
3001 0 : return -ECANCELED;
3002 :
3003 : return 0;
3004 : }
3005 :
3006 : /* Count the number of owners and non-owners of this range of blocks. */
3007 : int
3008 400412551 : xfs_rmap_count_owners(
3009 : struct xfs_btree_cur *cur,
3010 : xfs_agblock_t bno,
3011 : xfs_extlen_t len,
3012 : const struct xfs_owner_info *oinfo,
3013 : struct xfs_rmap_matches *results)
3014 : {
3015 400412551 : struct xfs_rmap_ownercount roc;
3016 400412551 : int error;
3017 :
3018 400412551 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, results);
3019 400413594 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
3020 : xfs_rmap_count_owners_helper, &roc);
3021 400414215 : if (error)
3022 : return error;
3023 :
3024 : /*
3025 : * There can't be any non-owner rmaps that conflict with the given
3026 : * owner if we didn't find any rmaps matching the owner.
3027 : */
3028 400414215 : if (!results->matches)
3029 15885268 : results->bad_non_owner_matches = 0;
3030 :
3031 : return 0;
3032 : }
3033 :
3034 : /*
3035 : * Given an extent and some owner info, can we find records overlapping
3036 : * the extent whose owner info does not match the given owner?
3037 : */
3038 : int
3039 972523 : xfs_rmap_has_other_keys(
3040 : struct xfs_btree_cur *cur,
3041 : xfs_agblock_t bno,
3042 : xfs_extlen_t len,
3043 : const struct xfs_owner_info *oinfo,
3044 : bool *has_other)
3045 : {
3046 972523 : struct xfs_rmap_matches res;
3047 972523 : struct xfs_rmap_ownercount roc;
3048 972523 : int error;
3049 :
3050 972523 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, &res);
3051 972522 : roc.stop_on_nonmatch = true;
3052 :
3053 972522 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
3054 : xfs_rmap_count_owners_helper, &roc);
3055 972531 : if (error == -ECANCELED) {
3056 0 : *has_other = true;
3057 0 : return 0;
3058 : }
3059 972531 : if (error)
3060 : return error;
3061 :
3062 972531 : *has_other = false;
3063 972531 : return 0;
3064 : }
3065 :
3066 : const struct xfs_owner_info XFS_RMAP_OINFO_SKIP_UPDATE = {
3067 : .oi_owner = XFS_RMAP_OWN_NULL,
3068 : };
3069 : const struct xfs_owner_info XFS_RMAP_OINFO_ANY_OWNER = {
3070 : .oi_owner = XFS_RMAP_OWN_UNKNOWN,
3071 : };
3072 : const struct xfs_owner_info XFS_RMAP_OINFO_FS = {
3073 : .oi_owner = XFS_RMAP_OWN_FS,
3074 : };
3075 : const struct xfs_owner_info XFS_RMAP_OINFO_LOG = {
3076 : .oi_owner = XFS_RMAP_OWN_LOG,
3077 : };
3078 : const struct xfs_owner_info XFS_RMAP_OINFO_AG = {
3079 : .oi_owner = XFS_RMAP_OWN_AG,
3080 : };
3081 : const struct xfs_owner_info XFS_RMAP_OINFO_INOBT = {
3082 : .oi_owner = XFS_RMAP_OWN_INOBT,
3083 : };
3084 : const struct xfs_owner_info XFS_RMAP_OINFO_INODES = {
3085 : .oi_owner = XFS_RMAP_OWN_INODES,
3086 : };
3087 : const struct xfs_owner_info XFS_RMAP_OINFO_REFC = {
3088 : .oi_owner = XFS_RMAP_OWN_REFC,
3089 : };
3090 : const struct xfs_owner_info XFS_RMAP_OINFO_COW = {
3091 : .oi_owner = XFS_RMAP_OWN_COW,
3092 : };
3093 :
3094 : int __init
3095 12 : xfs_rmap_intent_init_cache(void)
3096 : {
3097 12 : xfs_rmap_intent_cache = kmem_cache_create("xfs_rmap_intent",
3098 : sizeof(struct xfs_rmap_intent),
3099 : 0, 0, NULL);
3100 :
3101 12 : return xfs_rmap_intent_cache != NULL ? 0 : -ENOMEM;
3102 : }
3103 :
3104 : void
3105 12 : xfs_rmap_intent_destroy_cache(void)
3106 : {
3107 12 : kmem_cache_destroy(xfs_rmap_intent_cache);
3108 12 : xfs_rmap_intent_cache = NULL;
3109 12 : }
|
