Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2014 Red Hat, Inc.
4 : * All Rights Reserved.
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_bit.h"
13 : #include "xfs_mount.h"
14 : #include "xfs_sb.h"
15 : #include "xfs_defer.h"
16 : #include "xfs_btree.h"
17 : #include "xfs_trans.h"
18 : #include "xfs_alloc.h"
19 : #include "xfs_rmap.h"
20 : #include "xfs_rmap_btree.h"
21 : #include "xfs_trace.h"
22 : #include "xfs_errortag.h"
23 : #include "xfs_error.h"
24 : #include "xfs_inode.h"
25 : #include "xfs_ag.h"
26 : #include "xfs_health.h"
27 : #include "xfs_rtgroup.h"
28 : #include "xfs_rtrmap_btree.h"
29 :
30 : struct kmem_cache *xfs_rmap_intent_cache;
31 :
32 : /*
33 : * Lookup the first record less than or equal to [bno, len, owner, offset]
34 : * in the btree given by cur.
35 : */
36 : int
37 943799719 : xfs_rmap_lookup_le(
38 : struct xfs_btree_cur *cur,
39 : xfs_agblock_t bno,
40 : uint64_t owner,
41 : uint64_t offset,
42 : unsigned int flags,
43 : struct xfs_rmap_irec *irec,
44 : int *stat)
45 : {
46 943799719 : int get_stat = 0;
47 943799719 : int error;
48 :
49 943799719 : cur->bc_rec.r.rm_startblock = bno;
50 943799719 : cur->bc_rec.r.rm_blockcount = 0;
51 943799719 : cur->bc_rec.r.rm_owner = owner;
52 943799719 : cur->bc_rec.r.rm_offset = offset;
53 943799719 : cur->bc_rec.r.rm_flags = flags;
54 :
55 943799719 : error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat);
56 943803595 : if (error || !(*stat) || !irec)
57 : return error;
58 :
59 934857089 : error = xfs_rmap_get_rec(cur, irec, &get_stat);
60 934865277 : if (error)
61 : return error;
62 934865277 : if (!get_stat) {
63 0 : xfs_btree_mark_sick(cur);
64 0 : return -EFSCORRUPTED;
65 : }
66 :
67 : return 0;
68 : }
69 :
70 : /*
71 : * Lookup the record exactly matching [bno, len, owner, offset]
72 : * in the btree given by cur.
73 : */
74 : int
75 0 : xfs_rmap_lookup_eq(
76 : struct xfs_btree_cur *cur,
77 : xfs_agblock_t bno,
78 : xfs_extlen_t len,
79 : uint64_t owner,
80 : uint64_t offset,
81 : unsigned int flags,
82 : int *stat)
83 : {
84 400911984 : cur->bc_rec.r.rm_startblock = bno;
85 400911984 : cur->bc_rec.r.rm_blockcount = len;
86 400911984 : cur->bc_rec.r.rm_owner = owner;
87 400911984 : cur->bc_rec.r.rm_offset = offset;
88 400911984 : cur->bc_rec.r.rm_flags = flags;
89 0 : return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat);
90 : }
91 :
92 : /*
93 : * Update the record referred to by cur to the value given
94 : * by [bno, len, owner, offset].
95 : * This either works (return 0) or gets an EFSCORRUPTED error.
96 : */
97 : STATIC int
98 57471716 : xfs_rmap_update(
99 : struct xfs_btree_cur *cur,
100 : struct xfs_rmap_irec *irec)
101 : {
102 57471716 : union xfs_btree_rec rec;
103 57471716 : int error;
104 :
105 57471716 : trace_xfs_rmap_update(cur, irec->rm_startblock, irec->rm_blockcount,
106 : irec->rm_owner, irec->rm_offset, irec->rm_flags);
107 :
108 57471890 : rec.rmap.rm_startblock = cpu_to_be32(irec->rm_startblock);
109 57471890 : rec.rmap.rm_blockcount = cpu_to_be32(irec->rm_blockcount);
110 57471890 : rec.rmap.rm_owner = cpu_to_be64(irec->rm_owner);
111 57471890 : rec.rmap.rm_offset = cpu_to_be64(
112 : xfs_rmap_irec_offset_pack(irec));
113 57471890 : error = xfs_btree_update(cur, &rec);
114 57471891 : if (error)
115 0 : trace_xfs_rmap_update_error(cur, error, _RET_IP_);
116 57471891 : return error;
117 : }
118 :
119 : int
120 135685248 : xfs_rmap_insert(
121 : struct xfs_btree_cur *rcur,
122 : xfs_agblock_t agbno,
123 : xfs_extlen_t len,
124 : uint64_t owner,
125 : uint64_t offset,
126 : unsigned int flags)
127 : {
128 135685248 : int i;
129 135685248 : int error;
130 :
131 135685248 : trace_xfs_rmap_insert(rcur, agbno, len, owner, offset, flags);
132 :
133 135685646 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
134 135685791 : if (error)
135 0 : goto done;
136 135685791 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 0)) {
137 0 : xfs_btree_mark_sick(rcur);
138 0 : error = -EFSCORRUPTED;
139 0 : goto done;
140 : }
141 :
142 135685791 : rcur->bc_rec.r.rm_startblock = agbno;
143 135685791 : rcur->bc_rec.r.rm_blockcount = len;
144 135685791 : rcur->bc_rec.r.rm_owner = owner;
145 135685791 : rcur->bc_rec.r.rm_offset = offset;
146 135685791 : rcur->bc_rec.r.rm_flags = flags;
147 135685791 : error = xfs_btree_insert(rcur, &i);
148 135685280 : if (error)
149 173 : goto done;
150 135685107 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
151 0 : xfs_btree_mark_sick(rcur);
152 0 : error = -EFSCORRUPTED;
153 0 : goto done;
154 : }
155 135685280 : done:
156 135685280 : if (error)
157 173 : trace_xfs_rmap_insert_error(rcur, error, _RET_IP_);
158 135685280 : return error;
159 : }
160 :
161 : STATIC int
162 78310813 : xfs_rmap_delete(
163 : struct xfs_btree_cur *rcur,
164 : xfs_agblock_t agbno,
165 : xfs_extlen_t len,
166 : uint64_t owner,
167 : uint64_t offset,
168 : unsigned int flags)
169 : {
170 78310813 : int i;
171 78310813 : int error;
172 :
173 78310813 : trace_xfs_rmap_delete(rcur, agbno, len, owner, offset, flags);
174 :
175 78310845 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
176 78310924 : if (error)
177 0 : goto done;
178 78310924 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
179 0 : xfs_btree_mark_sick(rcur);
180 0 : error = -EFSCORRUPTED;
181 0 : goto done;
182 : }
183 :
184 78310924 : error = xfs_btree_delete(rcur, &i);
185 78310861 : if (error)
186 1 : goto done;
187 78310860 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
188 0 : xfs_btree_mark_sick(rcur);
189 0 : error = -EFSCORRUPTED;
190 0 : goto done;
191 : }
192 78310861 : done:
193 78310861 : if (error)
194 1 : trace_xfs_rmap_delete_error(rcur, error, _RET_IP_);
195 78310861 : return error;
196 : }
197 :
198 : /* Convert an internal btree record to an rmap record. */
199 : xfs_failaddr_t
200 >10511*10^7 : xfs_rmap_btrec_to_irec(
201 : const union xfs_btree_rec *rec,
202 : struct xfs_rmap_irec *irec)
203 : {
204 >10511*10^7 : irec->rm_startblock = be32_to_cpu(rec->rmap.rm_startblock);
205 >10511*10^7 : irec->rm_blockcount = be32_to_cpu(rec->rmap.rm_blockcount);
206 >10511*10^7 : irec->rm_owner = be64_to_cpu(rec->rmap.rm_owner);
207 >10511*10^7 : return xfs_rmap_irec_offset_unpack(be64_to_cpu(rec->rmap.rm_offset),
208 : irec);
209 : }
210 :
211 : inline xfs_failaddr_t
212 62479362659 : xfs_rmap_check_perag_irec(
213 : struct xfs_perag *pag,
214 : const struct xfs_rmap_irec *irec)
215 : {
216 62479362659 : struct xfs_mount *mp = pag->pag_mount;
217 62479362659 : bool is_inode;
218 62479362659 : bool is_unwritten;
219 62479362659 : bool is_bmbt;
220 62479362659 : bool is_attr;
221 :
222 62479362659 : if (irec->rm_blockcount == 0)
223 0 : return __this_address;
224 62479362659 : if (irec->rm_startblock <= XFS_AGFL_BLOCK(mp)) {
225 5062176 : if (irec->rm_owner != XFS_RMAP_OWN_FS)
226 0 : return __this_address;
227 5062176 : if (irec->rm_blockcount != XFS_AGFL_BLOCK(mp) + 1)
228 0 : return __this_address;
229 : } else {
230 : /* check for valid extent range, including overflow */
231 62474300483 : if (!xfs_verify_agbext(pag, irec->rm_startblock,
232 : irec->rm_blockcount))
233 0 : return __this_address;
234 : }
235 :
236 62479362659 : if (!(xfs_verify_ino(mp, irec->rm_owner) ||
237 1864124861 : (irec->rm_owner <= XFS_RMAP_OWN_FS &&
238 : irec->rm_owner >= XFS_RMAP_OWN_MIN)))
239 0 : return __this_address;
240 :
241 : /* Check flags. */
242 62694694060 : is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner);
243 62694694060 : is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK;
244 62694694060 : is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK;
245 62694694060 : is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN;
246 :
247 62694694060 : if (is_bmbt && irec->rm_offset != 0)
248 0 : return __this_address;
249 :
250 62694694060 : if (!is_inode && irec->rm_offset != 0)
251 0 : return __this_address;
252 :
253 62694694060 : if (is_unwritten && (is_bmbt || !is_inode || is_attr))
254 0 : return __this_address;
255 :
256 62694694060 : if (!is_inode && (is_bmbt || is_unwritten || is_attr))
257 0 : return __this_address;
258 :
259 : /* Check for a valid fork offset, if applicable. */
260 >12065*10^7 : if (is_inode && !is_bmbt &&
261 57891460024 : !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
262 0 : return __this_address;
263 :
264 : return NULL;
265 : }
266 :
267 : inline xfs_failaddr_t
268 42904305984 : xfs_rmap_check_rtgroup_irec(
269 : struct xfs_rtgroup *rtg,
270 : const struct xfs_rmap_irec *irec)
271 : {
272 42904305984 : struct xfs_mount *mp = rtg->rtg_mount;
273 42904305984 : bool is_inode;
274 42904305984 : bool is_unwritten;
275 42904305984 : bool is_bmbt;
276 42904305984 : bool is_attr;
277 42904305984 : bool is_cow;
278 :
279 42904305984 : if (irec->rm_blockcount == 0)
280 0 : return __this_address;
281 :
282 42904305984 : if (irec->rm_owner == XFS_RMAP_OWN_FS) {
283 1145949 : if (irec->rm_startblock != 0)
284 0 : return __this_address;
285 1145949 : if (irec->rm_blockcount != mp->m_sb.sb_rextsize)
286 0 : return __this_address;
287 1145949 : if (irec->rm_offset != 0)
288 0 : return __this_address;
289 42903160035 : } else if (irec->rm_owner == XFS_RMAP_OWN_COW) {
290 100270970 : if (!xfs_has_rtreflink(mp))
291 0 : return __this_address;
292 100270970 : if (!xfs_verify_rgbext(rtg, irec->rm_startblock,
293 : irec->rm_blockcount))
294 0 : return __this_address;
295 : } else {
296 42802889065 : if (!xfs_verify_rgbext(rtg, irec->rm_startblock,
297 : irec->rm_blockcount))
298 0 : return __this_address;
299 : }
300 :
301 42904305984 : if (!(xfs_verify_ino(mp, irec->rm_owner) ||
302 101417282 : (irec->rm_owner <= XFS_RMAP_OWN_FS &&
303 : irec->rm_owner >= XFS_RMAP_OWN_MIN)))
304 0 : return __this_address;
305 :
306 : /* Check flags. */
307 43466606472 : is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner);
308 43466606472 : is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK;
309 43466606472 : is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK;
310 43466606472 : is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN;
311 43466606472 : is_cow = xfs_has_rtreflink(mp) &&
312 : irec->rm_owner == XFS_RMAP_OWN_COW;
313 :
314 43466606472 : if (!is_inode && !is_cow && irec->rm_owner != XFS_RMAP_OWN_FS)
315 0 : return __this_address;
316 :
317 43466606472 : if (!is_inode && irec->rm_offset != 0)
318 0 : return __this_address;
319 :
320 43466606472 : if (is_bmbt || is_attr)
321 0 : return __this_address;
322 :
323 43466606472 : if (is_unwritten && !is_inode)
324 0 : return __this_address;
325 :
326 43466606472 : if (is_unwritten && is_cow)
327 0 : return __this_address;
328 :
329 : /* Check for a valid fork offset, if applicable. */
330 86862050099 : if (is_inode &&
331 43365188847 : !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
332 0 : return __this_address;
333 :
334 : return NULL;
335 : }
336 :
337 : /* Simple checks for rmap records. */
338 : xfs_failaddr_t
339 >10618*10^7 : xfs_rmap_check_irec(
340 : struct xfs_btree_cur *cur,
341 : const struct xfs_rmap_irec *irec)
342 : {
343 >10618*10^7 : if (cur->bc_btnum == XFS_BTNUM_RTRMAP) {
344 43455429098 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
345 30462577 : return xfs_rmap_check_rtgroup_irec(cur->bc_mem.rtg,
346 : irec);
347 43424966521 : return xfs_rmap_check_rtgroup_irec(cur->bc_ino.rtg, irec);
348 : }
349 :
350 62730443420 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
351 82140935 : return xfs_rmap_check_perag_irec(cur->bc_mem.pag, irec);
352 62648302485 : return xfs_rmap_check_perag_irec(cur->bc_ag.pag, irec);
353 : }
354 :
355 : static inline int
356 0 : xfs_rmap_complain_bad_rec(
357 : struct xfs_btree_cur *cur,
358 : xfs_failaddr_t fa,
359 : const struct xfs_rmap_irec *irec)
360 : {
361 0 : struct xfs_mount *mp = cur->bc_mp;
362 :
363 0 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
364 0 : xfs_warn(mp,
365 : "In-Memory Reverse Mapping BTree record corruption detected at %pS!", fa);
366 0 : else if (cur->bc_btnum == XFS_BTNUM_RTRMAP)
367 0 : xfs_warn(mp,
368 : "RT Reverse Mapping BTree record corruption in rtgroup %u detected at %pS!",
369 : cur->bc_ino.rtg->rtg_rgno, fa);
370 : else
371 0 : xfs_warn(mp,
372 : "Reverse Mapping BTree record corruption in AG %d detected at %pS!",
373 : cur->bc_ag.pag->pag_agno, fa);
374 0 : xfs_warn(mp,
375 : "Owner 0x%llx, flags 0x%x, start block 0x%x block count 0x%x",
376 : irec->rm_owner, irec->rm_flags, irec->rm_startblock,
377 : irec->rm_blockcount);
378 0 : xfs_btree_mark_sick(cur);
379 0 : return -EFSCORRUPTED;
380 : }
381 :
382 : /*
383 : * Get the data from the pointed-to record.
384 : */
385 : int
386 2262354512 : xfs_rmap_get_rec(
387 : struct xfs_btree_cur *cur,
388 : struct xfs_rmap_irec *irec,
389 : int *stat)
390 : {
391 2262354512 : union xfs_btree_rec *rec;
392 2262354512 : xfs_failaddr_t fa;
393 2262354512 : int error;
394 :
395 2262354512 : error = xfs_btree_get_rec(cur, &rec, stat);
396 2262379678 : if (error || !*stat)
397 : return error;
398 :
399 2262381387 : fa = xfs_rmap_btrec_to_irec(rec, irec);
400 2262386045 : if (!fa)
401 2262387085 : fa = xfs_rmap_check_irec(cur, irec);
402 2262375196 : if (fa)
403 0 : return xfs_rmap_complain_bad_rec(cur, fa, irec);
404 :
405 : return 0;
406 : }
407 :
408 : struct xfs_find_left_neighbor_info {
409 : struct xfs_rmap_irec high;
410 : struct xfs_rmap_irec *irec;
411 : };
412 :
413 : /* For each rmap given, figure out if it matches the key we want. */
414 : STATIC int
415 314321969 : xfs_rmap_find_left_neighbor_helper(
416 : struct xfs_btree_cur *cur,
417 : const struct xfs_rmap_irec *rec,
418 : void *priv)
419 : {
420 314321969 : struct xfs_find_left_neighbor_info *info = priv;
421 :
422 314321969 : trace_xfs_rmap_find_left_neighbor_candidate(cur, rec->rm_startblock,
423 314321969 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
424 314321969 : rec->rm_flags);
425 :
426 314323562 : if (rec->rm_owner != info->high.rm_owner)
427 : return 0;
428 49141764 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
429 49141764 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
430 49093846 : rec->rm_offset + rec->rm_blockcount - 1 != info->high.rm_offset)
431 : return 0;
432 :
433 11682758 : *info->irec = *rec;
434 11682758 : return -ECANCELED;
435 : }
436 :
437 : /*
438 : * Find the record to the left of the given extent, being careful only to
439 : * return a match with the same owner and adjacent physical and logical
440 : * block ranges.
441 : */
442 : STATIC int
443 149099225 : xfs_rmap_find_left_neighbor(
444 : struct xfs_btree_cur *cur,
445 : xfs_agblock_t bno,
446 : uint64_t owner,
447 : uint64_t offset,
448 : unsigned int flags,
449 : struct xfs_rmap_irec *irec,
450 : int *stat)
451 : {
452 149099225 : struct xfs_find_left_neighbor_info info;
453 149099225 : int found = 0;
454 149099225 : int error;
455 :
456 149099225 : *stat = 0;
457 149099225 : if (bno == 0)
458 : return 0;
459 149099225 : info.high.rm_startblock = bno - 1;
460 149099225 : info.high.rm_owner = owner;
461 149099225 : if (!XFS_RMAP_NON_INODE_OWNER(owner) &&
462 149098646 : !(flags & XFS_RMAP_BMBT_BLOCK)) {
463 149099225 : if (offset == 0)
464 : return 0;
465 147604905 : info.high.rm_offset = offset - 1;
466 : } else
467 0 : info.high.rm_offset = 0;
468 147604905 : info.high.rm_flags = flags;
469 147604905 : info.high.rm_blockcount = 0;
470 147604905 : info.irec = irec;
471 :
472 147604905 : trace_xfs_rmap_find_left_neighbor_query(cur, bno, 0, owner, offset,
473 : flags);
474 :
475 : /*
476 : * Historically, we always used the range query to walk every reverse
477 : * mapping that could possibly overlap the key that the caller asked
478 : * for, and filter out the ones that don't. That is very slow when
479 : * there are a lot of records.
480 : *
481 : * However, there are two scenarios where the classic btree search can
482 : * produce correct results -- if the index contains a record that is an
483 : * exact match for the lookup key; and if there are no other records
484 : * between the record we want and the key we supplied.
485 : *
486 : * As an optimization, try a non-overlapped lookup first. This makes
487 : * extent conversion and remap operations run a bit faster if the
488 : * physical extents aren't being shared. If we don't find what we
489 : * want, we fall back to the overlapped query.
490 : */
491 147604768 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
492 : &found);
493 147605145 : if (error)
494 : return error;
495 147604943 : if (found)
496 147604975 : error = xfs_rmap_find_left_neighbor_helper(cur, irec, &info);
497 147605041 : if (!error)
498 136710004 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
499 : xfs_rmap_find_left_neighbor_helper, &info);
500 147605049 : if (error != -ECANCELED)
501 : return error;
502 :
503 11682758 : *stat = 1;
504 11682758 : trace_xfs_rmap_find_left_neighbor_result(cur, irec->rm_startblock,
505 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
506 : irec->rm_flags);
507 11682758 : return 0;
508 : }
509 :
510 : /* For each rmap given, figure out if it matches the key we want. */
511 : STATIC int
512 200954430 : xfs_rmap_lookup_le_range_helper(
513 : struct xfs_btree_cur *cur,
514 : const struct xfs_rmap_irec *rec,
515 : void *priv)
516 : {
517 200954430 : struct xfs_find_left_neighbor_info *info = priv;
518 :
519 200954430 : trace_xfs_rmap_lookup_le_range_candidate(cur, rec->rm_startblock,
520 200954430 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
521 200954430 : rec->rm_flags);
522 :
523 200954748 : if (rec->rm_owner != info->high.rm_owner)
524 : return 0;
525 171985208 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
526 171985208 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
527 171985208 : (rec->rm_offset > info->high.rm_offset ||
528 171950500 : rec->rm_offset + rec->rm_blockcount <= info->high.rm_offset))
529 : return 0;
530 :
531 171913774 : *info->irec = *rec;
532 171913774 : return -ECANCELED;
533 : }
534 :
535 : /*
536 : * Find the record to the left of the given extent, being careful only to
537 : * return a match with the same owner and overlapping physical and logical
538 : * block ranges. This is the overlapping-interval version of
539 : * xfs_rmap_lookup_le.
540 : */
541 : int
542 171914907 : xfs_rmap_lookup_le_range(
543 : struct xfs_btree_cur *cur,
544 : xfs_agblock_t bno,
545 : uint64_t owner,
546 : uint64_t offset,
547 : unsigned int flags,
548 : struct xfs_rmap_irec *irec,
549 : int *stat)
550 : {
551 171914907 : struct xfs_find_left_neighbor_info info;
552 171914907 : int found = 0;
553 171914907 : int error;
554 :
555 171914907 : info.high.rm_startblock = bno;
556 171914907 : info.high.rm_owner = owner;
557 171914907 : if (!XFS_RMAP_NON_INODE_OWNER(owner) && !(flags & XFS_RMAP_BMBT_BLOCK))
558 171914907 : info.high.rm_offset = offset;
559 : else
560 0 : info.high.rm_offset = 0;
561 171914907 : info.high.rm_flags = flags;
562 171914907 : info.high.rm_blockcount = 0;
563 171914907 : *stat = 0;
564 171914907 : info.irec = irec;
565 :
566 171914907 : trace_xfs_rmap_lookup_le_range(cur, bno, 0, owner, offset, flags);
567 :
568 : /*
569 : * Historically, we always used the range query to walk every reverse
570 : * mapping that could possibly overlap the key that the caller asked
571 : * for, and filter out the ones that don't. That is very slow when
572 : * there are a lot of records.
573 : *
574 : * However, there are two scenarios where the classic btree search can
575 : * produce correct results -- if the index contains a record that is an
576 : * exact match for the lookup key; and if there are no other records
577 : * between the record we want and the key we supplied.
578 : *
579 : * As an optimization, try a non-overlapped lookup first. This makes
580 : * scrub run much faster on most filesystems because bmbt records are
581 : * usually an exact match for rmap records. If we don't find what we
582 : * want, we fall back to the overlapped query.
583 : */
584 171914676 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
585 : &found);
586 171913932 : if (error)
587 : return error;
588 171913648 : if (found)
589 171913865 : error = xfs_rmap_lookup_le_range_helper(cur, irec, &info);
590 171914815 : if (!error)
591 732405 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
592 : xfs_rmap_lookup_le_range_helper, &info);
593 171914815 : if (error != -ECANCELED)
594 : return error;
595 :
596 171914691 : *stat = 1;
597 171914691 : trace_xfs_rmap_lookup_le_range_result(cur, irec->rm_startblock,
598 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
599 : irec->rm_flags);
600 171914691 : return 0;
601 : }
602 :
603 : /*
604 : * Perform all the relevant owner checks for a removal op. If we're doing an
605 : * unknown-owner removal then we have no owner information to check.
606 : */
607 : static int
608 59931222 : xfs_rmap_free_check_owner(
609 : struct xfs_btree_cur *cur,
610 : uint64_t ltoff,
611 : struct xfs_rmap_irec *rec,
612 : xfs_extlen_t len,
613 : uint64_t owner,
614 : uint64_t offset,
615 : unsigned int flags)
616 : {
617 59931222 : struct xfs_mount *mp = cur->bc_mp;
618 59931222 : int error = 0;
619 :
620 59931222 : if (owner == XFS_RMAP_OWN_UNKNOWN)
621 : return 0;
622 :
623 : /* Make sure the unwritten flag matches. */
624 59930786 : if (XFS_IS_CORRUPT(mp,
625 : (flags & XFS_RMAP_UNWRITTEN) !=
626 : (rec->rm_flags & XFS_RMAP_UNWRITTEN))) {
627 0 : xfs_btree_mark_sick(cur);
628 0 : error = -EFSCORRUPTED;
629 0 : goto out;
630 : }
631 :
632 : /* Make sure the owner matches what we expect to find in the tree. */
633 59930786 : if (XFS_IS_CORRUPT(mp, owner != rec->rm_owner)) {
634 0 : xfs_btree_mark_sick(cur);
635 0 : error = -EFSCORRUPTED;
636 0 : goto out;
637 : }
638 :
639 : /* Check the offset, if necessary. */
640 59930786 : if (XFS_RMAP_NON_INODE_OWNER(owner))
641 4344982 : goto out;
642 :
643 55585804 : if (flags & XFS_RMAP_BMBT_BLOCK) {
644 1011594 : if (XFS_IS_CORRUPT(mp,
645 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK))) {
646 0 : xfs_btree_mark_sick(cur);
647 0 : error = -EFSCORRUPTED;
648 0 : goto out;
649 : }
650 : } else {
651 54574210 : if (XFS_IS_CORRUPT(mp, rec->rm_offset > offset)) {
652 0 : xfs_btree_mark_sick(cur);
653 0 : error = -EFSCORRUPTED;
654 0 : goto out;
655 : }
656 54574210 : if (XFS_IS_CORRUPT(mp,
657 : offset + len > ltoff + rec->rm_blockcount)) {
658 0 : xfs_btree_mark_sick(cur);
659 0 : error = -EFSCORRUPTED;
660 0 : goto out;
661 : }
662 : }
663 :
664 : out:
665 : return error;
666 : }
667 :
668 : /*
669 : * Find the extent in the rmap btree and remove it.
670 : *
671 : * The record we find should always be an exact match for the extent that we're
672 : * looking for, since we insert them into the btree without modification.
673 : *
674 : * Special Case #1: when growing the filesystem, we "free" an extent when
675 : * growing the last AG. This extent is new space and so it is not tracked as
676 : * used space in the btree. The growfs code will pass in an owner of
677 : * XFS_RMAP_OWN_NULL to indicate that it expected that there is no owner of this
678 : * extent. We verify that - the extent lookup result in a record that does not
679 : * overlap.
680 : *
681 : * Special Case #2: EFIs do not record the owner of the extent, so when
682 : * recovering EFIs from the log we pass in XFS_RMAP_OWN_UNKNOWN to tell the rmap
683 : * btree to ignore the owner (i.e. wildcard match) so we don't trigger
684 : * corruption checks during log recovery.
685 : */
686 : STATIC int
687 59944568 : xfs_rmap_unmap(
688 : struct xfs_btree_cur *cur,
689 : xfs_agblock_t bno,
690 : xfs_extlen_t len,
691 : bool unwritten,
692 : const struct xfs_owner_info *oinfo)
693 : {
694 59944568 : struct xfs_mount *mp = cur->bc_mp;
695 59944568 : struct xfs_rmap_irec ltrec;
696 59944568 : uint64_t ltoff;
697 59944568 : int error = 0;
698 59944568 : int i;
699 59944568 : uint64_t owner;
700 59944568 : uint64_t offset;
701 59944568 : unsigned int flags;
702 59944568 : bool ignore_off;
703 :
704 59944568 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
705 59944568 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
706 55586272 : (flags & XFS_RMAP_BMBT_BLOCK);
707 59944568 : if (unwritten)
708 2576707 : flags |= XFS_RMAP_UNWRITTEN;
709 59944568 : trace_xfs_rmap_unmap(cur, bno, len, unwritten, oinfo);
710 :
711 : /*
712 : * We should always have a left record because there's a static record
713 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
714 : * will not ever be removed from the tree.
715 : */
716 59944510 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec, &i);
717 59944342 : if (error)
718 80 : goto out_error;
719 59944262 : if (XFS_IS_CORRUPT(mp, i != 1)) {
720 0 : xfs_btree_mark_sick(cur);
721 0 : error = -EFSCORRUPTED;
722 0 : goto out_error;
723 : }
724 :
725 59944262 : trace_xfs_rmap_lookup_le_range_result(cur, ltrec.rm_startblock,
726 : ltrec.rm_blockcount, ltrec.rm_owner, ltrec.rm_offset,
727 : ltrec.rm_flags);
728 59944041 : ltoff = ltrec.rm_offset;
729 :
730 : /*
731 : * For growfs, the incoming extent must be beyond the left record we
732 : * just found as it is new space and won't be used by anyone. This is
733 : * just a corruption check as we don't actually do anything with this
734 : * extent. Note that we need to use >= instead of > because it might
735 : * be the case that the "left" extent goes all the way to EOFS.
736 : */
737 59944041 : if (owner == XFS_RMAP_OWN_NULL) {
738 28 : if (XFS_IS_CORRUPT(mp,
739 : bno <
740 : ltrec.rm_startblock + ltrec.rm_blockcount)) {
741 0 : xfs_btree_mark_sick(cur);
742 0 : error = -EFSCORRUPTED;
743 0 : goto out_error;
744 : }
745 28 : goto out_done;
746 : }
747 :
748 : /*
749 : * If we're doing an unknown-owner removal for EFI recovery, we expect
750 : * to find the full range in the rmapbt or nothing at all. If we
751 : * don't find any rmaps overlapping either end of the range, we're
752 : * done. Hopefully this means that the EFI creator already queued
753 : * (and finished) a RUI to remove the rmap.
754 : */
755 59944013 : if (owner == XFS_RMAP_OWN_UNKNOWN &&
756 13263 : ltrec.rm_startblock + ltrec.rm_blockcount <= bno) {
757 12827 : struct xfs_rmap_irec rtrec;
758 :
759 12827 : error = xfs_btree_increment(cur, 0, &i);
760 12826 : if (error)
761 0 : goto out_error;
762 12826 : if (i == 0)
763 12826 : goto out_done;
764 12607 : error = xfs_rmap_get_rec(cur, &rtrec, &i);
765 12607 : if (error)
766 0 : goto out_error;
767 12607 : if (XFS_IS_CORRUPT(mp, i != 1)) {
768 0 : xfs_btree_mark_sick(cur);
769 0 : error = -EFSCORRUPTED;
770 0 : goto out_error;
771 : }
772 12607 : if (rtrec.rm_startblock >= bno + len)
773 12607 : goto out_done;
774 : }
775 :
776 : /* Make sure the extent we found covers the entire freeing range. */
777 59931186 : if (XFS_IS_CORRUPT(mp,
778 : ltrec.rm_startblock > bno ||
779 : ltrec.rm_startblock + ltrec.rm_blockcount <
780 : bno + len)) {
781 0 : xfs_btree_mark_sick(cur);
782 0 : error = -EFSCORRUPTED;
783 0 : goto out_error;
784 : }
785 :
786 : /* Check owner information. */
787 59931186 : error = xfs_rmap_free_check_owner(cur, ltoff, <rec, len, owner,
788 : offset, flags);
789 59931381 : if (error)
790 0 : goto out_error;
791 :
792 59931381 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
793 : /* exact match, simply remove the record from rmap tree */
794 53144968 : trace_xfs_rmap_delete(cur, ltrec.rm_startblock,
795 : ltrec.rm_blockcount, ltrec.rm_owner,
796 : ltrec.rm_offset, ltrec.rm_flags);
797 53144723 : error = xfs_btree_delete(cur, &i);
798 53144632 : if (error)
799 2 : goto out_error;
800 53144630 : if (XFS_IS_CORRUPT(mp, i != 1)) {
801 0 : xfs_btree_mark_sick(cur);
802 0 : error = -EFSCORRUPTED;
803 0 : goto out_error;
804 : }
805 6786413 : } else if (ltrec.rm_startblock == bno) {
806 : /*
807 : * overlap left hand side of extent: move the start, trim the
808 : * length and update the current record.
809 : *
810 : * ltbno ltlen
811 : * Orig: |oooooooooooooooooooo|
812 : * Freeing: |fffffffff|
813 : * Result: |rrrrrrrrrr|
814 : * bno len
815 : */
816 1986195 : ltrec.rm_startblock += len;
817 1986195 : ltrec.rm_blockcount -= len;
818 1986195 : if (!ignore_off)
819 269555 : ltrec.rm_offset += len;
820 1986195 : error = xfs_rmap_update(cur, <rec);
821 1986195 : if (error)
822 0 : goto out_error;
823 4800218 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
824 : /*
825 : * overlap right hand side of extent: trim the length and update
826 : * the current record.
827 : *
828 : * ltbno ltlen
829 : * Orig: |oooooooooooooooooooo|
830 : * Freeing: |fffffffff|
831 : * Result: |rrrrrrrrrr|
832 : * bno len
833 : */
834 661862 : ltrec.rm_blockcount -= len;
835 661862 : error = xfs_rmap_update(cur, <rec);
836 661861 : if (error)
837 0 : goto out_error;
838 : } else {
839 :
840 : /*
841 : * overlap middle of extent: trim the length of the existing
842 : * record to the length of the new left-extent size, increment
843 : * the insertion position so we can insert a new record
844 : * containing the remaining right-extent space.
845 : *
846 : * ltbno ltlen
847 : * Orig: |oooooooooooooooooooo|
848 : * Freeing: |fffffffff|
849 : * Result: |rrrrr| |rrrr|
850 : * bno len
851 : */
852 4138356 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
853 :
854 4138356 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
855 4138356 : error = xfs_rmap_update(cur, <rec);
856 4138356 : if (error)
857 0 : goto out_error;
858 :
859 4138356 : error = xfs_btree_increment(cur, 0, &i);
860 4138356 : if (error)
861 0 : goto out_error;
862 :
863 4138356 : cur->bc_rec.r.rm_startblock = bno + len;
864 4138356 : cur->bc_rec.r.rm_blockcount = orig_len - len -
865 4138356 : ltrec.rm_blockcount;
866 4138356 : cur->bc_rec.r.rm_owner = ltrec.rm_owner;
867 4138356 : if (ignore_off)
868 807308 : cur->bc_rec.r.rm_offset = 0;
869 : else
870 3331048 : cur->bc_rec.r.rm_offset = offset + len;
871 4138356 : cur->bc_rec.r.rm_flags = flags;
872 4138356 : trace_xfs_rmap_insert(cur, cur->bc_rec.r.rm_startblock,
873 : cur->bc_rec.r.rm_blockcount,
874 : cur->bc_rec.r.rm_owner,
875 : cur->bc_rec.r.rm_offset,
876 : cur->bc_rec.r.rm_flags);
877 4138356 : error = xfs_btree_insert(cur, &i);
878 4138356 : if (error)
879 0 : goto out_error;
880 : }
881 :
882 4138356 : out_done:
883 59943896 : trace_xfs_rmap_unmap_done(cur, bno, len, unwritten, oinfo);
884 59943889 : out_error:
885 59943889 : if (error)
886 82 : trace_xfs_rmap_unmap_error(cur, error, _RET_IP_);
887 59943889 : return error;
888 : }
889 :
890 : #ifdef CONFIG_XFS_LIVE_HOOKS
891 : /*
892 : * Use a static key here to reduce the overhead of rmapbt live updates. If
893 : * the compiler supports jump labels, the static branch will be replaced by a
894 : * nop sled when there are no hook users. Online fsck is currently the only
895 : * caller, so this is a reasonable tradeoff.
896 : *
897 : * Note: Patching the kernel code requires taking the cpu hotplug lock. Other
898 : * parts of the kernel allocate memory with that lock held, which means that
899 : * XFS callers cannot hold any locks that might be used by memory reclaim or
900 : * writeback when calling the static_branch_{inc,dec} functions.
901 : */
902 : DEFINE_STATIC_XFS_HOOK_SWITCH(xfs_rmap_hooks_switch);
903 :
904 : void
905 12659 : xfs_rmap_hook_disable(void)
906 : {
907 12659 : xfs_hooks_switch_off(&xfs_rmap_hooks_switch);
908 12675 : }
909 :
910 : void
911 12660 : xfs_rmap_hook_enable(void)
912 : {
913 12660 : xfs_hooks_switch_on(&xfs_rmap_hooks_switch);
914 12672 : }
915 :
916 : /* Call downstream hooks for a reverse mapping update. */
917 : static inline void
918 356070101 : xfs_rmap_update_hook(
919 : struct xfs_trans *tp,
920 : struct xfs_perag *pag,
921 : struct xfs_rtgroup *rtg,
922 : enum xfs_rmap_intent_type op,
923 : xfs_agblock_t startblock,
924 : xfs_extlen_t blockcount,
925 : bool unwritten,
926 : const struct xfs_owner_info *oinfo)
927 : {
928 361022705 : if (xfs_hooks_switched_on(&xfs_rmap_hooks_switch)) {
929 4952589 : struct xfs_rmap_update_params p = {
930 : .startblock = startblock,
931 : .blockcount = blockcount,
932 : .unwritten = unwritten,
933 : .oinfo = *oinfo, /* struct copy */
934 : };
935 :
936 4952589 : if (pag)
937 1275740 : xfs_hooks_call(&pag->pag_rmap_update_hooks, op, &p);
938 3676849 : else if (rtg)
939 3676849 : xfs_hooks_call(&rtg->rtg_rmap_update_hooks, op, &p);
940 : }
941 356072869 : }
942 :
943 : /* Call the specified function during a reverse mapping update. */
944 : int
945 9251 : xfs_rmap_hook_add(
946 : struct xfs_perag *pag,
947 : struct xfs_rmap_hook *hook)
948 : {
949 9251 : return xfs_hooks_add(&pag->pag_rmap_update_hooks, &hook->update_hook);
950 : }
951 :
952 : /* Stop calling the specified function during a reverse mapping update. */
953 : void
954 9260 : xfs_rmap_hook_del(
955 : struct xfs_perag *pag,
956 : struct xfs_rmap_hook *hook)
957 : {
958 9260 : xfs_hooks_del(&pag->pag_rmap_update_hooks, &hook->update_hook);
959 9259 : }
960 :
961 : # ifdef CONFIG_XFS_RT
962 : /* Call the specified function during a rt reverse mapping update. */
963 : int
964 3249 : xfs_rtrmap_hook_add(
965 : struct xfs_rtgroup *rtg,
966 : struct xfs_rmap_hook *hook)
967 : {
968 3249 : return xfs_hooks_add(&rtg->rtg_rmap_update_hooks, &hook->update_hook);
969 : }
970 :
971 : /* Stop calling the specified function during a rt reverse mapping update. */
972 : void
973 3249 : xfs_rtrmap_hook_del(
974 : struct xfs_rtgroup *rtg,
975 : struct xfs_rmap_hook *hook)
976 : {
977 3249 : xfs_hooks_del(&rtg->rtg_rmap_update_hooks, &hook->update_hook);
978 3248 : }
979 : # endif /* CONFIG_XFS_RT */
980 : #else
981 : # define xfs_rmap_update_hook(t, p, r, o, s, b, u, oi) do { } while(0)
982 : #endif /* CONFIG_XFS_LIVE_HOOKS */
983 :
984 : /*
985 : * Remove a reference to an extent in the rmap btree.
986 : */
987 : int
988 1596472 : xfs_rmap_free(
989 : struct xfs_trans *tp,
990 : struct xfs_buf *agbp,
991 : struct xfs_perag *pag,
992 : xfs_agblock_t bno,
993 : xfs_extlen_t len,
994 : const struct xfs_owner_info *oinfo)
995 : {
996 1596472 : struct xfs_mount *mp = tp->t_mountp;
997 1596472 : struct xfs_btree_cur *cur;
998 1596472 : int error;
999 :
1000 1596472 : if (!xfs_has_rmapbt(mp))
1001 : return 0;
1002 :
1003 1596435 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
1004 1596435 : xfs_rmap_update_hook(tp, pag, NULL, XFS_RMAP_UNMAP, bno, len, false,
1005 : oinfo);
1006 1596435 : error = xfs_rmap_unmap(cur, bno, len, false, oinfo);
1007 :
1008 1596429 : xfs_btree_del_cursor(cur, error);
1009 1596429 : return error;
1010 : }
1011 :
1012 : /*
1013 : * A mergeable rmap must have the same owner and the same values for
1014 : * the unwritten, attr_fork, and bmbt flags. The startblock and
1015 : * offset are checked separately.
1016 : */
1017 : static bool
1018 165095623 : xfs_rmap_is_mergeable(
1019 : struct xfs_rmap_irec *irec,
1020 : uint64_t owner,
1021 : unsigned int flags)
1022 : {
1023 165095623 : if (irec->rm_owner == XFS_RMAP_OWN_NULL)
1024 : return false;
1025 165095623 : if (irec->rm_owner != owner)
1026 : return false;
1027 60700273 : if ((flags & XFS_RMAP_UNWRITTEN) ^
1028 60700273 : (irec->rm_flags & XFS_RMAP_UNWRITTEN))
1029 : return false;
1030 58138505 : if ((flags & XFS_RMAP_ATTR_FORK) ^
1031 : (irec->rm_flags & XFS_RMAP_ATTR_FORK))
1032 : return false;
1033 58134064 : if ((flags & XFS_RMAP_BMBT_BLOCK) ^
1034 : (irec->rm_flags & XFS_RMAP_BMBT_BLOCK))
1035 210722 : return false;
1036 : return true;
1037 : }
1038 :
1039 : /*
1040 : * When we allocate a new block, the first thing we do is add a reference to
1041 : * the extent in the rmap btree. This takes the form of a [agbno, length,
1042 : * owner, offset] record. Flags are encoded in the high bits of the offset
1043 : * field.
1044 : */
1045 : STATIC int
1046 80440582 : xfs_rmap_map(
1047 : struct xfs_btree_cur *cur,
1048 : xfs_agblock_t bno,
1049 : xfs_extlen_t len,
1050 : bool unwritten,
1051 : const struct xfs_owner_info *oinfo)
1052 : {
1053 80440582 : struct xfs_mount *mp = cur->bc_mp;
1054 80440582 : struct xfs_rmap_irec ltrec;
1055 80440582 : struct xfs_rmap_irec gtrec;
1056 80440582 : int have_gt;
1057 80440582 : int have_lt;
1058 80440582 : int error = 0;
1059 80440582 : int i;
1060 80440582 : uint64_t owner;
1061 80440582 : uint64_t offset;
1062 80440582 : unsigned int flags = 0;
1063 80440582 : bool ignore_off;
1064 :
1065 80440582 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1066 80440582 : ASSERT(owner != 0);
1067 80440582 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
1068 77779374 : (flags & XFS_RMAP_BMBT_BLOCK);
1069 80440582 : if (unwritten)
1070 19214639 : flags |= XFS_RMAP_UNWRITTEN;
1071 80440582 : trace_xfs_rmap_map(cur, bno, len, unwritten, oinfo);
1072 80441720 : ASSERT(!xfs_rmap_should_skip_owner_update(oinfo));
1073 :
1074 : /*
1075 : * For the initial lookup, look for an exact match or the left-adjacent
1076 : * record for our insertion point. This will also give us the record for
1077 : * start block contiguity tests.
1078 : */
1079 80441720 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec,
1080 : &have_lt);
1081 80440382 : if (error)
1082 572 : goto out_error;
1083 80439810 : if (have_lt) {
1084 80419131 : trace_xfs_rmap_lookup_le_range_result(cur, ltrec.rm_startblock,
1085 : ltrec.rm_blockcount, ltrec.rm_owner,
1086 : ltrec.rm_offset, ltrec.rm_flags);
1087 :
1088 80419008 : if (!xfs_rmap_is_mergeable(<rec, owner, flags))
1089 55292645 : have_lt = 0;
1090 : }
1091 :
1092 80439687 : if (XFS_IS_CORRUPT(mp,
1093 : have_lt != 0 &&
1094 : ltrec.rm_startblock + ltrec.rm_blockcount > bno)) {
1095 0 : xfs_btree_mark_sick(cur);
1096 0 : error = -EFSCORRUPTED;
1097 0 : goto out_error;
1098 : }
1099 :
1100 : /*
1101 : * Increment the cursor to see if we have a right-adjacent record to our
1102 : * insertion point. This will give us the record for end block
1103 : * contiguity tests.
1104 : */
1105 80439687 : error = xfs_btree_increment(cur, 0, &have_gt);
1106 80440023 : if (error)
1107 4 : goto out_error;
1108 80440019 : if (have_gt) {
1109 71560770 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
1110 71561849 : if (error)
1111 0 : goto out_error;
1112 71561849 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
1113 0 : xfs_btree_mark_sick(cur);
1114 0 : error = -EFSCORRUPTED;
1115 0 : goto out_error;
1116 : }
1117 71561849 : if (XFS_IS_CORRUPT(mp, bno + len > gtrec.rm_startblock)) {
1118 0 : xfs_btree_mark_sick(cur);
1119 0 : error = -EFSCORRUPTED;
1120 0 : goto out_error;
1121 : }
1122 71561849 : trace_xfs_rmap_find_right_neighbor_result(cur,
1123 : gtrec.rm_startblock, gtrec.rm_blockcount,
1124 : gtrec.rm_owner, gtrec.rm_offset,
1125 : gtrec.rm_flags);
1126 71561568 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
1127 51038543 : have_gt = 0;
1128 : }
1129 :
1130 : /*
1131 : * Note: cursor currently points one record to the right of ltrec, even
1132 : * if there is no record in the tree to the right.
1133 : */
1134 80440817 : if (have_lt &&
1135 25126409 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
1136 5354814 : (ignore_off || ltrec.rm_offset + ltrec.rm_blockcount == offset)) {
1137 : /*
1138 : * left edge contiguous, merge into left record.
1139 : *
1140 : * ltbno ltlen
1141 : * orig: |ooooooooo|
1142 : * adding: |aaaaaaaaa|
1143 : * result: |rrrrrrrrrrrrrrrrrrr|
1144 : * bno len
1145 : */
1146 4049617 : ltrec.rm_blockcount += len;
1147 4049617 : if (have_gt &&
1148 1363784 : bno + len == gtrec.rm_startblock &&
1149 14637 : (ignore_off || offset + len == gtrec.rm_offset) &&
1150 48993 : (unsigned long)ltrec.rm_blockcount + len +
1151 48993 : gtrec.rm_blockcount <= XFS_RMAP_LEN_MAX) {
1152 : /*
1153 : * right edge also contiguous, delete right record
1154 : * and merge into left record.
1155 : *
1156 : * ltbno ltlen gtbno gtlen
1157 : * orig: |ooooooooo| |ooooooooo|
1158 : * adding: |aaaaaaaaa|
1159 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
1160 : */
1161 48993 : ltrec.rm_blockcount += gtrec.rm_blockcount;
1162 48993 : trace_xfs_rmap_delete(cur, gtrec.rm_startblock,
1163 : gtrec.rm_blockcount, gtrec.rm_owner,
1164 : gtrec.rm_offset, gtrec.rm_flags);
1165 48993 : error = xfs_btree_delete(cur, &i);
1166 48993 : if (error)
1167 0 : goto out_error;
1168 48993 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1169 0 : xfs_btree_mark_sick(cur);
1170 0 : error = -EFSCORRUPTED;
1171 0 : goto out_error;
1172 : }
1173 : }
1174 :
1175 : /* point the cursor back to the left record and update */
1176 4049617 : error = xfs_btree_decrement(cur, 0, &have_gt);
1177 4049618 : if (error)
1178 0 : goto out_error;
1179 4049618 : error = xfs_rmap_update(cur, <rec);
1180 4049613 : if (error)
1181 0 : goto out_error;
1182 76391200 : } else if (have_gt &&
1183 19159598 : bno + len == gtrec.rm_startblock &&
1184 898111 : (ignore_off || offset + len == gtrec.rm_offset)) {
1185 : /*
1186 : * right edge contiguous, merge into right record.
1187 : *
1188 : * gtbno gtlen
1189 : * Orig: |ooooooooo|
1190 : * adding: |aaaaaaaaa|
1191 : * Result: |rrrrrrrrrrrrrrrrrrr|
1192 : * bno len
1193 : */
1194 105691 : gtrec.rm_startblock = bno;
1195 105691 : gtrec.rm_blockcount += len;
1196 105691 : if (!ignore_off)
1197 6929 : gtrec.rm_offset = offset;
1198 105691 : error = xfs_rmap_update(cur, >rec);
1199 105690 : if (error)
1200 0 : goto out_error;
1201 : } else {
1202 : /*
1203 : * no contiguous edge with identical owner, insert
1204 : * new record at current cursor position.
1205 : */
1206 76285509 : cur->bc_rec.r.rm_startblock = bno;
1207 76285509 : cur->bc_rec.r.rm_blockcount = len;
1208 76285509 : cur->bc_rec.r.rm_owner = owner;
1209 76285509 : cur->bc_rec.r.rm_offset = offset;
1210 76285509 : cur->bc_rec.r.rm_flags = flags;
1211 76285509 : trace_xfs_rmap_insert(cur, bno, len, owner, offset, flags);
1212 76285672 : error = xfs_btree_insert(cur, &i);
1213 76285820 : if (error)
1214 37 : goto out_error;
1215 76285783 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1216 0 : xfs_btree_mark_sick(cur);
1217 0 : error = -EFSCORRUPTED;
1218 0 : goto out_error;
1219 : }
1220 : }
1221 :
1222 80441086 : trace_xfs_rmap_map_done(cur, bno, len, unwritten, oinfo);
1223 80441691 : out_error:
1224 80441691 : if (error)
1225 613 : trace_xfs_rmap_map_error(cur, error, _RET_IP_);
1226 80441691 : return error;
1227 : }
1228 :
1229 : /*
1230 : * Add a reference to an extent in the rmap btree.
1231 : */
1232 : int
1233 3341842 : xfs_rmap_alloc(
1234 : struct xfs_trans *tp,
1235 : struct xfs_buf *agbp,
1236 : struct xfs_perag *pag,
1237 : xfs_agblock_t bno,
1238 : xfs_extlen_t len,
1239 : const struct xfs_owner_info *oinfo)
1240 : {
1241 3341842 : struct xfs_mount *mp = tp->t_mountp;
1242 3341842 : struct xfs_btree_cur *cur;
1243 3341842 : int error;
1244 :
1245 3341842 : if (!xfs_has_rmapbt(mp))
1246 : return 0;
1247 :
1248 3340811 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
1249 3340798 : xfs_rmap_update_hook(tp, pag, NULL, XFS_RMAP_MAP, bno, len, false,
1250 : oinfo);
1251 3340805 : error = xfs_rmap_map(cur, bno, len, false, oinfo);
1252 :
1253 3340806 : xfs_btree_del_cursor(cur, error);
1254 3340806 : return error;
1255 : }
1256 :
1257 : #define RMAP_LEFT_CONTIG (1 << 0)
1258 : #define RMAP_RIGHT_CONTIG (1 << 1)
1259 : #define RMAP_LEFT_FILLING (1 << 2)
1260 : #define RMAP_RIGHT_FILLING (1 << 3)
1261 : #define RMAP_LEFT_VALID (1 << 6)
1262 : #define RMAP_RIGHT_VALID (1 << 7)
1263 :
1264 : #define LEFT r[0]
1265 : #define RIGHT r[1]
1266 : #define PREV r[2]
1267 : #define NEW r[3]
1268 :
1269 : /*
1270 : * Convert an unwritten extent to a real extent or vice versa.
1271 : * Does not handle overlapping extents.
1272 : */
1273 : STATIC int
1274 8923269 : xfs_rmap_convert(
1275 : struct xfs_btree_cur *cur,
1276 : xfs_agblock_t bno,
1277 : xfs_extlen_t len,
1278 : bool unwritten,
1279 : const struct xfs_owner_info *oinfo)
1280 : {
1281 8923269 : struct xfs_mount *mp = cur->bc_mp;
1282 8923269 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1283 : /* left is 0, right is 1, */
1284 : /* prev is 2, new is 3 */
1285 8923269 : uint64_t owner;
1286 8923269 : uint64_t offset;
1287 8923269 : uint64_t new_endoff;
1288 8923269 : unsigned int oldext;
1289 8923269 : unsigned int newext;
1290 8923269 : unsigned int flags = 0;
1291 8923269 : int i;
1292 8923269 : int state = 0;
1293 8923269 : int error;
1294 :
1295 8923269 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1296 8923269 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1297 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1298 8923269 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1299 8923269 : new_endoff = offset + len;
1300 8923269 : trace_xfs_rmap_convert(cur, bno, len, unwritten, oinfo);
1301 :
1302 : /*
1303 : * For the initial lookup, look for an exact match or the left-adjacent
1304 : * record for our insertion point. This will also give us the record for
1305 : * start block contiguity tests.
1306 : */
1307 8923255 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, &PREV, &i);
1308 8923214 : if (error)
1309 0 : goto done;
1310 8923214 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1311 0 : xfs_btree_mark_sick(cur);
1312 0 : error = -EFSCORRUPTED;
1313 0 : goto done;
1314 : }
1315 :
1316 8923214 : trace_xfs_rmap_lookup_le_range_result(cur, PREV.rm_startblock,
1317 : PREV.rm_blockcount, PREV.rm_owner, PREV.rm_offset,
1318 : PREV.rm_flags);
1319 :
1320 8923189 : ASSERT(PREV.rm_offset <= offset);
1321 8923189 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1322 8923189 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1323 8923189 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1324 :
1325 : /*
1326 : * Set flags determining what part of the previous oldext allocation
1327 : * extent is being replaced by a newext allocation.
1328 : */
1329 8923189 : if (PREV.rm_offset == offset)
1330 8518747 : state |= RMAP_LEFT_FILLING;
1331 8923189 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1332 8303095 : state |= RMAP_RIGHT_FILLING;
1333 :
1334 : /*
1335 : * Decrement the cursor to see if we have a left-adjacent record to our
1336 : * insertion point. This will give us the record for end block
1337 : * contiguity tests.
1338 : */
1339 8923189 : error = xfs_btree_decrement(cur, 0, &i);
1340 8923237 : if (error)
1341 0 : goto done;
1342 8923237 : if (i) {
1343 8923270 : state |= RMAP_LEFT_VALID;
1344 8923270 : error = xfs_rmap_get_rec(cur, &LEFT, &i);
1345 8923250 : if (error)
1346 0 : goto done;
1347 8923250 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1348 0 : xfs_btree_mark_sick(cur);
1349 0 : error = -EFSCORRUPTED;
1350 0 : goto done;
1351 : }
1352 8923250 : if (XFS_IS_CORRUPT(mp,
1353 : LEFT.rm_startblock + LEFT.rm_blockcount >
1354 : bno)) {
1355 0 : xfs_btree_mark_sick(cur);
1356 0 : error = -EFSCORRUPTED;
1357 0 : goto done;
1358 : }
1359 8923250 : trace_xfs_rmap_find_left_neighbor_result(cur,
1360 : LEFT.rm_startblock, LEFT.rm_blockcount,
1361 : LEFT.rm_owner, LEFT.rm_offset, LEFT.rm_flags);
1362 8923255 : if (LEFT.rm_startblock + LEFT.rm_blockcount == bno &&
1363 8288643 : LEFT.rm_offset + LEFT.rm_blockcount == offset &&
1364 378600 : xfs_rmap_is_mergeable(&LEFT, owner, newext))
1365 370593 : state |= RMAP_LEFT_CONTIG;
1366 : }
1367 :
1368 : /*
1369 : * Increment the cursor to see if we have a right-adjacent record to our
1370 : * insertion point. This will give us the record for end block
1371 : * contiguity tests.
1372 : */
1373 8923222 : error = xfs_btree_increment(cur, 0, &i);
1374 8923264 : if (error)
1375 0 : goto done;
1376 8923264 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1377 0 : xfs_btree_mark_sick(cur);
1378 0 : error = -EFSCORRUPTED;
1379 0 : goto done;
1380 : }
1381 8923264 : error = xfs_btree_increment(cur, 0, &i);
1382 8923260 : if (error)
1383 0 : goto done;
1384 8923260 : if (i) {
1385 8364834 : state |= RMAP_RIGHT_VALID;
1386 8364834 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1387 8364837 : if (error)
1388 0 : goto done;
1389 8364837 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1390 0 : xfs_btree_mark_sick(cur);
1391 0 : error = -EFSCORRUPTED;
1392 0 : goto done;
1393 : }
1394 8364837 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1395 0 : xfs_btree_mark_sick(cur);
1396 0 : error = -EFSCORRUPTED;
1397 0 : goto done;
1398 : }
1399 8364837 : trace_xfs_rmap_find_right_neighbor_result(cur,
1400 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1401 : RIGHT.rm_owner, RIGHT.rm_offset,
1402 : RIGHT.rm_flags);
1403 8364838 : if (bno + len == RIGHT.rm_startblock &&
1404 5250858 : offset + len == RIGHT.rm_offset &&
1405 103771 : xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1406 99082 : state |= RMAP_RIGHT_CONTIG;
1407 : }
1408 :
1409 : /* check that left + prev + right is not too long */
1410 8923264 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1411 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1412 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1413 28760 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1414 28760 : (unsigned long)LEFT.rm_blockcount + len +
1415 28760 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1416 0 : state &= ~RMAP_RIGHT_CONTIG;
1417 :
1418 8923264 : trace_xfs_rmap_convert_state(cur, state, _RET_IP_);
1419 :
1420 : /* reset the cursor back to PREV */
1421 8923256 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, NULL, &i);
1422 8923258 : if (error)
1423 0 : goto done;
1424 8923258 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1425 0 : xfs_btree_mark_sick(cur);
1426 0 : error = -EFSCORRUPTED;
1427 0 : goto done;
1428 : }
1429 :
1430 : /*
1431 : * Switch out based on the FILLING and CONTIG state bits.
1432 : */
1433 8923258 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1434 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1435 28760 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1436 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1437 : /*
1438 : * Setting all of a previous oldext extent to newext.
1439 : * The left and right neighbors are both contiguous with new.
1440 : */
1441 28760 : error = xfs_btree_increment(cur, 0, &i);
1442 28760 : if (error)
1443 0 : goto done;
1444 28760 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1445 0 : xfs_btree_mark_sick(cur);
1446 0 : error = -EFSCORRUPTED;
1447 0 : goto done;
1448 : }
1449 28760 : trace_xfs_rmap_delete(cur, RIGHT.rm_startblock,
1450 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1451 : RIGHT.rm_offset, RIGHT.rm_flags);
1452 28760 : error = xfs_btree_delete(cur, &i);
1453 28760 : if (error)
1454 0 : goto done;
1455 28760 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1456 0 : xfs_btree_mark_sick(cur);
1457 0 : error = -EFSCORRUPTED;
1458 0 : goto done;
1459 : }
1460 28760 : error = xfs_btree_decrement(cur, 0, &i);
1461 28760 : if (error)
1462 0 : goto done;
1463 28760 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1464 0 : xfs_btree_mark_sick(cur);
1465 0 : error = -EFSCORRUPTED;
1466 0 : goto done;
1467 : }
1468 28760 : trace_xfs_rmap_delete(cur, PREV.rm_startblock,
1469 : PREV.rm_blockcount, PREV.rm_owner,
1470 : PREV.rm_offset, PREV.rm_flags);
1471 28760 : error = xfs_btree_delete(cur, &i);
1472 28760 : if (error)
1473 0 : goto done;
1474 28760 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1475 0 : xfs_btree_mark_sick(cur);
1476 0 : error = -EFSCORRUPTED;
1477 0 : goto done;
1478 : }
1479 28760 : error = xfs_btree_decrement(cur, 0, &i);
1480 28760 : if (error)
1481 0 : goto done;
1482 28760 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1483 0 : xfs_btree_mark_sick(cur);
1484 0 : error = -EFSCORRUPTED;
1485 0 : goto done;
1486 : }
1487 28760 : NEW = LEFT;
1488 28760 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1489 28760 : error = xfs_rmap_update(cur, &NEW);
1490 28760 : if (error)
1491 0 : goto done;
1492 : break;
1493 :
1494 233132 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1495 : /*
1496 : * Setting all of a previous oldext extent to newext.
1497 : * The left neighbor is contiguous, the right is not.
1498 : */
1499 233132 : trace_xfs_rmap_delete(cur, PREV.rm_startblock,
1500 : PREV.rm_blockcount, PREV.rm_owner,
1501 : PREV.rm_offset, PREV.rm_flags);
1502 233132 : error = xfs_btree_delete(cur, &i);
1503 233132 : if (error)
1504 0 : goto done;
1505 233132 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1506 0 : xfs_btree_mark_sick(cur);
1507 0 : error = -EFSCORRUPTED;
1508 0 : goto done;
1509 : }
1510 233132 : error = xfs_btree_decrement(cur, 0, &i);
1511 233132 : if (error)
1512 0 : goto done;
1513 233132 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1514 0 : xfs_btree_mark_sick(cur);
1515 0 : error = -EFSCORRUPTED;
1516 0 : goto done;
1517 : }
1518 233132 : NEW = LEFT;
1519 233132 : NEW.rm_blockcount += PREV.rm_blockcount;
1520 233132 : error = xfs_rmap_update(cur, &NEW);
1521 233132 : if (error)
1522 0 : goto done;
1523 : break;
1524 :
1525 26975 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1526 : /*
1527 : * Setting all of a previous oldext extent to newext.
1528 : * The right neighbor is contiguous, the left is not.
1529 : */
1530 26975 : error = xfs_btree_increment(cur, 0, &i);
1531 26975 : if (error)
1532 0 : goto done;
1533 26975 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1534 0 : xfs_btree_mark_sick(cur);
1535 0 : error = -EFSCORRUPTED;
1536 0 : goto done;
1537 : }
1538 26975 : trace_xfs_rmap_delete(cur, RIGHT.rm_startblock,
1539 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1540 : RIGHT.rm_offset, RIGHT.rm_flags);
1541 26975 : error = xfs_btree_delete(cur, &i);
1542 26975 : if (error)
1543 0 : goto done;
1544 26975 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1545 0 : xfs_btree_mark_sick(cur);
1546 0 : error = -EFSCORRUPTED;
1547 0 : goto done;
1548 : }
1549 26975 : error = xfs_btree_decrement(cur, 0, &i);
1550 26975 : if (error)
1551 0 : goto done;
1552 26975 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1553 0 : xfs_btree_mark_sick(cur);
1554 0 : error = -EFSCORRUPTED;
1555 0 : goto done;
1556 : }
1557 26975 : NEW = PREV;
1558 26975 : NEW.rm_blockcount = len + RIGHT.rm_blockcount;
1559 26975 : NEW.rm_flags = newext;
1560 26975 : error = xfs_rmap_update(cur, &NEW);
1561 26975 : if (error)
1562 0 : goto done;
1563 : break;
1564 :
1565 7871724 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1566 : /*
1567 : * Setting all of a previous oldext extent to newext.
1568 : * Neither the left nor right neighbors are contiguous with
1569 : * the new one.
1570 : */
1571 7871724 : NEW = PREV;
1572 7871724 : NEW.rm_flags = newext;
1573 7871724 : error = xfs_rmap_update(cur, &NEW);
1574 7871718 : if (error)
1575 0 : goto done;
1576 : break;
1577 :
1578 108701 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1579 : /*
1580 : * Setting the first part of a previous oldext extent to newext.
1581 : * The left neighbor is contiguous.
1582 : */
1583 108701 : NEW = PREV;
1584 108701 : NEW.rm_offset += len;
1585 108701 : NEW.rm_startblock += len;
1586 108701 : NEW.rm_blockcount -= len;
1587 108701 : error = xfs_rmap_update(cur, &NEW);
1588 108701 : if (error)
1589 0 : goto done;
1590 108701 : error = xfs_btree_decrement(cur, 0, &i);
1591 108701 : if (error)
1592 0 : goto done;
1593 108701 : NEW = LEFT;
1594 108701 : NEW.rm_blockcount += len;
1595 108701 : error = xfs_rmap_update(cur, &NEW);
1596 108701 : if (error)
1597 0 : goto done;
1598 : break;
1599 :
1600 249498 : case RMAP_LEFT_FILLING:
1601 : /*
1602 : * Setting the first part of a previous oldext extent to newext.
1603 : * The left neighbor is not contiguous.
1604 : */
1605 249498 : NEW = PREV;
1606 249498 : NEW.rm_startblock += len;
1607 249498 : NEW.rm_offset += len;
1608 249498 : NEW.rm_blockcount -= len;
1609 249498 : error = xfs_rmap_update(cur, &NEW);
1610 249498 : if (error)
1611 0 : goto done;
1612 249498 : NEW.rm_startblock = bno;
1613 249498 : NEW.rm_owner = owner;
1614 249498 : NEW.rm_offset = offset;
1615 249498 : NEW.rm_blockcount = len;
1616 249498 : NEW.rm_flags = newext;
1617 249498 : cur->bc_rec.r = NEW;
1618 249498 : trace_xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1619 249498 : error = xfs_btree_insert(cur, &i);
1620 249498 : if (error)
1621 1 : goto done;
1622 249497 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1623 0 : xfs_btree_mark_sick(cur);
1624 0 : error = -EFSCORRUPTED;
1625 0 : goto done;
1626 : }
1627 : break;
1628 :
1629 43347 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1630 : /*
1631 : * Setting the last part of a previous oldext extent to newext.
1632 : * The right neighbor is contiguous with the new allocation.
1633 : */
1634 43347 : NEW = PREV;
1635 43347 : NEW.rm_blockcount -= len;
1636 43347 : error = xfs_rmap_update(cur, &NEW);
1637 43347 : if (error)
1638 0 : goto done;
1639 43347 : error = xfs_btree_increment(cur, 0, &i);
1640 43347 : if (error)
1641 0 : goto done;
1642 43347 : NEW = RIGHT;
1643 43347 : NEW.rm_offset = offset;
1644 43347 : NEW.rm_startblock = bno;
1645 43347 : NEW.rm_blockcount += len;
1646 43347 : error = xfs_rmap_update(cur, &NEW);
1647 43347 : if (error)
1648 0 : goto done;
1649 : break;
1650 :
1651 99205 : case RMAP_RIGHT_FILLING:
1652 : /*
1653 : * Setting the last part of a previous oldext extent to newext.
1654 : * The right neighbor is not contiguous.
1655 : */
1656 99205 : NEW = PREV;
1657 99205 : NEW.rm_blockcount -= len;
1658 99205 : error = xfs_rmap_update(cur, &NEW);
1659 99205 : if (error)
1660 0 : goto done;
1661 99205 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1662 : oldext, &i);
1663 99205 : if (error)
1664 0 : goto done;
1665 99205 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1666 0 : xfs_btree_mark_sick(cur);
1667 0 : error = -EFSCORRUPTED;
1668 0 : goto done;
1669 : }
1670 99205 : NEW.rm_startblock = bno;
1671 99205 : NEW.rm_owner = owner;
1672 99205 : NEW.rm_offset = offset;
1673 99205 : NEW.rm_blockcount = len;
1674 99205 : NEW.rm_flags = newext;
1675 99205 : cur->bc_rec.r = NEW;
1676 99205 : trace_xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1677 99205 : error = xfs_btree_insert(cur, &i);
1678 99205 : if (error)
1679 0 : goto done;
1680 99205 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1681 0 : xfs_btree_mark_sick(cur);
1682 0 : error = -EFSCORRUPTED;
1683 0 : goto done;
1684 : }
1685 : break;
1686 :
1687 261916 : case 0:
1688 : /*
1689 : * Setting the middle part of a previous oldext extent to
1690 : * newext. Contiguity is impossible here.
1691 : * One extent becomes three extents.
1692 : */
1693 : /* new right extent - oldext */
1694 261916 : NEW.rm_startblock = bno + len;
1695 261916 : NEW.rm_owner = owner;
1696 261916 : NEW.rm_offset = new_endoff;
1697 261916 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1698 : new_endoff;
1699 261916 : NEW.rm_flags = PREV.rm_flags;
1700 261916 : error = xfs_rmap_update(cur, &NEW);
1701 261916 : if (error)
1702 0 : goto done;
1703 : /* new left extent - oldext */
1704 261916 : NEW = PREV;
1705 261916 : NEW.rm_blockcount = offset - PREV.rm_offset;
1706 261916 : cur->bc_rec.r = NEW;
1707 261916 : trace_xfs_rmap_insert(cur, NEW.rm_startblock,
1708 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
1709 : NEW.rm_flags);
1710 261916 : error = xfs_btree_insert(cur, &i);
1711 261916 : if (error)
1712 0 : goto done;
1713 261916 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1714 0 : xfs_btree_mark_sick(cur);
1715 0 : error = -EFSCORRUPTED;
1716 0 : goto done;
1717 : }
1718 : /*
1719 : * Reset the cursor to the position of the new extent
1720 : * we are about to insert as we can't trust it after
1721 : * the previous insert.
1722 : */
1723 261916 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1724 : oldext, &i);
1725 261916 : if (error)
1726 0 : goto done;
1727 261916 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1728 0 : xfs_btree_mark_sick(cur);
1729 0 : error = -EFSCORRUPTED;
1730 0 : goto done;
1731 : }
1732 : /* new middle extent - newext */
1733 261916 : cur->bc_rec.r.rm_flags &= ~XFS_RMAP_UNWRITTEN;
1734 261916 : cur->bc_rec.r.rm_flags |= newext;
1735 261916 : trace_xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1736 261916 : error = xfs_btree_insert(cur, &i);
1737 261916 : if (error)
1738 0 : goto done;
1739 261916 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1740 0 : xfs_btree_mark_sick(cur);
1741 0 : error = -EFSCORRUPTED;
1742 0 : goto done;
1743 : }
1744 : break;
1745 :
1746 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1747 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1748 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1749 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1750 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1751 : case RMAP_LEFT_CONTIG:
1752 : case RMAP_RIGHT_CONTIG:
1753 : /*
1754 : * These cases are all impossible.
1755 : */
1756 0 : ASSERT(0);
1757 : }
1758 :
1759 8923251 : trace_xfs_rmap_convert_done(cur, bno, len, unwritten, oinfo);
1760 8923242 : done:
1761 8923242 : if (error)
1762 1 : trace_xfs_rmap_convert_error(cur, error, _RET_IP_);
1763 8923242 : return error;
1764 : }
1765 :
1766 : /*
1767 : * Convert an unwritten extent to a real extent or vice versa. If there is no
1768 : * possibility of overlapping extents, delegate to the simpler convert
1769 : * function.
1770 : */
1771 : STATIC int
1772 27654567 : xfs_rmap_convert_shared(
1773 : struct xfs_btree_cur *cur,
1774 : xfs_agblock_t bno,
1775 : xfs_extlen_t len,
1776 : bool unwritten,
1777 : const struct xfs_owner_info *oinfo)
1778 : {
1779 27654567 : struct xfs_mount *mp = cur->bc_mp;
1780 27654567 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1781 : /* left is 0, right is 1, */
1782 : /* prev is 2, new is 3 */
1783 27654567 : uint64_t owner;
1784 27654567 : uint64_t offset;
1785 27654567 : uint64_t new_endoff;
1786 27654567 : unsigned int oldext;
1787 27654567 : unsigned int newext;
1788 27654567 : unsigned int flags = 0;
1789 27654567 : int i;
1790 27654567 : int state = 0;
1791 27654567 : int error;
1792 :
1793 27654567 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1794 27654567 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1795 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1796 27654567 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1797 27654567 : new_endoff = offset + len;
1798 27654567 : trace_xfs_rmap_convert(cur, bno, len, unwritten, oinfo);
1799 :
1800 : /*
1801 : * For the initial lookup, look for and exact match or the left-adjacent
1802 : * record for our insertion point. This will also give us the record for
1803 : * start block contiguity tests.
1804 : */
1805 27654569 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, oldext,
1806 : &PREV, &i);
1807 27654567 : if (error)
1808 1 : goto done;
1809 27654566 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1810 0 : xfs_btree_mark_sick(cur);
1811 0 : error = -EFSCORRUPTED;
1812 0 : goto done;
1813 : }
1814 :
1815 27654566 : ASSERT(PREV.rm_offset <= offset);
1816 27654566 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1817 27654566 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1818 27654566 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1819 :
1820 : /*
1821 : * Set flags determining what part of the previous oldext allocation
1822 : * extent is being replaced by a newext allocation.
1823 : */
1824 27654566 : if (PREV.rm_offset == offset)
1825 27058595 : state |= RMAP_LEFT_FILLING;
1826 27654566 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1827 24745259 : state |= RMAP_RIGHT_FILLING;
1828 :
1829 : /* Is there a left record that abuts our range? */
1830 27654566 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, newext,
1831 : &LEFT, &i);
1832 27654568 : if (error)
1833 0 : goto done;
1834 27654568 : if (i) {
1835 412951 : state |= RMAP_LEFT_VALID;
1836 412951 : if (XFS_IS_CORRUPT(mp,
1837 : LEFT.rm_startblock + LEFT.rm_blockcount >
1838 : bno)) {
1839 0 : xfs_btree_mark_sick(cur);
1840 0 : error = -EFSCORRUPTED;
1841 0 : goto done;
1842 : }
1843 412951 : if (xfs_rmap_is_mergeable(&LEFT, owner, newext))
1844 412949 : state |= RMAP_LEFT_CONTIG;
1845 : }
1846 :
1847 : /* Is there a right record that abuts our range? */
1848 27654568 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
1849 : newext, &i);
1850 27654565 : if (error)
1851 0 : goto done;
1852 27654565 : if (i) {
1853 210243 : state |= RMAP_RIGHT_VALID;
1854 210243 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1855 210243 : if (error)
1856 0 : goto done;
1857 210243 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1858 0 : xfs_btree_mark_sick(cur);
1859 0 : error = -EFSCORRUPTED;
1860 0 : goto done;
1861 : }
1862 210243 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1863 0 : xfs_btree_mark_sick(cur);
1864 0 : error = -EFSCORRUPTED;
1865 0 : goto done;
1866 : }
1867 210243 : trace_xfs_rmap_find_right_neighbor_result(cur,
1868 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1869 : RIGHT.rm_owner, RIGHT.rm_offset,
1870 : RIGHT.rm_flags);
1871 210243 : if (xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1872 210243 : state |= RMAP_RIGHT_CONTIG;
1873 : }
1874 :
1875 : /* check that left + prev + right is not too long */
1876 27654565 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1877 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1878 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1879 47890 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1880 47890 : (unsigned long)LEFT.rm_blockcount + len +
1881 47890 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1882 0 : state &= ~RMAP_RIGHT_CONTIG;
1883 :
1884 27654565 : trace_xfs_rmap_convert_state(cur, state, _RET_IP_);
1885 : /*
1886 : * Switch out based on the FILLING and CONTIG state bits.
1887 : */
1888 27654568 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1889 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1890 47890 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1891 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1892 : /*
1893 : * Setting all of a previous oldext extent to newext.
1894 : * The left and right neighbors are both contiguous with new.
1895 : */
1896 47890 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1897 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1898 : RIGHT.rm_offset, RIGHT.rm_flags);
1899 47890 : if (error)
1900 0 : goto done;
1901 47890 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1902 : PREV.rm_blockcount, PREV.rm_owner,
1903 : PREV.rm_offset, PREV.rm_flags);
1904 47890 : if (error)
1905 0 : goto done;
1906 47890 : NEW = LEFT;
1907 47890 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1908 : NEW.rm_blockcount, NEW.rm_owner,
1909 : NEW.rm_offset, NEW.rm_flags, &i);
1910 47890 : if (error)
1911 0 : goto done;
1912 47890 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1913 0 : xfs_btree_mark_sick(cur);
1914 0 : error = -EFSCORRUPTED;
1915 0 : goto done;
1916 : }
1917 47890 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1918 47890 : error = xfs_rmap_update(cur, &NEW);
1919 47890 : if (error)
1920 0 : goto done;
1921 : break;
1922 :
1923 286274 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1924 : /*
1925 : * Setting all of a previous oldext extent to newext.
1926 : * The left neighbor is contiguous, the right is not.
1927 : */
1928 286274 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1929 : PREV.rm_blockcount, PREV.rm_owner,
1930 : PREV.rm_offset, PREV.rm_flags);
1931 286274 : if (error)
1932 0 : goto done;
1933 286274 : NEW = LEFT;
1934 286274 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1935 : NEW.rm_blockcount, NEW.rm_owner,
1936 : NEW.rm_offset, NEW.rm_flags, &i);
1937 286274 : if (error)
1938 0 : goto done;
1939 286274 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1940 0 : xfs_btree_mark_sick(cur);
1941 0 : error = -EFSCORRUPTED;
1942 0 : goto done;
1943 : }
1944 286274 : NEW.rm_blockcount += PREV.rm_blockcount;
1945 286274 : error = xfs_rmap_update(cur, &NEW);
1946 286274 : if (error)
1947 0 : goto done;
1948 : break;
1949 :
1950 139584 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1951 : /*
1952 : * Setting all of a previous oldext extent to newext.
1953 : * The right neighbor is contiguous, the left is not.
1954 : */
1955 139584 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1956 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1957 : RIGHT.rm_offset, RIGHT.rm_flags);
1958 139584 : if (error)
1959 0 : goto done;
1960 139584 : NEW = PREV;
1961 139584 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1962 : NEW.rm_blockcount, NEW.rm_owner,
1963 : NEW.rm_offset, NEW.rm_flags, &i);
1964 139584 : if (error)
1965 0 : goto done;
1966 139584 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1967 0 : xfs_btree_mark_sick(cur);
1968 0 : error = -EFSCORRUPTED;
1969 0 : goto done;
1970 : }
1971 139584 : NEW.rm_blockcount += RIGHT.rm_blockcount;
1972 139584 : NEW.rm_flags = RIGHT.rm_flags;
1973 139584 : error = xfs_rmap_update(cur, &NEW);
1974 139584 : if (error)
1975 0 : goto done;
1976 : break;
1977 :
1978 23942958 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1979 : /*
1980 : * Setting all of a previous oldext extent to newext.
1981 : * Neither the left nor right neighbors are contiguous with
1982 : * the new one.
1983 : */
1984 23942958 : NEW = PREV;
1985 23942958 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1986 : NEW.rm_blockcount, NEW.rm_owner,
1987 : NEW.rm_offset, NEW.rm_flags, &i);
1988 23942958 : if (error)
1989 0 : goto done;
1990 23942958 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1991 0 : xfs_btree_mark_sick(cur);
1992 0 : error = -EFSCORRUPTED;
1993 0 : goto done;
1994 : }
1995 23942958 : NEW.rm_flags = newext;
1996 23942958 : error = xfs_rmap_update(cur, &NEW);
1997 23942958 : if (error)
1998 0 : goto done;
1999 : break;
2000 :
2001 78785 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
2002 : /*
2003 : * Setting the first part of a previous oldext extent to newext.
2004 : * The left neighbor is contiguous.
2005 : */
2006 78785 : NEW = PREV;
2007 78785 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
2008 : NEW.rm_blockcount, NEW.rm_owner,
2009 : NEW.rm_offset, NEW.rm_flags);
2010 78785 : if (error)
2011 0 : goto done;
2012 78785 : NEW.rm_offset += len;
2013 78785 : NEW.rm_startblock += len;
2014 78785 : NEW.rm_blockcount -= len;
2015 78785 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2016 : NEW.rm_blockcount, NEW.rm_owner,
2017 : NEW.rm_offset, NEW.rm_flags);
2018 78785 : if (error)
2019 0 : goto done;
2020 78785 : NEW = LEFT;
2021 78785 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2022 : NEW.rm_blockcount, NEW.rm_owner,
2023 : NEW.rm_offset, NEW.rm_flags, &i);
2024 78785 : if (error)
2025 0 : goto done;
2026 78785 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2027 0 : xfs_btree_mark_sick(cur);
2028 0 : error = -EFSCORRUPTED;
2029 0 : goto done;
2030 : }
2031 78785 : NEW.rm_blockcount += len;
2032 78785 : error = xfs_rmap_update(cur, &NEW);
2033 78785 : if (error)
2034 0 : goto done;
2035 : break;
2036 :
2037 2563128 : case RMAP_LEFT_FILLING:
2038 : /*
2039 : * Setting the first part of a previous oldext extent to newext.
2040 : * The left neighbor is not contiguous.
2041 : */
2042 2563128 : NEW = PREV;
2043 2563128 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
2044 : NEW.rm_blockcount, NEW.rm_owner,
2045 : NEW.rm_offset, NEW.rm_flags);
2046 2563128 : if (error)
2047 0 : goto done;
2048 2563128 : NEW.rm_offset += len;
2049 2563128 : NEW.rm_startblock += len;
2050 2563128 : NEW.rm_blockcount -= len;
2051 2563128 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2052 : NEW.rm_blockcount, NEW.rm_owner,
2053 : NEW.rm_offset, NEW.rm_flags);
2054 2563128 : if (error)
2055 0 : goto done;
2056 2563128 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
2057 2563128 : if (error)
2058 2 : goto done;
2059 : break;
2060 :
2061 22769 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
2062 : /*
2063 : * Setting the last part of a previous oldext extent to newext.
2064 : * The right neighbor is contiguous with the new allocation.
2065 : */
2066 22769 : NEW = PREV;
2067 22769 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2068 : NEW.rm_blockcount, NEW.rm_owner,
2069 : NEW.rm_offset, NEW.rm_flags, &i);
2070 22769 : if (error)
2071 0 : goto done;
2072 22769 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2073 0 : xfs_btree_mark_sick(cur);
2074 0 : error = -EFSCORRUPTED;
2075 0 : goto done;
2076 : }
2077 22769 : NEW.rm_blockcount = offset - NEW.rm_offset;
2078 22769 : error = xfs_rmap_update(cur, &NEW);
2079 22769 : if (error)
2080 0 : goto done;
2081 22769 : NEW = RIGHT;
2082 22769 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
2083 : NEW.rm_blockcount, NEW.rm_owner,
2084 : NEW.rm_offset, NEW.rm_flags);
2085 22769 : if (error)
2086 0 : goto done;
2087 22769 : NEW.rm_offset = offset;
2088 22769 : NEW.rm_startblock = bno;
2089 22769 : NEW.rm_blockcount += len;
2090 22769 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2091 : NEW.rm_blockcount, NEW.rm_owner,
2092 : NEW.rm_offset, NEW.rm_flags);
2093 22769 : if (error)
2094 0 : goto done;
2095 : break;
2096 :
2097 305784 : case RMAP_RIGHT_FILLING:
2098 : /*
2099 : * Setting the last part of a previous oldext extent to newext.
2100 : * The right neighbor is not contiguous.
2101 : */
2102 305784 : NEW = PREV;
2103 305784 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2104 : NEW.rm_blockcount, NEW.rm_owner,
2105 : NEW.rm_offset, NEW.rm_flags, &i);
2106 305784 : if (error)
2107 0 : goto done;
2108 305784 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2109 0 : xfs_btree_mark_sick(cur);
2110 0 : error = -EFSCORRUPTED;
2111 0 : goto done;
2112 : }
2113 305784 : NEW.rm_blockcount -= len;
2114 305784 : error = xfs_rmap_update(cur, &NEW);
2115 305784 : if (error)
2116 0 : goto done;
2117 305784 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
2118 305784 : if (error)
2119 0 : goto done;
2120 : break;
2121 :
2122 267396 : case 0:
2123 : /*
2124 : * Setting the middle part of a previous oldext extent to
2125 : * newext. Contiguity is impossible here.
2126 : * One extent becomes three extents.
2127 : */
2128 : /* new right extent - oldext */
2129 267396 : NEW.rm_startblock = bno + len;
2130 267396 : NEW.rm_owner = owner;
2131 267396 : NEW.rm_offset = new_endoff;
2132 267396 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
2133 : new_endoff;
2134 267396 : NEW.rm_flags = PREV.rm_flags;
2135 267396 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2136 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
2137 : NEW.rm_flags);
2138 267396 : if (error)
2139 0 : goto done;
2140 : /* new left extent - oldext */
2141 267396 : NEW = PREV;
2142 267396 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2143 : NEW.rm_blockcount, NEW.rm_owner,
2144 : NEW.rm_offset, NEW.rm_flags, &i);
2145 267396 : if (error)
2146 0 : goto done;
2147 267396 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2148 0 : xfs_btree_mark_sick(cur);
2149 0 : error = -EFSCORRUPTED;
2150 0 : goto done;
2151 : }
2152 267396 : NEW.rm_blockcount = offset - NEW.rm_offset;
2153 267396 : error = xfs_rmap_update(cur, &NEW);
2154 267396 : if (error)
2155 0 : goto done;
2156 : /* new middle extent - newext */
2157 267396 : NEW.rm_startblock = bno;
2158 267396 : NEW.rm_blockcount = len;
2159 267396 : NEW.rm_owner = owner;
2160 267396 : NEW.rm_offset = offset;
2161 267396 : NEW.rm_flags = newext;
2162 267396 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2163 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
2164 : NEW.rm_flags);
2165 267396 : if (error)
2166 0 : goto done;
2167 : break;
2168 :
2169 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2170 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2171 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
2172 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
2173 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2174 : case RMAP_LEFT_CONTIG:
2175 : case RMAP_RIGHT_CONTIG:
2176 : /*
2177 : * These cases are all impossible.
2178 : */
2179 0 : ASSERT(0);
2180 : }
2181 :
2182 27654566 : trace_xfs_rmap_convert_done(cur, bno, len, unwritten, oinfo);
2183 27654569 : done:
2184 27654569 : if (error)
2185 3 : trace_xfs_rmap_convert_error(cur, error, _RET_IP_);
2186 27654569 : return error;
2187 : }
2188 :
2189 : #undef NEW
2190 : #undef LEFT
2191 : #undef RIGHT
2192 : #undef PREV
2193 :
2194 : /*
2195 : * Find an extent in the rmap btree and unmap it. For rmap extent types that
2196 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2197 : * that the prev/next records in the btree might belong to another owner.
2198 : * Therefore we must use delete+insert to alter any of the key fields.
2199 : *
2200 : * For every other situation there can only be one owner for a given extent,
2201 : * so we can call the regular _free function.
2202 : */
2203 : STATIC int
2204 84108799 : xfs_rmap_unmap_shared(
2205 : struct xfs_btree_cur *cur,
2206 : xfs_agblock_t bno,
2207 : xfs_extlen_t len,
2208 : bool unwritten,
2209 : const struct xfs_owner_info *oinfo)
2210 : {
2211 84108799 : struct xfs_mount *mp = cur->bc_mp;
2212 84108799 : struct xfs_rmap_irec ltrec;
2213 84108799 : uint64_t ltoff;
2214 84108799 : int error = 0;
2215 84108799 : int i;
2216 84108799 : uint64_t owner;
2217 84108799 : uint64_t offset;
2218 84108799 : unsigned int flags;
2219 :
2220 84108799 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2221 84108799 : if (unwritten)
2222 13407647 : flags |= XFS_RMAP_UNWRITTEN;
2223 84108799 : trace_xfs_rmap_unmap(cur, bno, len, unwritten, oinfo);
2224 :
2225 : /*
2226 : * We should always have a left record because there's a static record
2227 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
2228 : * will not ever be removed from the tree.
2229 : */
2230 84108852 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, flags,
2231 : <rec, &i);
2232 84108790 : if (error)
2233 283 : goto out_error;
2234 84108507 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2235 0 : xfs_btree_mark_sick(cur);
2236 0 : error = -EFSCORRUPTED;
2237 0 : goto out_error;
2238 : }
2239 84108507 : ltoff = ltrec.rm_offset;
2240 :
2241 : /* Make sure the extent we found covers the entire freeing range. */
2242 84108507 : if (XFS_IS_CORRUPT(mp,
2243 : ltrec.rm_startblock > bno ||
2244 : ltrec.rm_startblock + ltrec.rm_blockcount <
2245 : bno + len)) {
2246 0 : xfs_btree_mark_sick(cur);
2247 0 : error = -EFSCORRUPTED;
2248 0 : goto out_error;
2249 : }
2250 :
2251 : /* Make sure the owner matches what we expect to find in the tree. */
2252 84108507 : if (XFS_IS_CORRUPT(mp, owner != ltrec.rm_owner)) {
2253 0 : xfs_btree_mark_sick(cur);
2254 0 : error = -EFSCORRUPTED;
2255 0 : goto out_error;
2256 : }
2257 :
2258 : /* Make sure the unwritten flag matches. */
2259 84108507 : if (XFS_IS_CORRUPT(mp,
2260 : (flags & XFS_RMAP_UNWRITTEN) !=
2261 : (ltrec.rm_flags & XFS_RMAP_UNWRITTEN))) {
2262 0 : xfs_btree_mark_sick(cur);
2263 0 : error = -EFSCORRUPTED;
2264 0 : goto out_error;
2265 : }
2266 :
2267 : /* Check the offset. */
2268 84108507 : if (XFS_IS_CORRUPT(mp, ltrec.rm_offset > offset)) {
2269 0 : xfs_btree_mark_sick(cur);
2270 0 : error = -EFSCORRUPTED;
2271 0 : goto out_error;
2272 : }
2273 84108507 : if (XFS_IS_CORRUPT(mp, offset > ltoff + ltrec.rm_blockcount)) {
2274 0 : xfs_btree_mark_sick(cur);
2275 0 : error = -EFSCORRUPTED;
2276 0 : goto out_error;
2277 : }
2278 :
2279 84108507 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
2280 : /* Exact match, simply remove the record from rmap tree. */
2281 69478308 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2282 : ltrec.rm_blockcount, ltrec.rm_owner,
2283 : ltrec.rm_offset, ltrec.rm_flags);
2284 69478369 : if (error)
2285 1 : goto out_error;
2286 14630199 : } else if (ltrec.rm_startblock == bno) {
2287 : /*
2288 : * Overlap left hand side of extent: move the start, trim the
2289 : * length and update the current record.
2290 : *
2291 : * ltbno ltlen
2292 : * Orig: |oooooooooooooooooooo|
2293 : * Freeing: |fffffffff|
2294 : * Result: |rrrrrrrrrr|
2295 : * bno len
2296 : */
2297 :
2298 : /* Delete prev rmap. */
2299 5090717 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2300 : ltrec.rm_blockcount, ltrec.rm_owner,
2301 : ltrec.rm_offset, ltrec.rm_flags);
2302 5090720 : if (error)
2303 0 : goto out_error;
2304 :
2305 : /* Add an rmap at the new offset. */
2306 5090720 : ltrec.rm_startblock += len;
2307 5090720 : ltrec.rm_blockcount -= len;
2308 5090720 : ltrec.rm_offset += len;
2309 5090720 : error = xfs_rmap_insert(cur, ltrec.rm_startblock,
2310 : ltrec.rm_blockcount, ltrec.rm_owner,
2311 : ltrec.rm_offset, ltrec.rm_flags);
2312 5090720 : if (error)
2313 0 : goto out_error;
2314 9539482 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
2315 : /*
2316 : * Overlap right hand side of extent: trim the length and
2317 : * update the current record.
2318 : *
2319 : * ltbno ltlen
2320 : * Orig: |oooooooooooooooooooo|
2321 : * Freeing: |fffffffff|
2322 : * Result: |rrrrrrrrrr|
2323 : * bno len
2324 : */
2325 3633997 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2326 : ltrec.rm_blockcount, ltrec.rm_owner,
2327 : ltrec.rm_offset, ltrec.rm_flags, &i);
2328 3633998 : if (error)
2329 0 : goto out_error;
2330 3633998 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2331 0 : xfs_btree_mark_sick(cur);
2332 0 : error = -EFSCORRUPTED;
2333 0 : goto out_error;
2334 : }
2335 3633998 : ltrec.rm_blockcount -= len;
2336 3633998 : error = xfs_rmap_update(cur, <rec);
2337 3633996 : if (error)
2338 0 : goto out_error;
2339 : } else {
2340 : /*
2341 : * Overlap middle of extent: trim the length of the existing
2342 : * record to the length of the new left-extent size, increment
2343 : * the insertion position so we can insert a new record
2344 : * containing the remaining right-extent space.
2345 : *
2346 : * ltbno ltlen
2347 : * Orig: |oooooooooooooooooooo|
2348 : * Freeing: |fffffffff|
2349 : * Result: |rrrrr| |rrrr|
2350 : * bno len
2351 : */
2352 5905485 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
2353 :
2354 : /* Shrink the left side of the rmap */
2355 5905485 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2356 : ltrec.rm_blockcount, ltrec.rm_owner,
2357 : ltrec.rm_offset, ltrec.rm_flags, &i);
2358 5905481 : if (error)
2359 0 : goto out_error;
2360 5905481 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2361 0 : xfs_btree_mark_sick(cur);
2362 0 : error = -EFSCORRUPTED;
2363 0 : goto out_error;
2364 : }
2365 5905481 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
2366 5905481 : error = xfs_rmap_update(cur, <rec);
2367 5905483 : if (error)
2368 0 : goto out_error;
2369 :
2370 : /* Add an rmap at the new offset */
2371 5905483 : error = xfs_rmap_insert(cur, bno + len,
2372 5905483 : orig_len - len - ltrec.rm_blockcount,
2373 : ltrec.rm_owner, offset + len,
2374 : ltrec.rm_flags);
2375 5905489 : if (error)
2376 1 : goto out_error;
2377 : }
2378 :
2379 84108572 : trace_xfs_rmap_unmap_done(cur, bno, len, unwritten, oinfo);
2380 84108783 : out_error:
2381 84108783 : if (error)
2382 285 : trace_xfs_rmap_unmap_error(cur, error, _RET_IP_);
2383 84108783 : return error;
2384 : }
2385 :
2386 : /*
2387 : * Find an extent in the rmap btree and map it. For rmap extent types that
2388 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2389 : * that the prev/next records in the btree might belong to another owner.
2390 : * Therefore we must use delete+insert to alter any of the key fields.
2391 : *
2392 : * For every other situation there can only be one owner for a given extent,
2393 : * so we can call the regular _alloc function.
2394 : */
2395 : STATIC int
2396 121445062 : xfs_rmap_map_shared(
2397 : struct xfs_btree_cur *cur,
2398 : xfs_agblock_t bno,
2399 : xfs_extlen_t len,
2400 : bool unwritten,
2401 : const struct xfs_owner_info *oinfo)
2402 : {
2403 121445062 : struct xfs_mount *mp = cur->bc_mp;
2404 121445062 : struct xfs_rmap_irec ltrec;
2405 121445062 : struct xfs_rmap_irec gtrec;
2406 121445062 : int have_gt;
2407 121445062 : int have_lt;
2408 121445062 : int error = 0;
2409 121445062 : int i;
2410 121445062 : uint64_t owner;
2411 121445062 : uint64_t offset;
2412 121445062 : unsigned int flags = 0;
2413 :
2414 121445062 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2415 121445062 : if (unwritten)
2416 34622094 : flags |= XFS_RMAP_UNWRITTEN;
2417 121445062 : trace_xfs_rmap_map(cur, bno, len, unwritten, oinfo);
2418 :
2419 : /* Is there a left record that abuts our range? */
2420 121445257 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, flags,
2421 : <rec, &have_lt);
2422 121445101 : if (error)
2423 202 : goto out_error;
2424 121444899 : if (have_lt &&
2425 11269807 : !xfs_rmap_is_mergeable(<rec, owner, flags))
2426 644607 : have_lt = 0;
2427 :
2428 : /* Is there a right record that abuts our range? */
2429 121444899 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
2430 : flags, &have_gt);
2431 121445174 : if (error)
2432 1 : goto out_error;
2433 121445173 : if (have_gt) {
2434 741992 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
2435 741992 : if (error)
2436 0 : goto out_error;
2437 741992 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
2438 0 : xfs_btree_mark_sick(cur);
2439 0 : error = -EFSCORRUPTED;
2440 0 : goto out_error;
2441 : }
2442 741992 : trace_xfs_rmap_find_right_neighbor_result(cur,
2443 : gtrec.rm_startblock, gtrec.rm_blockcount,
2444 : gtrec.rm_owner, gtrec.rm_offset,
2445 : gtrec.rm_flags);
2446 :
2447 741992 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
2448 186495 : have_gt = 0;
2449 : }
2450 :
2451 121445173 : if (have_lt &&
2452 10625200 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
2453 2823983 : ltrec.rm_offset + ltrec.rm_blockcount == offset) {
2454 : /*
2455 : * Left edge contiguous, merge into left record.
2456 : *
2457 : * ltbno ltlen
2458 : * orig: |ooooooooo|
2459 : * adding: |aaaaaaaaa|
2460 : * result: |rrrrrrrrrrrrrrrrrrr|
2461 : * bno len
2462 : */
2463 2823983 : ltrec.rm_blockcount += len;
2464 2823983 : if (have_gt &&
2465 353092 : bno + len == gtrec.rm_startblock &&
2466 353092 : offset + len == gtrec.rm_offset) {
2467 : /*
2468 : * Right edge also contiguous, delete right record
2469 : * and merge into left record.
2470 : *
2471 : * ltbno ltlen gtbno gtlen
2472 : * orig: |ooooooooo| |ooooooooo|
2473 : * adding: |aaaaaaaaa|
2474 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
2475 : */
2476 353092 : ltrec.rm_blockcount += gtrec.rm_blockcount;
2477 353092 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2478 : gtrec.rm_blockcount, gtrec.rm_owner,
2479 : gtrec.rm_offset, gtrec.rm_flags);
2480 353092 : if (error)
2481 0 : goto out_error;
2482 : }
2483 :
2484 : /* Point the cursor back to the left record and update. */
2485 2823983 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2486 : ltrec.rm_blockcount, ltrec.rm_owner,
2487 : ltrec.rm_offset, ltrec.rm_flags, &i);
2488 2823982 : if (error)
2489 0 : goto out_error;
2490 2823982 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2491 0 : xfs_btree_mark_sick(cur);
2492 0 : error = -EFSCORRUPTED;
2493 0 : goto out_error;
2494 : }
2495 :
2496 2823982 : error = xfs_rmap_update(cur, <rec);
2497 2823983 : if (error)
2498 0 : goto out_error;
2499 118621190 : } else if (have_gt &&
2500 202405 : bno + len == gtrec.rm_startblock &&
2501 202405 : offset + len == gtrec.rm_offset) {
2502 : /*
2503 : * Right edge contiguous, merge into right record.
2504 : *
2505 : * gtbno gtlen
2506 : * Orig: |ooooooooo|
2507 : * adding: |aaaaaaaaa|
2508 : * Result: |rrrrrrrrrrrrrrrrrrr|
2509 : * bno len
2510 : */
2511 : /* Delete the old record. */
2512 202405 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2513 : gtrec.rm_blockcount, gtrec.rm_owner,
2514 : gtrec.rm_offset, gtrec.rm_flags);
2515 202405 : if (error)
2516 0 : goto out_error;
2517 :
2518 : /* Move the start and re-add it. */
2519 202405 : gtrec.rm_startblock = bno;
2520 202405 : gtrec.rm_blockcount += len;
2521 202405 : gtrec.rm_offset = offset;
2522 202405 : error = xfs_rmap_insert(cur, gtrec.rm_startblock,
2523 : gtrec.rm_blockcount, gtrec.rm_owner,
2524 : gtrec.rm_offset, gtrec.rm_flags);
2525 202405 : if (error)
2526 0 : goto out_error;
2527 : } else {
2528 : /*
2529 : * No contiguous edge with identical owner, insert
2530 : * new record at current cursor position.
2531 : */
2532 118418785 : error = xfs_rmap_insert(cur, bno, len, owner, offset, flags);
2533 118418747 : if (error)
2534 170 : goto out_error;
2535 : }
2536 :
2537 121444965 : trace_xfs_rmap_map_done(cur, bno, len, unwritten, oinfo);
2538 121445344 : out_error:
2539 121445344 : if (error)
2540 373 : trace_xfs_rmap_map_error(cur, error, _RET_IP_);
2541 121445344 : return error;
2542 : }
2543 :
2544 : /* Insert a raw rmap into the rmapbt. */
2545 : int
2546 25884720 : xfs_rmap_map_raw(
2547 : struct xfs_btree_cur *cur,
2548 : struct xfs_rmap_irec *rmap)
2549 : {
2550 25884720 : struct xfs_owner_info oinfo;
2551 :
2552 25884720 : xfs_owner_info_pack(&oinfo, rmap->rm_owner, rmap->rm_offset,
2553 : rmap->rm_flags);
2554 :
2555 25884720 : if ((rmap->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK |
2556 19246017 : XFS_RMAP_UNWRITTEN)) ||
2557 19246017 : XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2558 7224306 : return xfs_rmap_map(cur, rmap->rm_startblock,
2559 : rmap->rm_blockcount,
2560 7224306 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2561 : &oinfo);
2562 :
2563 18660414 : return xfs_rmap_map_shared(cur, rmap->rm_startblock,
2564 : rmap->rm_blockcount,
2565 18660414 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2566 : &oinfo);
2567 : }
2568 :
2569 : struct xfs_rmap_query_range_info {
2570 : xfs_rmap_query_range_fn fn;
2571 : void *priv;
2572 : };
2573 :
2574 : /* Format btree record and pass to our callback. */
2575 : STATIC int
2576 >10160*10^7 : xfs_rmap_query_range_helper(
2577 : struct xfs_btree_cur *cur,
2578 : const union xfs_btree_rec *rec,
2579 : void *priv)
2580 : {
2581 >10160*10^7 : struct xfs_rmap_query_range_info *query = priv;
2582 >10160*10^7 : struct xfs_rmap_irec irec;
2583 >10160*10^7 : xfs_failaddr_t fa;
2584 :
2585 >10160*10^7 : fa = xfs_rmap_btrec_to_irec(rec, &irec);
2586 >10156*10^7 : if (!fa)
2587 >10153*10^7 : fa = xfs_rmap_check_irec(cur, &irec);
2588 >10208*10^7 : if (fa)
2589 0 : return xfs_rmap_complain_bad_rec(cur, fa, &irec);
2590 :
2591 >10208*10^7 : return query->fn(cur, &irec, query->priv);
2592 : }
2593 :
2594 : /* Find all rmaps between two keys. */
2595 : int
2596 1181306004 : xfs_rmap_query_range(
2597 : struct xfs_btree_cur *cur,
2598 : const struct xfs_rmap_irec *low_rec,
2599 : const struct xfs_rmap_irec *high_rec,
2600 : xfs_rmap_query_range_fn fn,
2601 : void *priv)
2602 : {
2603 1181306004 : union xfs_btree_irec low_brec = { .r = *low_rec };
2604 1181306004 : union xfs_btree_irec high_brec = { .r = *high_rec };
2605 1181306004 : struct xfs_rmap_query_range_info query = { .priv = priv, .fn = fn };
2606 :
2607 1181306004 : return xfs_btree_query_range(cur, &low_brec, &high_brec,
2608 : xfs_rmap_query_range_helper, &query);
2609 : }
2610 :
2611 : /* Find all rmaps. */
2612 : int
2613 3649744 : xfs_rmap_query_all(
2614 : struct xfs_btree_cur *cur,
2615 : xfs_rmap_query_range_fn fn,
2616 : void *priv)
2617 : {
2618 3649744 : struct xfs_rmap_query_range_info query;
2619 :
2620 3649744 : query.priv = priv;
2621 3649744 : query.fn = fn;
2622 3649744 : return xfs_btree_query_all(cur, xfs_rmap_query_range_helper, &query);
2623 : }
2624 :
2625 : /* Clean up after calling xfs_rmap_finish_one. */
2626 : void
2627 329845827 : xfs_rmap_finish_one_cleanup(
2628 : struct xfs_trans *tp,
2629 : struct xfs_btree_cur *rcur,
2630 : int error)
2631 : {
2632 329845827 : struct xfs_buf *agbp = NULL;
2633 :
2634 329845827 : if (rcur == NULL)
2635 : return;
2636 329844855 : if (rcur->bc_btnum == XFS_BTNUM_RMAP)
2637 255974555 : agbp = rcur->bc_ag.agbp;
2638 329844855 : xfs_btree_del_cursor(rcur, error);
2639 329847609 : if (error && agbp)
2640 1288 : xfs_trans_brelse(tp, agbp);
2641 : }
2642 :
2643 : /* Commit an rmap operation into the ondisk tree. */
2644 : int
2645 351695033 : __xfs_rmap_finish_intent(
2646 : struct xfs_btree_cur *rcur,
2647 : enum xfs_rmap_intent_type op,
2648 : xfs_agblock_t bno,
2649 : xfs_extlen_t len,
2650 : const struct xfs_owner_info *oinfo,
2651 : bool unwritten)
2652 : {
2653 351695033 : switch (op) {
2654 69875447 : case XFS_RMAP_ALLOC:
2655 : case XFS_RMAP_MAP:
2656 69875447 : return xfs_rmap_map(rcur, bno, len, unwritten, oinfo);
2657 102785058 : case XFS_RMAP_MAP_SHARED:
2658 102785058 : return xfs_rmap_map_shared(rcur, bno, len, unwritten, oinfo);
2659 58347902 : case XFS_RMAP_FREE:
2660 : case XFS_RMAP_UNMAP:
2661 58347902 : return xfs_rmap_unmap(rcur, bno, len, unwritten, oinfo);
2662 84108818 : case XFS_RMAP_UNMAP_SHARED:
2663 84108818 : return xfs_rmap_unmap_shared(rcur, bno, len, unwritten, oinfo);
2664 8923244 : case XFS_RMAP_CONVERT:
2665 8923244 : return xfs_rmap_convert(rcur, bno, len, !unwritten, oinfo);
2666 27654564 : case XFS_RMAP_CONVERT_SHARED:
2667 27654564 : return xfs_rmap_convert_shared(rcur, bno, len, !unwritten,
2668 27654564 : oinfo);
2669 0 : default:
2670 0 : ASSERT(0);
2671 0 : return -EFSCORRUPTED;
2672 : }
2673 : }
2674 :
2675 : /* Does this btree cursor match the given group object? */
2676 : static inline bool
2677 : xfs_rmap_is_wrong_cursor(
2678 : struct xfs_btree_cur *cur,
2679 : struct xfs_rmap_intent *ri)
2680 : {
2681 21295239 : if (cur->bc_btnum == XFS_BTNUM_RTRMAP)
2682 1474305 : return cur->bc_ino.rtg != ri->ri_rtg;
2683 19820934 : return cur->bc_ag.pag != ri->ri_pag;
2684 : }
2685 :
2686 : /*
2687 : * Process one of the deferred rmap operations. We pass back the
2688 : * btree cursor to maintain our lock on the rmapbt between calls.
2689 : * This saves time and eliminates a buffer deadlock between the
2690 : * superblock and the AGF because we'll always grab them in the same
2691 : * order.
2692 : */
2693 : int
2694 351135991 : xfs_rmap_finish_one(
2695 : struct xfs_trans *tp,
2696 : struct xfs_rmap_intent *ri,
2697 : struct xfs_btree_cur **pcur)
2698 : {
2699 351135991 : struct xfs_owner_info oinfo;
2700 351135991 : struct xfs_mount *mp = tp->t_mountp;
2701 351135991 : struct xfs_btree_cur *rcur;
2702 351135991 : struct xfs_buf *agbp = NULL;
2703 351135991 : xfs_agblock_t bno;
2704 351135991 : bool unwritten;
2705 351135991 : int error = 0;
2706 :
2707 351135991 : trace_xfs_rmap_deferred(mp, ri);
2708 :
2709 351140154 : if (ri->ri_realtime) {
2710 75344606 : xfs_rgnumber_t rgno;
2711 :
2712 75344606 : bno = xfs_rtb_to_rgbno(mp, ri->ri_bmap.br_startblock, &rgno);
2713 : } else {
2714 275795548 : bno = XFS_FSB_TO_AGBNO(mp, ri->ri_bmap.br_startblock);
2715 : }
2716 :
2717 351140280 : if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_RMAP_FINISH_ONE))
2718 : return -EIO;
2719 :
2720 : /*
2721 : * If we haven't gotten a cursor or the cursor AG doesn't match
2722 : * the startblock, get one now.
2723 : */
2724 351138451 : rcur = *pcur;
2725 372433690 : if (rcur != NULL && xfs_rmap_is_wrong_cursor(rcur, ri)) {
2726 3751 : xfs_rmap_finish_one_cleanup(tp, rcur, 0);
2727 3751 : rcur = NULL;
2728 3751 : *pcur = NULL;
2729 : }
2730 351138451 : if (rcur == NULL) {
2731 329848727 : if (ri->ri_realtime) {
2732 73870589 : xfs_rtgroup_lock(tp, ri->ri_rtg, XFS_RTGLOCK_RMAP);
2733 73870631 : rcur = xfs_rtrmapbt_init_cursor(mp, tp, ri->ri_rtg,
2734 : ri->ri_rtg->rtg_rmapip);
2735 73870723 : rcur->bc_ino.flags = 0;
2736 : } else {
2737 : /*
2738 : * Refresh the freelist before we start changing the
2739 : * rmapbt, because a shape change could cause us to
2740 : * allocate blocks.
2741 : */
2742 255978138 : error = xfs_free_extent_fix_freelist(tp, ri->ri_pag,
2743 : &agbp);
2744 255977510 : if (error) {
2745 966 : xfs_ag_mark_sick(ri->ri_pag, XFS_SICK_AG_AGFL);
2746 966 : return error;
2747 : }
2748 255976544 : if (XFS_IS_CORRUPT(tp->t_mountp, !agbp)) {
2749 0 : xfs_ag_mark_sick(ri->ri_pag, XFS_SICK_AG_AGFL);
2750 0 : return -EFSCORRUPTED;
2751 : }
2752 :
2753 255976544 : rcur = xfs_rmapbt_init_cursor(mp, tp, agbp, ri->ri_pag);
2754 : }
2755 : }
2756 351137624 : *pcur = rcur;
2757 :
2758 351137624 : xfs_rmap_ino_owner(&oinfo, ri->ri_owner, ri->ri_whichfork,
2759 : ri->ri_bmap.br_startoff);
2760 351137624 : unwritten = ri->ri_bmap.br_state == XFS_EXT_UNWRITTEN;
2761 :
2762 351137624 : error = __xfs_rmap_finish_intent(rcur, ri->ri_type, bno,
2763 351137624 : ri->ri_bmap.br_blockcount, &oinfo, unwritten);
2764 351137136 : if (error)
2765 : return error;
2766 :
2767 351135848 : if (ri->ri_realtime)
2768 75344783 : xfs_rmap_update_hook(tp, NULL, ri->ri_rtg, ri->ri_type, bno,
2769 75344783 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2770 : else
2771 275791065 : xfs_rmap_update_hook(tp, ri->ri_pag, NULL, ri->ri_type, bno,
2772 275791065 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2773 : return 0;
2774 : }
2775 :
2776 : /*
2777 : * Don't defer an rmap if we aren't an rmap filesystem.
2778 : */
2779 : static bool
2780 : xfs_rmap_update_is_needed(
2781 : struct xfs_mount *mp,
2782 : int whichfork)
2783 : {
2784 349289511 : return xfs_has_rmapbt(mp) && whichfork != XFS_COW_FORK;
2785 : }
2786 :
2787 : /*
2788 : * Record a rmap intent; the list is kept sorted first by AG and then by
2789 : * increasing age.
2790 : */
2791 : static void
2792 351117376 : __xfs_rmap_add(
2793 : struct xfs_trans *tp,
2794 : enum xfs_rmap_intent_type type,
2795 : uint64_t owner,
2796 : bool isrt,
2797 : int whichfork,
2798 : struct xfs_bmbt_irec *bmap)
2799 : {
2800 351117376 : struct xfs_rmap_intent *ri;
2801 351117376 : enum xfs_defer_ops_type optype;
2802 :
2803 351117376 : ri = kmem_cache_alloc(xfs_rmap_intent_cache, GFP_NOFS | __GFP_NOFAIL);
2804 351121833 : INIT_LIST_HEAD(&ri->ri_list);
2805 351121833 : ri->ri_type = type;
2806 351121833 : ri->ri_owner = owner;
2807 351121833 : ri->ri_whichfork = whichfork;
2808 351121833 : ri->ri_bmap = *bmap;
2809 351121833 : ri->ri_realtime = isrt;
2810 :
2811 : /*
2812 : * Deferred rmap updates for the realtime and data sections must use
2813 : * separate transactions to finish deferred work because updates to
2814 : * realtime metadata files can lock AGFs to allocate btree blocks and
2815 : * we don't want that mixing with the AGF locks taken to finish data
2816 : * section updates.
2817 : */
2818 351121833 : if (isrt)
2819 : optype = XFS_DEFER_OPS_TYPE_RMAP_RT;
2820 : else
2821 275776510 : optype = XFS_DEFER_OPS_TYPE_RMAP;
2822 :
2823 351121833 : trace_xfs_rmap_defer(tp->t_mountp, ri);
2824 :
2825 351125179 : xfs_rmap_update_get_group(tp->t_mountp, ri);
2826 351120423 : xfs_defer_add(tp, optype, &ri->ri_list);
2827 351122289 : }
2828 :
2829 : /* Map an extent into a file. */
2830 : void
2831 172320317 : xfs_rmap_map_extent(
2832 : struct xfs_trans *tp,
2833 : struct xfs_inode *ip,
2834 : int whichfork,
2835 : struct xfs_bmbt_irec *PREV)
2836 : {
2837 172320317 : enum xfs_rmap_intent_type type = XFS_RMAP_MAP;
2838 172320317 : bool isrt = xfs_ifork_is_realtime(ip, whichfork);
2839 :
2840 344641736 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2841 : return;
2842 :
2843 171647379 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2844 102589850 : type = XFS_RMAP_MAP_SHARED;
2845 :
2846 171647379 : __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV);
2847 : }
2848 :
2849 : /* Unmap an extent out of a file. */
2850 : void
2851 138625659 : xfs_rmap_unmap_extent(
2852 : struct xfs_trans *tp,
2853 : struct xfs_inode *ip,
2854 : int whichfork,
2855 : struct xfs_bmbt_irec *PREV)
2856 : {
2857 138625659 : enum xfs_rmap_intent_type type = XFS_RMAP_UNMAP;
2858 138625659 : bool isrt = xfs_ifork_is_realtime(ip, whichfork);
2859 :
2860 277252430 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2861 : return;
2862 :
2863 138504033 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2864 83944763 : type = XFS_RMAP_UNMAP_SHARED;
2865 :
2866 138504033 : __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV);
2867 : }
2868 :
2869 : /*
2870 : * Convert a data fork extent from unwritten to real or vice versa.
2871 : *
2872 : * Note that tp can be NULL here as no transaction is used for COW fork
2873 : * unwritten conversion.
2874 : */
2875 : void
2876 38474285 : xfs_rmap_convert_extent(
2877 : struct xfs_mount *mp,
2878 : struct xfs_trans *tp,
2879 : struct xfs_inode *ip,
2880 : int whichfork,
2881 : struct xfs_bmbt_irec *PREV)
2882 : {
2883 38474285 : enum xfs_rmap_intent_type type = XFS_RMAP_CONVERT;
2884 38474285 : bool isrt = xfs_ifork_is_realtime(ip, whichfork);
2885 :
2886 76948496 : if (!xfs_rmap_update_is_needed(mp, whichfork))
2887 : return;
2888 :
2889 36465917 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2890 27559887 : type = XFS_RMAP_CONVERT_SHARED;
2891 :
2892 36465917 : __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV);
2893 : }
2894 :
2895 : /* Schedule the creation of an rmap for non-file data. */
2896 : void
2897 757545 : xfs_rmap_alloc_extent(
2898 : struct xfs_trans *tp,
2899 : bool isrt,
2900 : xfs_fsblock_t fsbno,
2901 : xfs_extlen_t len,
2902 : uint64_t owner)
2903 : {
2904 757545 : struct xfs_bmbt_irec bmap;
2905 :
2906 757545 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2907 0 : return;
2908 :
2909 757545 : bmap.br_startblock = fsbno;
2910 757545 : bmap.br_blockcount = len;
2911 757545 : bmap.br_startoff = 0;
2912 757545 : bmap.br_state = XFS_EXT_NORM;
2913 :
2914 757545 : __xfs_rmap_add(tp, XFS_RMAP_ALLOC, owner, isrt, XFS_DATA_FORK, &bmap);
2915 : }
2916 :
2917 : /* Schedule the deletion of an rmap for non-file data. */
2918 : void
2919 3756507 : xfs_rmap_free_extent(
2920 : struct xfs_trans *tp,
2921 : bool isrt,
2922 : xfs_fsblock_t fsbno,
2923 : xfs_extlen_t len,
2924 : uint64_t owner)
2925 : {
2926 3756507 : struct xfs_bmbt_irec bmap;
2927 :
2928 3756507 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2929 0 : return;
2930 :
2931 3756507 : bmap.br_startblock = fsbno;
2932 3756507 : bmap.br_blockcount = len;
2933 3756507 : bmap.br_startoff = 0;
2934 3756507 : bmap.br_state = XFS_EXT_NORM;
2935 :
2936 3756507 : __xfs_rmap_add(tp, XFS_RMAP_FREE, owner, isrt, XFS_DATA_FORK, &bmap);
2937 : }
2938 :
2939 : /* Compare rmap records. Returns -1 if a < b, 1 if a > b, and 0 if equal. */
2940 : int
2941 80652799 : xfs_rmap_compare(
2942 : const struct xfs_rmap_irec *a,
2943 : const struct xfs_rmap_irec *b)
2944 : {
2945 80652799 : __u64 oa;
2946 80652799 : __u64 ob;
2947 :
2948 80652799 : oa = xfs_rmap_irec_offset_pack(a);
2949 80652799 : ob = xfs_rmap_irec_offset_pack(b);
2950 :
2951 80652799 : if (a->rm_startblock < b->rm_startblock)
2952 : return -1;
2953 80619869 : else if (a->rm_startblock > b->rm_startblock)
2954 : return 1;
2955 8004309 : else if (a->rm_owner < b->rm_owner)
2956 : return -1;
2957 7860089 : else if (a->rm_owner > b->rm_owner)
2958 : return 1;
2959 7666669 : else if (oa < ob)
2960 : return -1;
2961 7604550 : else if (oa > ob)
2962 : return 1;
2963 : else
2964 3714 : return 0;
2965 : }
2966 :
2967 : /*
2968 : * Scan the physical storage part of the keyspace of the reverse mapping index
2969 : * and tell us if the area has no records, is fully mapped by records, or is
2970 : * partially filled.
2971 : */
2972 : int
2973 395182093 : xfs_rmap_has_records(
2974 : struct xfs_btree_cur *cur,
2975 : xfs_agblock_t bno,
2976 : xfs_extlen_t len,
2977 : enum xbtree_recpacking *outcome)
2978 : {
2979 395182093 : union xfs_btree_key mask = {
2980 : .rmap.rm_startblock = cpu_to_be32(-1U),
2981 : };
2982 395182093 : union xfs_btree_irec low;
2983 395182093 : union xfs_btree_irec high;
2984 :
2985 395182093 : memset(&low, 0, sizeof(low));
2986 395182093 : low.r.rm_startblock = bno;
2987 395182093 : memset(&high, 0xFF, sizeof(high));
2988 395182093 : high.r.rm_startblock = bno + len - 1;
2989 :
2990 395182093 : return xfs_btree_has_records(cur, &low, &high, &mask, outcome);
2991 : }
2992 :
2993 : struct xfs_rmap_ownercount {
2994 : /* Owner that we're looking for. */
2995 : struct xfs_rmap_irec good;
2996 :
2997 : /* rmap search keys */
2998 : struct xfs_rmap_irec low;
2999 : struct xfs_rmap_irec high;
3000 :
3001 : struct xfs_rmap_matches *results;
3002 :
3003 : /* Stop early if we find a nonmatch? */
3004 : bool stop_on_nonmatch;
3005 : };
3006 :
3007 : /* Does this rmap represent space that can have multiple owners? */
3008 : static inline bool
3009 : xfs_rmap_shareable(
3010 : struct xfs_mount *mp,
3011 : const struct xfs_rmap_irec *rmap)
3012 : {
3013 84927740 : if (!xfs_has_reflink(mp))
3014 : return false;
3015 169855404 : if (XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
3016 : return false;
3017 83900440 : if (rmap->rm_flags & (XFS_RMAP_ATTR_FORK |
3018 : XFS_RMAP_BMBT_BLOCK))
3019 7528407 : return false;
3020 : return true;
3021 : }
3022 :
3023 : static inline void
3024 939756123 : xfs_rmap_ownercount_init(
3025 : struct xfs_rmap_ownercount *roc,
3026 : xfs_agblock_t bno,
3027 : xfs_extlen_t len,
3028 : const struct xfs_owner_info *oinfo,
3029 : struct xfs_rmap_matches *results)
3030 : {
3031 939756123 : memset(roc, 0, sizeof(*roc));
3032 939756123 : roc->results = results;
3033 :
3034 939756123 : roc->low.rm_startblock = bno;
3035 939756123 : memset(&roc->high, 0xFF, sizeof(roc->high));
3036 939756123 : roc->high.rm_startblock = bno + len - 1;
3037 :
3038 939756123 : memset(results, 0, sizeof(*results));
3039 939756123 : roc->good.rm_startblock = bno;
3040 939756123 : roc->good.rm_blockcount = len;
3041 939756123 : roc->good.rm_owner = oinfo->oi_owner;
3042 939756123 : roc->good.rm_offset = oinfo->oi_offset;
3043 939756123 : if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK)
3044 2265254 : roc->good.rm_flags |= XFS_RMAP_ATTR_FORK;
3045 939756123 : if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK)
3046 8856080 : roc->good.rm_flags |= XFS_RMAP_BMBT_BLOCK;
3047 939756123 : }
3048 :
3049 : /* Figure out if this is a match for the owner. */
3050 : STATIC int
3051 997167375 : xfs_rmap_count_owners_helper(
3052 : struct xfs_btree_cur *cur,
3053 : const struct xfs_rmap_irec *rec,
3054 : void *priv)
3055 : {
3056 997167375 : struct xfs_rmap_ownercount *roc = priv;
3057 997167375 : struct xfs_rmap_irec check = *rec;
3058 997167375 : unsigned int keyflags;
3059 997167375 : bool filedata;
3060 997167375 : int64_t delta;
3061 :
3062 997167375 : filedata = !XFS_RMAP_NON_INODE_OWNER(check.rm_owner) &&
3063 560788810 : !(check.rm_flags & XFS_RMAP_BMBT_BLOCK);
3064 :
3065 : /* Trim the part of check that comes before the comparison range. */
3066 997167375 : delta = (int64_t)roc->good.rm_startblock - check.rm_startblock;
3067 997167375 : if (delta > 0) {
3068 298661237 : check.rm_startblock += delta;
3069 298661237 : check.rm_blockcount -= delta;
3070 298661237 : if (filedata)
3071 3362989 : check.rm_offset += delta;
3072 : }
3073 :
3074 : /* Trim the part of check that comes after the comparison range. */
3075 997167375 : delta = (check.rm_startblock + check.rm_blockcount) -
3076 997167375 : (roc->good.rm_startblock + roc->good.rm_blockcount);
3077 997167375 : if (delta > 0)
3078 364248600 : check.rm_blockcount -= delta;
3079 :
3080 : /* Don't care about unwritten status for establishing ownership. */
3081 997167375 : keyflags = check.rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK);
3082 :
3083 997167375 : if (check.rm_startblock == roc->good.rm_startblock &&
3084 937195516 : check.rm_blockcount == roc->good.rm_blockcount &&
3085 937195516 : check.rm_owner == roc->good.rm_owner &&
3086 912239458 : check.rm_offset == roc->good.rm_offset &&
3087 912239537 : keyflags == roc->good.rm_flags) {
3088 912239635 : roc->results->matches++;
3089 : } else {
3090 84927740 : roc->results->non_owner_matches++;
3091 169855480 : if (xfs_rmap_shareable(cur->bc_mp, &roc->good) ^
3092 : xfs_rmap_shareable(cur->bc_mp, &check))
3093 76372061 : roc->results->bad_non_owner_matches++;
3094 : }
3095 :
3096 997167375 : if (roc->results->non_owner_matches && roc->stop_on_nonmatch)
3097 0 : return -ECANCELED;
3098 :
3099 : return 0;
3100 : }
3101 :
3102 : /* Count the number of owners and non-owners of this range of blocks. */
3103 : int
3104 935487457 : xfs_rmap_count_owners(
3105 : struct xfs_btree_cur *cur,
3106 : xfs_agblock_t bno,
3107 : xfs_extlen_t len,
3108 : const struct xfs_owner_info *oinfo,
3109 : struct xfs_rmap_matches *results)
3110 : {
3111 935487457 : struct xfs_rmap_ownercount roc;
3112 935487457 : int error;
3113 :
3114 935487457 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, results);
3115 935489018 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
3116 : xfs_rmap_count_owners_helper, &roc);
3117 935493132 : if (error)
3118 : return error;
3119 :
3120 : /*
3121 : * There can't be any non-owner rmaps that conflict with the given
3122 : * owner if we didn't find any rmaps matching the owner.
3123 : */
3124 935493132 : if (!results->matches)
3125 27362529 : results->bad_non_owner_matches = 0;
3126 :
3127 : return 0;
3128 : }
3129 :
3130 : /*
3131 : * Given an extent and some owner info, can we find records overlapping
3132 : * the extent whose owner info does not match the given owner?
3133 : */
3134 : int
3135 4267745 : xfs_rmap_has_other_keys(
3136 : struct xfs_btree_cur *cur,
3137 : xfs_agblock_t bno,
3138 : xfs_extlen_t len,
3139 : const struct xfs_owner_info *oinfo,
3140 : bool *has_other)
3141 : {
3142 4267745 : struct xfs_rmap_matches res;
3143 4267745 : struct xfs_rmap_ownercount roc;
3144 4267745 : int error;
3145 :
3146 4267745 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, &res);
3147 4267739 : roc.stop_on_nonmatch = true;
3148 :
3149 4267739 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
3150 : xfs_rmap_count_owners_helper, &roc);
3151 4267741 : if (error == -ECANCELED) {
3152 0 : *has_other = true;
3153 0 : return 0;
3154 : }
3155 4267741 : if (error)
3156 : return error;
3157 :
3158 4267741 : *has_other = false;
3159 4267741 : return 0;
3160 : }
3161 :
3162 : const struct xfs_owner_info XFS_RMAP_OINFO_SKIP_UPDATE = {
3163 : .oi_owner = XFS_RMAP_OWN_NULL,
3164 : };
3165 : const struct xfs_owner_info XFS_RMAP_OINFO_ANY_OWNER = {
3166 : .oi_owner = XFS_RMAP_OWN_UNKNOWN,
3167 : };
3168 : const struct xfs_owner_info XFS_RMAP_OINFO_FS = {
3169 : .oi_owner = XFS_RMAP_OWN_FS,
3170 : };
3171 : const struct xfs_owner_info XFS_RMAP_OINFO_LOG = {
3172 : .oi_owner = XFS_RMAP_OWN_LOG,
3173 : };
3174 : const struct xfs_owner_info XFS_RMAP_OINFO_AG = {
3175 : .oi_owner = XFS_RMAP_OWN_AG,
3176 : };
3177 : const struct xfs_owner_info XFS_RMAP_OINFO_INOBT = {
3178 : .oi_owner = XFS_RMAP_OWN_INOBT,
3179 : };
3180 : const struct xfs_owner_info XFS_RMAP_OINFO_INODES = {
3181 : .oi_owner = XFS_RMAP_OWN_INODES,
3182 : };
3183 : const struct xfs_owner_info XFS_RMAP_OINFO_REFC = {
3184 : .oi_owner = XFS_RMAP_OWN_REFC,
3185 : };
3186 : const struct xfs_owner_info XFS_RMAP_OINFO_COW = {
3187 : .oi_owner = XFS_RMAP_OWN_COW,
3188 : };
3189 :
3190 : int __init
3191 12 : xfs_rmap_intent_init_cache(void)
3192 : {
3193 12 : xfs_rmap_intent_cache = kmem_cache_create("xfs_rmap_intent",
3194 : sizeof(struct xfs_rmap_intent),
3195 : 0, 0, NULL);
3196 :
3197 12 : return xfs_rmap_intent_cache != NULL ? 0 : -ENOMEM;
3198 : }
3199 :
3200 : void
3201 12 : xfs_rmap_intent_destroy_cache(void)
3202 : {
3203 12 : kmem_cache_destroy(xfs_rmap_intent_cache);
3204 12 : xfs_rmap_intent_cache = NULL;
3205 12 : }
|
