Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2014 Red Hat, Inc.
4 : * All Rights Reserved.
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_bit.h"
13 : #include "xfs_mount.h"
14 : #include "xfs_sb.h"
15 : #include "xfs_defer.h"
16 : #include "xfs_btree.h"
17 : #include "xfs_trans.h"
18 : #include "xfs_alloc.h"
19 : #include "xfs_rmap.h"
20 : #include "xfs_rmap_btree.h"
21 : #include "xfs_trace.h"
22 : #include "xfs_errortag.h"
23 : #include "xfs_error.h"
24 : #include "xfs_inode.h"
25 : #include "xfs_ag.h"
26 : #include "xfs_health.h"
27 :
28 : struct kmem_cache *xfs_rmap_intent_cache;
29 :
30 : /*
31 : * Lookup the first record less than or equal to [bno, len, owner, offset]
32 : * in the btree given by cur.
33 : */
34 : int
35 802365980 : xfs_rmap_lookup_le(
36 : struct xfs_btree_cur *cur,
37 : xfs_agblock_t bno,
38 : uint64_t owner,
39 : uint64_t offset,
40 : unsigned int flags,
41 : struct xfs_rmap_irec *irec,
42 : int *stat)
43 : {
44 802365980 : int get_stat = 0;
45 802365980 : int error;
46 :
47 802365980 : cur->bc_rec.r.rm_startblock = bno;
48 802365980 : cur->bc_rec.r.rm_blockcount = 0;
49 802365980 : cur->bc_rec.r.rm_owner = owner;
50 802365980 : cur->bc_rec.r.rm_offset = offset;
51 802365980 : cur->bc_rec.r.rm_flags = flags;
52 :
53 802365980 : error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat);
54 802397687 : if (error || !(*stat) || !irec)
55 : return error;
56 :
57 789404176 : error = xfs_rmap_get_rec(cur, irec, &get_stat);
58 789285732 : if (error)
59 : return error;
60 789285732 : if (!get_stat) {
61 0 : xfs_btree_mark_sick(cur);
62 0 : return -EFSCORRUPTED;
63 : }
64 :
65 : return 0;
66 : }
67 :
68 : /*
69 : * Lookup the record exactly matching [bno, len, owner, offset]
70 : * in the btree given by cur.
71 : */
72 : int
73 0 : xfs_rmap_lookup_eq(
74 : struct xfs_btree_cur *cur,
75 : xfs_agblock_t bno,
76 : xfs_extlen_t len,
77 : uint64_t owner,
78 : uint64_t offset,
79 : unsigned int flags,
80 : int *stat)
81 : {
82 515265703 : cur->bc_rec.r.rm_startblock = bno;
83 515265703 : cur->bc_rec.r.rm_blockcount = len;
84 515265703 : cur->bc_rec.r.rm_owner = owner;
85 515265703 : cur->bc_rec.r.rm_offset = offset;
86 515265703 : cur->bc_rec.r.rm_flags = flags;
87 0 : return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat);
88 : }
89 :
90 : /*
91 : * Update the record referred to by cur to the value given
92 : * by [bno, len, owner, offset].
93 : * This either works (return 0) or gets an EFSCORRUPTED error.
94 : */
95 : STATIC int
96 86204080 : xfs_rmap_update(
97 : struct xfs_btree_cur *cur,
98 : struct xfs_rmap_irec *irec)
99 : {
100 86204080 : union xfs_btree_rec rec;
101 86204080 : int error;
102 :
103 86204080 : trace_xfs_rmap_update(cur->bc_mp, cur->bc_ag.pag->pag_agno,
104 : irec->rm_startblock, irec->rm_blockcount,
105 : irec->rm_owner, irec->rm_offset, irec->rm_flags);
106 :
107 86203542 : rec.rmap.rm_startblock = cpu_to_be32(irec->rm_startblock);
108 86203542 : rec.rmap.rm_blockcount = cpu_to_be32(irec->rm_blockcount);
109 86203542 : rec.rmap.rm_owner = cpu_to_be64(irec->rm_owner);
110 86203542 : rec.rmap.rm_offset = cpu_to_be64(
111 : xfs_rmap_irec_offset_pack(irec));
112 86203542 : error = xfs_btree_update(cur, &rec);
113 86205273 : if (error)
114 0 : trace_xfs_rmap_update_error(cur->bc_mp,
115 0 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
116 86205273 : return error;
117 : }
118 :
119 : int
120 190293267 : xfs_rmap_insert(
121 : struct xfs_btree_cur *rcur,
122 : xfs_agblock_t agbno,
123 : xfs_extlen_t len,
124 : uint64_t owner,
125 : uint64_t offset,
126 : unsigned int flags)
127 : {
128 190293267 : int i;
129 190293267 : int error;
130 :
131 190293267 : trace_xfs_rmap_insert(rcur->bc_mp, rcur->bc_ag.pag->pag_agno, agbno,
132 : len, owner, offset, flags);
133 :
134 190292443 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
135 190293886 : if (error)
136 0 : goto done;
137 190293886 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 0)) {
138 0 : xfs_btree_mark_sick(rcur);
139 0 : error = -EFSCORRUPTED;
140 0 : goto done;
141 : }
142 :
143 190293886 : rcur->bc_rec.r.rm_startblock = agbno;
144 190293886 : rcur->bc_rec.r.rm_blockcount = len;
145 190293886 : rcur->bc_rec.r.rm_owner = owner;
146 190293886 : rcur->bc_rec.r.rm_offset = offset;
147 190293886 : rcur->bc_rec.r.rm_flags = flags;
148 190293886 : error = xfs_btree_insert(rcur, &i);
149 190292804 : if (error)
150 175 : goto done;
151 190292629 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
152 0 : xfs_btree_mark_sick(rcur);
153 0 : error = -EFSCORRUPTED;
154 0 : goto done;
155 : }
156 190292804 : done:
157 190292804 : if (error)
158 350 : trace_xfs_rmap_insert_error(rcur->bc_mp,
159 175 : rcur->bc_ag.pag->pag_agno, error, _RET_IP_);
160 190292804 : return error;
161 : }
162 :
163 : STATIC int
164 99369466 : xfs_rmap_delete(
165 : struct xfs_btree_cur *rcur,
166 : xfs_agblock_t agbno,
167 : xfs_extlen_t len,
168 : uint64_t owner,
169 : uint64_t offset,
170 : unsigned int flags)
171 : {
172 99369466 : int i;
173 99369466 : int error;
174 :
175 99369466 : trace_xfs_rmap_delete(rcur->bc_mp, rcur->bc_ag.pag->pag_agno, agbno,
176 : len, owner, offset, flags);
177 :
178 99368628 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
179 99373027 : if (error)
180 0 : goto done;
181 99373027 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
182 0 : xfs_btree_mark_sick(rcur);
183 0 : error = -EFSCORRUPTED;
184 0 : goto done;
185 : }
186 :
187 99373027 : error = xfs_btree_delete(rcur, &i);
188 99372135 : if (error)
189 5 : goto done;
190 99372130 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
191 0 : xfs_btree_mark_sick(rcur);
192 0 : error = -EFSCORRUPTED;
193 0 : goto done;
194 : }
195 99372135 : done:
196 99372135 : if (error)
197 10 : trace_xfs_rmap_delete_error(rcur->bc_mp,
198 5 : rcur->bc_ag.pag->pag_agno, error, _RET_IP_);
199 99372135 : return error;
200 : }
201 :
202 : /* Convert an internal btree record to an rmap record. */
203 : xfs_failaddr_t
204 86281706756 : xfs_rmap_btrec_to_irec(
205 : const union xfs_btree_rec *rec,
206 : struct xfs_rmap_irec *irec)
207 : {
208 86281706756 : irec->rm_startblock = be32_to_cpu(rec->rmap.rm_startblock);
209 86281706756 : irec->rm_blockcount = be32_to_cpu(rec->rmap.rm_blockcount);
210 86281706756 : irec->rm_owner = be64_to_cpu(rec->rmap.rm_owner);
211 86281706756 : return xfs_rmap_irec_offset_unpack(be64_to_cpu(rec->rmap.rm_offset),
212 : irec);
213 : }
214 :
215 : inline xfs_failaddr_t
216 86184096198 : xfs_rmap_check_perag_irec(
217 : struct xfs_perag *pag,
218 : const struct xfs_rmap_irec *irec)
219 : {
220 86184096198 : struct xfs_mount *mp = pag->pag_mount;
221 86184096198 : bool is_inode;
222 86184096198 : bool is_unwritten;
223 86184096198 : bool is_bmbt;
224 86184096198 : bool is_attr;
225 :
226 86184096198 : if (irec->rm_blockcount == 0)
227 0 : return __this_address;
228 86184096198 : if (irec->rm_startblock <= XFS_AGFL_BLOCK(mp)) {
229 17788116 : if (irec->rm_owner != XFS_RMAP_OWN_FS)
230 0 : return __this_address;
231 17788116 : if (irec->rm_blockcount != XFS_AGFL_BLOCK(mp) + 1)
232 0 : return __this_address;
233 : } else {
234 : /* check for valid extent range, including overflow */
235 86166308082 : if (!xfs_verify_agbext(pag, irec->rm_startblock,
236 86166308082 : irec->rm_blockcount))
237 0 : return __this_address;
238 : }
239 :
240 86184096198 : if (!(xfs_verify_ino(mp, irec->rm_owner) ||
241 2932014304 : (irec->rm_owner <= XFS_RMAP_OWN_FS &&
242 : irec->rm_owner >= XFS_RMAP_OWN_MIN)))
243 0 : return __this_address;
244 :
245 : /* Check flags. */
246 86232256761 : is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner);
247 86232256761 : is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK;
248 86232256761 : is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK;
249 86232256761 : is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN;
250 :
251 86232256761 : if (is_bmbt && irec->rm_offset != 0)
252 0 : return __this_address;
253 :
254 86232256761 : if (!is_inode && irec->rm_offset != 0)
255 0 : return __this_address;
256 :
257 86232256761 : if (is_unwritten && (is_bmbt || !is_inode || is_attr))
258 0 : return __this_address;
259 :
260 86232256761 : if (!is_inode && (is_bmbt || is_unwritten || is_attr))
261 0 : return __this_address;
262 :
263 : /* Check for a valid fork offset, if applicable. */
264 >16815*10^7 : if (is_inode && !is_bmbt &&
265 81991818566 : !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
266 0 : return __this_address;
267 :
268 : return NULL;
269 : }
270 :
271 : /* Simple checks for rmap records. */
272 : xfs_failaddr_t
273 86153769914 : xfs_rmap_check_irec(
274 : struct xfs_btree_cur *cur,
275 : const struct xfs_rmap_irec *irec)
276 : {
277 86153769914 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
278 252320726 : return xfs_rmap_check_perag_irec(cur->bc_mem.pag, irec);
279 85901449188 : return xfs_rmap_check_perag_irec(cur->bc_ag.pag, irec);
280 : }
281 :
282 : static inline int
283 0 : xfs_rmap_complain_bad_rec(
284 : struct xfs_btree_cur *cur,
285 : xfs_failaddr_t fa,
286 : const struct xfs_rmap_irec *irec)
287 : {
288 0 : struct xfs_mount *mp = cur->bc_mp;
289 :
290 0 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
291 0 : xfs_warn(mp,
292 : "In-Memory Reverse Mapping BTree record corruption detected at %pS!", fa);
293 : else
294 0 : xfs_warn(mp,
295 : "Reverse Mapping BTree record corruption in AG %d detected at %pS!",
296 : cur->bc_ag.pag->pag_agno, fa);
297 0 : xfs_warn(mp,
298 : "Owner 0x%llx, flags 0x%x, start block 0x%x block count 0x%x",
299 : irec->rm_owner, irec->rm_flags, irec->rm_startblock,
300 : irec->rm_blockcount);
301 0 : xfs_btree_mark_sick(cur);
302 0 : return -EFSCORRUPTED;
303 : }
304 :
305 : /*
306 : * Get the data from the pointed-to record.
307 : */
308 : int
309 1447637833 : xfs_rmap_get_rec(
310 : struct xfs_btree_cur *cur,
311 : struct xfs_rmap_irec *irec,
312 : int *stat)
313 : {
314 1447637833 : union xfs_btree_rec *rec;
315 1447637833 : xfs_failaddr_t fa;
316 1447637833 : int error;
317 :
318 1447637833 : error = xfs_btree_get_rec(cur, &rec, stat);
319 1447642039 : if (error || !*stat)
320 : return error;
321 :
322 1447648489 : fa = xfs_rmap_btrec_to_irec(rec, irec);
323 1447606370 : if (!fa)
324 1447611385 : fa = xfs_rmap_check_irec(cur, irec);
325 1447523779 : if (fa)
326 0 : return xfs_rmap_complain_bad_rec(cur, fa, irec);
327 :
328 : return 0;
329 : }
330 :
331 : struct xfs_find_left_neighbor_info {
332 : struct xfs_rmap_irec high;
333 : struct xfs_rmap_irec *irec;
334 : };
335 :
336 : /* For each rmap given, figure out if it matches the key we want. */
337 : STATIC int
338 319236645 : xfs_rmap_find_left_neighbor_helper(
339 : struct xfs_btree_cur *cur,
340 : const struct xfs_rmap_irec *rec,
341 : void *priv)
342 : {
343 319236645 : struct xfs_find_left_neighbor_info *info = priv;
344 :
345 319236645 : trace_xfs_rmap_find_left_neighbor_candidate(cur->bc_mp,
346 319236645 : cur->bc_ag.pag->pag_agno, rec->rm_startblock,
347 319236645 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
348 319236645 : rec->rm_flags);
349 :
350 319234510 : if (rec->rm_owner != info->high.rm_owner)
351 : return 0;
352 79879154 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
353 79879154 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
354 79798369 : rec->rm_offset + rec->rm_blockcount - 1 != info->high.rm_offset)
355 : return 0;
356 :
357 33129612 : *info->irec = *rec;
358 33129612 : return -ECANCELED;
359 : }
360 :
361 : /*
362 : * Find the record to the left of the given extent, being careful only to
363 : * return a match with the same owner and adjacent physical and logical
364 : * block ranges.
365 : */
366 : STATIC int
367 182957104 : xfs_rmap_find_left_neighbor(
368 : struct xfs_btree_cur *cur,
369 : xfs_agblock_t bno,
370 : uint64_t owner,
371 : uint64_t offset,
372 : unsigned int flags,
373 : struct xfs_rmap_irec *irec,
374 : int *stat)
375 : {
376 182957104 : struct xfs_find_left_neighbor_info info;
377 182957104 : int found = 0;
378 182957104 : int error;
379 :
380 182957104 : *stat = 0;
381 182957104 : if (bno == 0)
382 : return 0;
383 182957104 : info.high.rm_startblock = bno - 1;
384 182957104 : info.high.rm_owner = owner;
385 182957104 : if (!XFS_RMAP_NON_INODE_OWNER(owner) &&
386 182956702 : !(flags & XFS_RMAP_BMBT_BLOCK)) {
387 182957104 : if (offset == 0)
388 : return 0;
389 181183998 : info.high.rm_offset = offset - 1;
390 : } else
391 0 : info.high.rm_offset = 0;
392 181183998 : info.high.rm_flags = flags;
393 181183998 : info.high.rm_blockcount = 0;
394 181183998 : info.irec = irec;
395 :
396 181183998 : trace_xfs_rmap_find_left_neighbor_query(cur->bc_mp,
397 181183998 : cur->bc_ag.pag->pag_agno, bno, 0, owner, offset, flags);
398 :
399 : /*
400 : * Historically, we always used the range query to walk every reverse
401 : * mapping that could possibly overlap the key that the caller asked
402 : * for, and filter out the ones that don't. That is very slow when
403 : * there are a lot of records.
404 : *
405 : * However, there are two scenarios where the classic btree search can
406 : * produce correct results -- if the index contains a record that is an
407 : * exact match for the lookup key; and if there are no other records
408 : * between the record we want and the key we supplied.
409 : *
410 : * As an optimization, try a non-overlapped lookup first. This makes
411 : * extent conversion and remap operations run a bit faster if the
412 : * physical extents aren't being shared. If we don't find what we
413 : * want, we fall back to the overlapped query.
414 : */
415 181182200 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
416 : &found);
417 181184732 : if (error)
418 : return error;
419 181184440 : if (found)
420 181183969 : error = xfs_rmap_find_left_neighbor_helper(cur, irec, &info);
421 181182013 : if (!error)
422 149667579 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
423 : xfs_rmap_find_left_neighbor_helper, &info);
424 181185894 : if (error != -ECANCELED)
425 : return error;
426 :
427 33129611 : *stat = 1;
428 33129611 : trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
429 33129611 : cur->bc_ag.pag->pag_agno, irec->rm_startblock,
430 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
431 : irec->rm_flags);
432 33129611 : return 0;
433 : }
434 :
435 : /* For each rmap given, figure out if it matches the key we want. */
436 : STATIC int
437 262994937 : xfs_rmap_lookup_le_range_helper(
438 : struct xfs_btree_cur *cur,
439 : const struct xfs_rmap_irec *rec,
440 : void *priv)
441 : {
442 262994937 : struct xfs_find_left_neighbor_info *info = priv;
443 :
444 262994937 : trace_xfs_rmap_lookup_le_range_candidate(cur->bc_mp,
445 262994937 : cur->bc_ag.pag->pag_agno, rec->rm_startblock,
446 262994937 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
447 262994937 : rec->rm_flags);
448 :
449 262986316 : if (rec->rm_owner != info->high.rm_owner)
450 : return 0;
451 237680104 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
452 237680104 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
453 237680104 : (rec->rm_offset > info->high.rm_offset ||
454 237476919 : rec->rm_offset + rec->rm_blockcount <= info->high.rm_offset))
455 : return 0;
456 :
457 237356755 : *info->irec = *rec;
458 237356755 : return -ECANCELED;
459 : }
460 :
461 : /*
462 : * Find the record to the left of the given extent, being careful only to
463 : * return a match with the same owner and overlapping physical and logical
464 : * block ranges. This is the overlapping-interval version of
465 : * xfs_rmap_lookup_le.
466 : */
467 : int
468 237379220 : xfs_rmap_lookup_le_range(
469 : struct xfs_btree_cur *cur,
470 : xfs_agblock_t bno,
471 : uint64_t owner,
472 : uint64_t offset,
473 : unsigned int flags,
474 : struct xfs_rmap_irec *irec,
475 : int *stat)
476 : {
477 237379220 : struct xfs_find_left_neighbor_info info;
478 237379220 : int found = 0;
479 237379220 : int error;
480 :
481 237379220 : info.high.rm_startblock = bno;
482 237379220 : info.high.rm_owner = owner;
483 237379220 : if (!XFS_RMAP_NON_INODE_OWNER(owner) && !(flags & XFS_RMAP_BMBT_BLOCK))
484 237379220 : info.high.rm_offset = offset;
485 : else
486 0 : info.high.rm_offset = 0;
487 237379220 : info.high.rm_flags = flags;
488 237379220 : info.high.rm_blockcount = 0;
489 237379220 : *stat = 0;
490 237379220 : info.irec = irec;
491 :
492 237379220 : trace_xfs_rmap_lookup_le_range(cur->bc_mp, cur->bc_ag.pag->pag_agno,
493 : bno, 0, owner, offset, flags);
494 :
495 : /*
496 : * Historically, we always used the range query to walk every reverse
497 : * mapping that could possibly overlap the key that the caller asked
498 : * for, and filter out the ones that don't. That is very slow when
499 : * there are a lot of records.
500 : *
501 : * However, there are two scenarios where the classic btree search can
502 : * produce correct results -- if the index contains a record that is an
503 : * exact match for the lookup key; and if there are no other records
504 : * between the record we want and the key we supplied.
505 : *
506 : * As an optimization, try a non-overlapped lookup first. This makes
507 : * scrub run much faster on most filesystems because bmbt records are
508 : * usually an exact match for rmap records. If we don't find what we
509 : * want, we fall back to the overlapped query.
510 : */
511 237371578 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
512 : &found);
513 237369588 : if (error)
514 : return error;
515 237369289 : if (found)
516 237364877 : error = xfs_rmap_lookup_le_range_helper(cur, irec, &info);
517 237361718 : if (!error)
518 1051981 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
519 : xfs_rmap_lookup_le_range_helper, &info);
520 237361726 : if (error != -ECANCELED)
521 : return error;
522 :
523 237357909 : *stat = 1;
524 237357909 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
525 237357909 : cur->bc_ag.pag->pag_agno, irec->rm_startblock,
526 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
527 : irec->rm_flags);
528 237357909 : return 0;
529 : }
530 :
531 : /*
532 : * Perform all the relevant owner checks for a removal op. If we're doing an
533 : * unknown-owner removal then we have no owner information to check.
534 : */
535 : static int
536 103278806 : xfs_rmap_free_check_owner(
537 : struct xfs_btree_cur *cur,
538 : uint64_t ltoff,
539 : struct xfs_rmap_irec *rec,
540 : xfs_filblks_t len,
541 : uint64_t owner,
542 : uint64_t offset,
543 : unsigned int flags)
544 : {
545 103278806 : struct xfs_mount *mp = cur->bc_mp;
546 103278806 : int error = 0;
547 :
548 103278806 : if (owner == XFS_RMAP_OWN_UNKNOWN)
549 : return 0;
550 :
551 : /* Make sure the unwritten flag matches. */
552 103278355 : if (XFS_IS_CORRUPT(mp,
553 : (flags & XFS_RMAP_UNWRITTEN) !=
554 : (rec->rm_flags & XFS_RMAP_UNWRITTEN))) {
555 0 : xfs_btree_mark_sick(cur);
556 0 : error = -EFSCORRUPTED;
557 0 : goto out;
558 : }
559 :
560 : /* Make sure the owner matches what we expect to find in the tree. */
561 103278355 : if (XFS_IS_CORRUPT(mp, owner != rec->rm_owner)) {
562 0 : xfs_btree_mark_sick(cur);
563 0 : error = -EFSCORRUPTED;
564 0 : goto out;
565 : }
566 :
567 : /* Check the offset, if necessary. */
568 103278355 : if (XFS_RMAP_NON_INODE_OWNER(owner))
569 16376871 : goto out;
570 :
571 86901484 : if (flags & XFS_RMAP_BMBT_BLOCK) {
572 1558352 : if (XFS_IS_CORRUPT(mp,
573 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK))) {
574 0 : xfs_btree_mark_sick(cur);
575 0 : error = -EFSCORRUPTED;
576 0 : goto out;
577 : }
578 : } else {
579 85343132 : if (XFS_IS_CORRUPT(mp, rec->rm_offset > offset)) {
580 0 : xfs_btree_mark_sick(cur);
581 0 : error = -EFSCORRUPTED;
582 0 : goto out;
583 : }
584 85343132 : if (XFS_IS_CORRUPT(mp,
585 : offset + len > ltoff + rec->rm_blockcount)) {
586 0 : xfs_btree_mark_sick(cur);
587 0 : error = -EFSCORRUPTED;
588 0 : goto out;
589 : }
590 : }
591 :
592 : out:
593 : return error;
594 : }
595 :
596 : /*
597 : * Find the extent in the rmap btree and remove it.
598 : *
599 : * The record we find should always be an exact match for the extent that we're
600 : * looking for, since we insert them into the btree without modification.
601 : *
602 : * Special Case #1: when growing the filesystem, we "free" an extent when
603 : * growing the last AG. This extent is new space and so it is not tracked as
604 : * used space in the btree. The growfs code will pass in an owner of
605 : * XFS_RMAP_OWN_NULL to indicate that it expected that there is no owner of this
606 : * extent. We verify that - the extent lookup result in a record that does not
607 : * overlap.
608 : *
609 : * Special Case #2: EFIs do not record the owner of the extent, so when
610 : * recovering EFIs from the log we pass in XFS_RMAP_OWN_UNKNOWN to tell the rmap
611 : * btree to ignore the owner (i.e. wildcard match) so we don't trigger
612 : * corruption checks during log recovery.
613 : */
614 : STATIC int
615 103375396 : xfs_rmap_unmap(
616 : struct xfs_btree_cur *cur,
617 : xfs_agblock_t bno,
618 : xfs_extlen_t len,
619 : bool unwritten,
620 : const struct xfs_owner_info *oinfo)
621 : {
622 103375396 : struct xfs_mount *mp = cur->bc_mp;
623 103375396 : struct xfs_rmap_irec ltrec;
624 103375396 : uint64_t ltoff;
625 103375396 : int error = 0;
626 103375396 : int i;
627 103375396 : uint64_t owner;
628 103375396 : uint64_t offset;
629 103375396 : unsigned int flags;
630 103375396 : bool ignore_off;
631 :
632 103375396 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
633 103375396 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
634 86912451 : (flags & XFS_RMAP_BMBT_BLOCK);
635 103375396 : if (unwritten)
636 3754181 : flags |= XFS_RMAP_UNWRITTEN;
637 103375396 : trace_xfs_rmap_unmap(mp, cur->bc_ag.pag->pag_agno, bno, len,
638 : unwritten, oinfo);
639 :
640 : /*
641 : * We should always have a left record because there's a static record
642 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
643 : * will not ever be removed from the tree.
644 : */
645 103368023 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec, &i);
646 103368529 : if (error)
647 77 : goto out_error;
648 103368452 : if (XFS_IS_CORRUPT(mp, i != 1)) {
649 0 : xfs_btree_mark_sick(cur);
650 0 : error = -EFSCORRUPTED;
651 0 : goto out_error;
652 : }
653 :
654 103368452 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
655 103368452 : cur->bc_ag.pag->pag_agno, ltrec.rm_startblock,
656 : ltrec.rm_blockcount, ltrec.rm_owner,
657 : ltrec.rm_offset, ltrec.rm_flags);
658 103362898 : ltoff = ltrec.rm_offset;
659 :
660 : /*
661 : * For growfs, the incoming extent must be beyond the left record we
662 : * just found as it is new space and won't be used by anyone. This is
663 : * just a corruption check as we don't actually do anything with this
664 : * extent. Note that we need to use >= instead of > because it might
665 : * be the case that the "left" extent goes all the way to EOFS.
666 : */
667 103362898 : if (owner == XFS_RMAP_OWN_NULL) {
668 79 : if (XFS_IS_CORRUPT(mp,
669 : bno <
670 : ltrec.rm_startblock + ltrec.rm_blockcount)) {
671 0 : xfs_btree_mark_sick(cur);
672 0 : error = -EFSCORRUPTED;
673 0 : goto out_error;
674 : }
675 79 : goto out_done;
676 : }
677 :
678 : /*
679 : * If we're doing an unknown-owner removal for EFI recovery, we expect
680 : * to find the full range in the rmapbt or nothing at all. If we
681 : * don't find any rmaps overlapping either end of the range, we're
682 : * done. Hopefully this means that the EFI creator already queued
683 : * (and finished) a RUI to remove the rmap.
684 : */
685 103362819 : if (owner == XFS_RMAP_OWN_UNKNOWN &&
686 82236 : ltrec.rm_startblock + ltrec.rm_blockcount <= bno) {
687 81759 : struct xfs_rmap_irec rtrec;
688 :
689 81759 : error = xfs_btree_increment(cur, 0, &i);
690 81788 : if (error)
691 0 : goto out_error;
692 81788 : if (i == 0)
693 81893 : goto out_done;
694 81761 : error = xfs_rmap_get_rec(cur, &rtrec, &i);
695 81866 : if (error)
696 0 : goto out_error;
697 81866 : if (XFS_IS_CORRUPT(mp, i != 1)) {
698 0 : xfs_btree_mark_sick(cur);
699 0 : error = -EFSCORRUPTED;
700 0 : goto out_error;
701 : }
702 81866 : if (rtrec.rm_startblock >= bno + len)
703 81866 : goto out_done;
704 : }
705 :
706 : /* Make sure the extent we found covers the entire freeing range. */
707 103281060 : if (XFS_IS_CORRUPT(mp,
708 : ltrec.rm_startblock > bno ||
709 : ltrec.rm_startblock + ltrec.rm_blockcount <
710 : bno + len)) {
711 0 : xfs_btree_mark_sick(cur);
712 0 : error = -EFSCORRUPTED;
713 0 : goto out_error;
714 : }
715 :
716 : /* Check owner information. */
717 103281060 : error = xfs_rmap_free_check_owner(cur, ltoff, <rec, len, owner,
718 : offset, flags);
719 103278438 : if (error)
720 0 : goto out_error;
721 :
722 103278438 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
723 : /* exact match, simply remove the record from rmap tree */
724 83227656 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
725 : ltrec.rm_startblock, ltrec.rm_blockcount,
726 : ltrec.rm_owner, ltrec.rm_offset,
727 : ltrec.rm_flags);
728 83226160 : error = xfs_btree_delete(cur, &i);
729 83235398 : if (error)
730 0 : goto out_error;
731 83235398 : if (XFS_IS_CORRUPT(mp, i != 1)) {
732 0 : xfs_btree_mark_sick(cur);
733 0 : error = -EFSCORRUPTED;
734 0 : goto out_error;
735 : }
736 20050782 : } else if (ltrec.rm_startblock == bno) {
737 : /*
738 : * overlap left hand side of extent: move the start, trim the
739 : * length and update the current record.
740 : *
741 : * ltbno ltlen
742 : * Orig: |oooooooooooooooooooo|
743 : * Freeing: |fffffffff|
744 : * Result: |rrrrrrrrrr|
745 : * bno len
746 : */
747 4779535 : ltrec.rm_startblock += len;
748 4779535 : ltrec.rm_blockcount -= len;
749 4779535 : if (!ignore_off)
750 301493 : ltrec.rm_offset += len;
751 4779535 : error = xfs_rmap_update(cur, <rec);
752 4779841 : if (error)
753 0 : goto out_error;
754 15271247 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
755 : /*
756 : * overlap right hand side of extent: trim the length and update
757 : * the current record.
758 : *
759 : * ltbno ltlen
760 : * Orig: |oooooooooooooooooooo|
761 : * Freeing: |fffffffff|
762 : * Result: |rrrrrrrrrr|
763 : * bno len
764 : */
765 2280824 : ltrec.rm_blockcount -= len;
766 2280824 : error = xfs_rmap_update(cur, <rec);
767 2281126 : if (error)
768 0 : goto out_error;
769 : } else {
770 :
771 : /*
772 : * overlap middle of extent: trim the length of the existing
773 : * record to the length of the new left-extent size, increment
774 : * the insertion position so we can insert a new record
775 : * containing the remaining right-extent space.
776 : *
777 : * ltbno ltlen
778 : * Orig: |oooooooooooooooooooo|
779 : * Freeing: |fffffffff|
780 : * Result: |rrrrr| |rrrr|
781 : * bno len
782 : */
783 12990423 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
784 :
785 12990423 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
786 12990423 : error = xfs_rmap_update(cur, <rec);
787 12990424 : if (error)
788 0 : goto out_error;
789 :
790 12990424 : error = xfs_btree_increment(cur, 0, &i);
791 12990429 : if (error)
792 0 : goto out_error;
793 :
794 12990429 : cur->bc_rec.r.rm_startblock = bno + len;
795 12990429 : cur->bc_rec.r.rm_blockcount = orig_len - len -
796 12990429 : ltrec.rm_blockcount;
797 12990429 : cur->bc_rec.r.rm_owner = ltrec.rm_owner;
798 12990429 : if (ignore_off)
799 3718843 : cur->bc_rec.r.rm_offset = 0;
800 : else
801 9271586 : cur->bc_rec.r.rm_offset = offset + len;
802 12990429 : cur->bc_rec.r.rm_flags = flags;
803 12990429 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno,
804 : cur->bc_rec.r.rm_startblock,
805 : cur->bc_rec.r.rm_blockcount,
806 : cur->bc_rec.r.rm_owner,
807 : cur->bc_rec.r.rm_offset,
808 : cur->bc_rec.r.rm_flags);
809 12990425 : error = xfs_btree_insert(cur, &i);
810 12990429 : if (error)
811 0 : goto out_error;
812 : }
813 :
814 12990429 : out_done:
815 103368766 : trace_xfs_rmap_unmap_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
816 : unwritten, oinfo);
817 103361335 : out_error:
818 103361335 : if (error)
819 77 : trace_xfs_rmap_unmap_error(mp, cur->bc_ag.pag->pag_agno,
820 77 : error, _RET_IP_);
821 103361335 : return error;
822 : }
823 :
824 : #ifdef CONFIG_XFS_LIVE_HOOKS
825 : /*
826 : * Use a static key here to reduce the overhead of rmapbt live updates. If
827 : * the compiler supports jump labels, the static branch will be replaced by a
828 : * nop sled when there are no hook users. Online fsck is currently the only
829 : * caller, so this is a reasonable tradeoff.
830 : *
831 : * Note: Patching the kernel code requires taking the cpu hotplug lock. Other
832 : * parts of the kernel allocate memory with that lock held, which means that
833 : * XFS callers cannot hold any locks that might be used by memory reclaim or
834 : * writeback when calling the static_branch_{inc,dec} functions.
835 : */
836 : DEFINE_STATIC_XFS_HOOK_SWITCH(xfs_rmap_hooks_switch);
837 :
838 : void
839 15762 : xfs_rmap_hook_disable(void)
840 : {
841 15762 : xfs_hooks_switch_off(&xfs_rmap_hooks_switch);
842 15861 : }
843 :
844 : void
845 15937 : xfs_rmap_hook_enable(void)
846 : {
847 15937 : xfs_hooks_switch_on(&xfs_rmap_hooks_switch);
848 15936 : }
849 :
850 : /* Call downstream hooks for a reverse mapping update. */
851 : static inline void
852 477173034 : xfs_rmap_update_hook(
853 : struct xfs_trans *tp,
854 : struct xfs_perag *pag,
855 : enum xfs_rmap_intent_type op,
856 : xfs_agblock_t startblock,
857 : xfs_extlen_t blockcount,
858 : bool unwritten,
859 : const struct xfs_owner_info *oinfo)
860 : {
861 479103064 : if (xfs_hooks_switched_on(&xfs_rmap_hooks_switch)) {
862 1929981 : struct xfs_rmap_update_params p = {
863 : .startblock = startblock,
864 : .blockcount = blockcount,
865 : .unwritten = unwritten,
866 : .oinfo = *oinfo, /* struct copy */
867 : };
868 :
869 1929981 : if (pag)
870 1929981 : xfs_hooks_call(&pag->pag_rmap_update_hooks, op, &p);
871 : }
872 477174173 : }
873 :
874 : /* Call the specified function during a reverse mapping update. */
875 : int
876 15445 : xfs_rmap_hook_add(
877 : struct xfs_perag *pag,
878 : struct xfs_rmap_hook *hook)
879 : {
880 15445 : return xfs_hooks_add(&pag->pag_rmap_update_hooks, &hook->update_hook);
881 : }
882 :
883 : /* Stop calling the specified function during a reverse mapping update. */
884 : void
885 15516 : xfs_rmap_hook_del(
886 : struct xfs_perag *pag,
887 : struct xfs_rmap_hook *hook)
888 : {
889 15516 : xfs_hooks_del(&pag->pag_rmap_update_hooks, &hook->update_hook);
890 15343 : }
891 : #else
892 : # define xfs_rmap_update_hook(t, p, o, s, b, u, oi) do { } while(0)
893 : #endif /* CONFIG_XFS_LIVE_HOOKS */
894 :
895 : /*
896 : * Remove a reference to an extent in the rmap btree.
897 : */
898 : int
899 3057471 : xfs_rmap_free(
900 : struct xfs_trans *tp,
901 : struct xfs_buf *agbp,
902 : struct xfs_perag *pag,
903 : xfs_agblock_t bno,
904 : xfs_extlen_t len,
905 : const struct xfs_owner_info *oinfo)
906 : {
907 3057471 : struct xfs_mount *mp = tp->t_mountp;
908 3057471 : struct xfs_btree_cur *cur;
909 3057471 : int error;
910 :
911 3057471 : if (!xfs_has_rmapbt(mp))
912 : return 0;
913 :
914 2418919 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
915 2419374 : xfs_rmap_update_hook(tp, pag, XFS_RMAP_UNMAP, bno, len, false, oinfo);
916 2419078 : error = xfs_rmap_unmap(cur, bno, len, false, oinfo);
917 :
918 2418771 : xfs_btree_del_cursor(cur, error);
919 2418771 : return error;
920 : }
921 :
922 : /*
923 : * A mergeable rmap must have the same owner and the same values for
924 : * the unwritten, attr_fork, and bmbt flags. The startblock and
925 : * offset are checked separately.
926 : */
927 : static bool
928 261929637 : xfs_rmap_is_mergeable(
929 : struct xfs_rmap_irec *irec,
930 : uint64_t owner,
931 : unsigned int flags)
932 : {
933 261929637 : if (irec->rm_owner == XFS_RMAP_OWN_NULL)
934 : return false;
935 261929637 : if (irec->rm_owner != owner)
936 : return false;
937 176530407 : if ((flags & XFS_RMAP_UNWRITTEN) ^
938 176530407 : (irec->rm_flags & XFS_RMAP_UNWRITTEN))
939 : return false;
940 166449706 : if ((flags & XFS_RMAP_ATTR_FORK) ^
941 : (irec->rm_flags & XFS_RMAP_ATTR_FORK))
942 : return false;
943 166442243 : if ((flags & XFS_RMAP_BMBT_BLOCK) ^
944 : (irec->rm_flags & XFS_RMAP_BMBT_BLOCK))
945 295102 : return false;
946 : return true;
947 : }
948 :
949 : /*
950 : * When we allocate a new block, the first thing we do is add a reference to
951 : * the extent in the rmap btree. This takes the form of a [agbno, length,
952 : * owner, offset] record. Flags are encoded in the high bits of the offset
953 : * field.
954 : */
955 : STATIC int
956 119160365 : xfs_rmap_map(
957 : struct xfs_btree_cur *cur,
958 : xfs_agblock_t bno,
959 : xfs_extlen_t len,
960 : bool unwritten,
961 : const struct xfs_owner_info *oinfo)
962 : {
963 119160365 : struct xfs_mount *mp = cur->bc_mp;
964 119160365 : struct xfs_rmap_irec ltrec;
965 119160365 : struct xfs_rmap_irec gtrec;
966 119160365 : int have_gt;
967 119160365 : int have_lt;
968 119160365 : int error = 0;
969 119160365 : int i;
970 119160365 : uint64_t owner;
971 119160365 : uint64_t offset;
972 119160365 : unsigned int flags = 0;
973 119160365 : bool ignore_off;
974 :
975 119160365 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
976 119160365 : ASSERT(owner != 0);
977 119160365 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
978 112495398 : (flags & XFS_RMAP_BMBT_BLOCK);
979 119160365 : if (unwritten)
980 32380347 : flags |= XFS_RMAP_UNWRITTEN;
981 119160365 : trace_xfs_rmap_map(mp, cur->bc_ag.pag->pag_agno, bno, len,
982 : unwritten, oinfo);
983 119154971 : ASSERT(!xfs_rmap_should_skip_owner_update(oinfo));
984 :
985 : /*
986 : * For the initial lookup, look for an exact match or the left-adjacent
987 : * record for our insertion point. This will also give us the record for
988 : * start block contiguity tests.
989 : */
990 119154971 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec,
991 : &have_lt);
992 119158788 : if (error)
993 836 : goto out_error;
994 119157952 : if (have_lt) {
995 119125213 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
996 119125213 : cur->bc_ag.pag->pag_agno, ltrec.rm_startblock,
997 : ltrec.rm_blockcount, ltrec.rm_owner,
998 : ltrec.rm_offset, ltrec.rm_flags);
999 :
1000 119121789 : if (!xfs_rmap_is_mergeable(<rec, owner, flags))
1001 46005117 : have_lt = 0;
1002 : }
1003 :
1004 119154528 : if (XFS_IS_CORRUPT(mp,
1005 : have_lt != 0 &&
1006 : ltrec.rm_startblock + ltrec.rm_blockcount > bno)) {
1007 0 : xfs_btree_mark_sick(cur);
1008 0 : error = -EFSCORRUPTED;
1009 0 : goto out_error;
1010 : }
1011 :
1012 : /*
1013 : * Increment the cursor to see if we have a right-adjacent record to our
1014 : * insertion point. This will give us the record for end block
1015 : * contiguity tests.
1016 : */
1017 119154528 : error = xfs_btree_increment(cur, 0, &have_gt);
1018 119161285 : if (error)
1019 19 : goto out_error;
1020 119161266 : if (have_gt) {
1021 105804517 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
1022 105804939 : if (error)
1023 0 : goto out_error;
1024 105804939 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
1025 0 : xfs_btree_mark_sick(cur);
1026 0 : error = -EFSCORRUPTED;
1027 0 : goto out_error;
1028 : }
1029 105804939 : if (XFS_IS_CORRUPT(mp, bno + len > gtrec.rm_startblock)) {
1030 0 : xfs_btree_mark_sick(cur);
1031 0 : error = -EFSCORRUPTED;
1032 0 : goto out_error;
1033 : }
1034 105804939 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1035 105804939 : cur->bc_ag.pag->pag_agno, gtrec.rm_startblock,
1036 : gtrec.rm_blockcount, gtrec.rm_owner,
1037 : gtrec.rm_offset, gtrec.rm_flags);
1038 105803583 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
1039 47626091 : have_gt = 0;
1040 : }
1041 :
1042 : /*
1043 : * Note: cursor currently points one record to the right of ltrec, even
1044 : * if there is no record in the tree to the right.
1045 : */
1046 119160332 : if (have_lt &&
1047 73119259 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
1048 12860664 : (ignore_off || ltrec.rm_offset + ltrec.rm_blockcount == offset)) {
1049 : /*
1050 : * left edge contiguous, merge into left record.
1051 : *
1052 : * ltbno ltlen
1053 : * orig: |ooooooooo|
1054 : * adding: |aaaaaaaaa|
1055 : * result: |rrrrrrrrrrrrrrrrrrr|
1056 : * bno len
1057 : */
1058 11311113 : ltrec.rm_blockcount += len;
1059 11311113 : if (have_gt &&
1060 3909205 : bno + len == gtrec.rm_startblock &&
1061 6847 : (ignore_off || offset + len == gtrec.rm_offset) &&
1062 175580 : (unsigned long)ltrec.rm_blockcount + len +
1063 175580 : gtrec.rm_blockcount <= XFS_RMAP_LEN_MAX) {
1064 : /*
1065 : * right edge also contiguous, delete right record
1066 : * and merge into left record.
1067 : *
1068 : * ltbno ltlen gtbno gtlen
1069 : * orig: |ooooooooo| |ooooooooo|
1070 : * adding: |aaaaaaaaa|
1071 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
1072 : */
1073 175580 : ltrec.rm_blockcount += gtrec.rm_blockcount;
1074 175580 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1075 : gtrec.rm_startblock,
1076 : gtrec.rm_blockcount,
1077 : gtrec.rm_owner,
1078 : gtrec.rm_offset,
1079 : gtrec.rm_flags);
1080 175574 : error = xfs_btree_delete(cur, &i);
1081 175593 : if (error)
1082 0 : goto out_error;
1083 175593 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1084 0 : xfs_btree_mark_sick(cur);
1085 0 : error = -EFSCORRUPTED;
1086 0 : goto out_error;
1087 : }
1088 : }
1089 :
1090 : /* point the cursor back to the left record and update */
1091 11311126 : error = xfs_btree_decrement(cur, 0, &have_gt);
1092 11311232 : if (error)
1093 0 : goto out_error;
1094 11311232 : error = xfs_rmap_update(cur, <rec);
1095 11311262 : if (error)
1096 0 : goto out_error;
1097 107849219 : } else if (have_gt &&
1098 54268193 : bno + len == gtrec.rm_startblock &&
1099 1235035 : (ignore_off || offset + len == gtrec.rm_offset)) {
1100 : /*
1101 : * right edge contiguous, merge into right record.
1102 : *
1103 : * gtbno gtlen
1104 : * Orig: |ooooooooo|
1105 : * adding: |aaaaaaaaa|
1106 : * Result: |rrrrrrrrrrrrrrrrrrr|
1107 : * bno len
1108 : */
1109 153869 : gtrec.rm_startblock = bno;
1110 153869 : gtrec.rm_blockcount += len;
1111 153869 : if (!ignore_off)
1112 6748 : gtrec.rm_offset = offset;
1113 153869 : error = xfs_rmap_update(cur, >rec);
1114 153871 : if (error)
1115 0 : goto out_error;
1116 : } else {
1117 : /*
1118 : * no contiguous edge with identical owner, insert
1119 : * new record at current cursor position.
1120 : */
1121 107695350 : cur->bc_rec.r.rm_startblock = bno;
1122 107695350 : cur->bc_rec.r.rm_blockcount = len;
1123 107695350 : cur->bc_rec.r.rm_owner = owner;
1124 107695350 : cur->bc_rec.r.rm_offset = offset;
1125 107695350 : cur->bc_rec.r.rm_flags = flags;
1126 107695350 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1127 : owner, offset, flags);
1128 107692448 : error = xfs_btree_insert(cur, &i);
1129 107696034 : if (error)
1130 60 : goto out_error;
1131 107695974 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1132 0 : xfs_btree_mark_sick(cur);
1133 0 : error = -EFSCORRUPTED;
1134 0 : goto out_error;
1135 : }
1136 : }
1137 :
1138 119161107 : trace_xfs_rmap_map_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1139 : unwritten, oinfo);
1140 119158995 : out_error:
1141 119158995 : if (error)
1142 915 : trace_xfs_rmap_map_error(mp, cur->bc_ag.pag->pag_agno,
1143 916 : error, _RET_IP_);
1144 119158994 : return error;
1145 : }
1146 :
1147 : /*
1148 : * Add a reference to an extent in the rmap btree.
1149 : */
1150 : int
1151 5531586 : xfs_rmap_alloc(
1152 : struct xfs_trans *tp,
1153 : struct xfs_buf *agbp,
1154 : struct xfs_perag *pag,
1155 : xfs_agblock_t bno,
1156 : xfs_extlen_t len,
1157 : const struct xfs_owner_info *oinfo)
1158 : {
1159 5531586 : struct xfs_mount *mp = tp->t_mountp;
1160 5531586 : struct xfs_btree_cur *cur;
1161 5531586 : int error;
1162 :
1163 5531586 : if (!xfs_has_rmapbt(mp))
1164 : return 0;
1165 :
1166 4146932 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
1167 4147154 : xfs_rmap_update_hook(tp, pag, XFS_RMAP_MAP, bno, len, false, oinfo);
1168 4147075 : error = xfs_rmap_map(cur, bno, len, false, oinfo);
1169 :
1170 4146857 : xfs_btree_del_cursor(cur, error);
1171 4146857 : return error;
1172 : }
1173 :
1174 : #define RMAP_LEFT_CONTIG (1 << 0)
1175 : #define RMAP_RIGHT_CONTIG (1 << 1)
1176 : #define RMAP_LEFT_FILLING (1 << 2)
1177 : #define RMAP_RIGHT_FILLING (1 << 3)
1178 : #define RMAP_LEFT_VALID (1 << 6)
1179 : #define RMAP_RIGHT_VALID (1 << 7)
1180 :
1181 : #define LEFT r[0]
1182 : #define RIGHT r[1]
1183 : #define PREV r[2]
1184 : #define NEW r[3]
1185 :
1186 : /*
1187 : * Convert an unwritten extent to a real extent or vice versa.
1188 : * Does not handle overlapping extents.
1189 : */
1190 : STATIC int
1191 12960043 : xfs_rmap_convert(
1192 : struct xfs_btree_cur *cur,
1193 : xfs_agblock_t bno,
1194 : xfs_extlen_t len,
1195 : bool unwritten,
1196 : const struct xfs_owner_info *oinfo)
1197 : {
1198 12960043 : struct xfs_mount *mp = cur->bc_mp;
1199 12960043 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1200 : /* left is 0, right is 1, */
1201 : /* prev is 2, new is 3 */
1202 12960043 : uint64_t owner;
1203 12960043 : uint64_t offset;
1204 12960043 : uint64_t new_endoff;
1205 12960043 : unsigned int oldext;
1206 12960043 : unsigned int newext;
1207 12960043 : unsigned int flags = 0;
1208 12960043 : int i;
1209 12960043 : int state = 0;
1210 12960043 : int error;
1211 :
1212 12960043 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1213 12960043 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1214 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1215 12960043 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1216 12960043 : new_endoff = offset + len;
1217 12960043 : trace_xfs_rmap_convert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1218 : unwritten, oinfo);
1219 :
1220 : /*
1221 : * For the initial lookup, look for an exact match or the left-adjacent
1222 : * record for our insertion point. This will also give us the record for
1223 : * start block contiguity tests.
1224 : */
1225 12959976 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, &PREV, &i);
1226 12959906 : if (error)
1227 0 : goto done;
1228 12959906 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1229 0 : xfs_btree_mark_sick(cur);
1230 0 : error = -EFSCORRUPTED;
1231 0 : goto done;
1232 : }
1233 :
1234 12959906 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
1235 12959906 : cur->bc_ag.pag->pag_agno, PREV.rm_startblock,
1236 : PREV.rm_blockcount, PREV.rm_owner,
1237 : PREV.rm_offset, PREV.rm_flags);
1238 :
1239 12959699 : ASSERT(PREV.rm_offset <= offset);
1240 12959699 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1241 12959699 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1242 12959699 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1243 :
1244 : /*
1245 : * Set flags determining what part of the previous oldext allocation
1246 : * extent is being replaced by a newext allocation.
1247 : */
1248 12959699 : if (PREV.rm_offset == offset)
1249 11619894 : state |= RMAP_LEFT_FILLING;
1250 12959699 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1251 11224564 : state |= RMAP_RIGHT_FILLING;
1252 :
1253 : /*
1254 : * Decrement the cursor to see if we have a left-adjacent record to our
1255 : * insertion point. This will give us the record for end block
1256 : * contiguity tests.
1257 : */
1258 12959699 : error = xfs_btree_decrement(cur, 0, &i);
1259 12959979 : if (error)
1260 0 : goto done;
1261 12959979 : if (i) {
1262 12959980 : state |= RMAP_LEFT_VALID;
1263 12959980 : error = xfs_rmap_get_rec(cur, &LEFT, &i);
1264 12960119 : if (error)
1265 0 : goto done;
1266 12960119 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1267 0 : xfs_btree_mark_sick(cur);
1268 0 : error = -EFSCORRUPTED;
1269 0 : goto done;
1270 : }
1271 12960119 : if (XFS_IS_CORRUPT(mp,
1272 : LEFT.rm_startblock + LEFT.rm_blockcount >
1273 : bno)) {
1274 0 : xfs_btree_mark_sick(cur);
1275 0 : error = -EFSCORRUPTED;
1276 0 : goto done;
1277 : }
1278 12960119 : trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
1279 12960119 : cur->bc_ag.pag->pag_agno, LEFT.rm_startblock,
1280 : LEFT.rm_blockcount, LEFT.rm_owner,
1281 : LEFT.rm_offset, LEFT.rm_flags);
1282 12959948 : if (LEFT.rm_startblock + LEFT.rm_blockcount == bno &&
1283 9702816 : LEFT.rm_offset + LEFT.rm_blockcount == offset &&
1284 925267 : xfs_rmap_is_mergeable(&LEFT, owner, newext))
1285 918498 : state |= RMAP_LEFT_CONTIG;
1286 : }
1287 :
1288 : /*
1289 : * Increment the cursor to see if we have a right-adjacent record to our
1290 : * insertion point. This will give us the record for end block
1291 : * contiguity tests.
1292 : */
1293 12959947 : error = xfs_btree_increment(cur, 0, &i);
1294 12960127 : if (error)
1295 0 : goto done;
1296 12960127 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1297 0 : xfs_btree_mark_sick(cur);
1298 0 : error = -EFSCORRUPTED;
1299 0 : goto done;
1300 : }
1301 12960127 : error = xfs_btree_increment(cur, 0, &i);
1302 12960117 : if (error)
1303 0 : goto done;
1304 12960117 : if (i) {
1305 11635017 : state |= RMAP_RIGHT_VALID;
1306 11635017 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1307 11635102 : if (error)
1308 0 : goto done;
1309 11635102 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1310 0 : xfs_btree_mark_sick(cur);
1311 0 : error = -EFSCORRUPTED;
1312 0 : goto done;
1313 : }
1314 11635102 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1315 0 : xfs_btree_mark_sick(cur);
1316 0 : error = -EFSCORRUPTED;
1317 0 : goto done;
1318 : }
1319 11635102 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1320 11635102 : cur->bc_ag.pag->pag_agno, RIGHT.rm_startblock,
1321 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1322 : RIGHT.rm_offset, RIGHT.rm_flags);
1323 11635004 : if (bno + len == RIGHT.rm_startblock &&
1324 7151478 : offset + len == RIGHT.rm_offset &&
1325 174764 : xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1326 171083 : state |= RMAP_RIGHT_CONTIG;
1327 : }
1328 :
1329 : /* check that left + prev + right is not too long */
1330 12960104 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1331 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1332 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1333 76904 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1334 76904 : (unsigned long)LEFT.rm_blockcount + len +
1335 76904 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1336 0 : state &= ~RMAP_RIGHT_CONTIG;
1337 :
1338 12960003 : trace_xfs_rmap_convert_state(mp, cur->bc_ag.pag->pag_agno, state,
1339 12960104 : _RET_IP_);
1340 :
1341 : /* reset the cursor back to PREV */
1342 12959941 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, NULL, &i);
1343 12960018 : if (error)
1344 0 : goto done;
1345 12960018 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1346 0 : xfs_btree_mark_sick(cur);
1347 0 : error = -EFSCORRUPTED;
1348 0 : goto done;
1349 : }
1350 :
1351 : /*
1352 : * Switch out based on the FILLING and CONTIG state bits.
1353 : */
1354 12960018 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1355 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1356 76904 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1357 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1358 : /*
1359 : * Setting all of a previous oldext extent to newext.
1360 : * The left and right neighbors are both contiguous with new.
1361 : */
1362 76904 : error = xfs_btree_increment(cur, 0, &i);
1363 76904 : if (error)
1364 0 : goto done;
1365 76904 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1366 0 : xfs_btree_mark_sick(cur);
1367 0 : error = -EFSCORRUPTED;
1368 0 : goto done;
1369 : }
1370 76904 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1371 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1372 : RIGHT.rm_owner, RIGHT.rm_offset,
1373 : RIGHT.rm_flags);
1374 76904 : error = xfs_btree_delete(cur, &i);
1375 76904 : if (error)
1376 0 : goto done;
1377 76904 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1378 0 : xfs_btree_mark_sick(cur);
1379 0 : error = -EFSCORRUPTED;
1380 0 : goto done;
1381 : }
1382 76904 : error = xfs_btree_decrement(cur, 0, &i);
1383 76904 : if (error)
1384 0 : goto done;
1385 76904 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1386 0 : xfs_btree_mark_sick(cur);
1387 0 : error = -EFSCORRUPTED;
1388 0 : goto done;
1389 : }
1390 76904 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1391 : PREV.rm_startblock, PREV.rm_blockcount,
1392 : PREV.rm_owner, PREV.rm_offset,
1393 : PREV.rm_flags);
1394 76904 : error = xfs_btree_delete(cur, &i);
1395 76904 : if (error)
1396 0 : goto done;
1397 76904 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1398 0 : xfs_btree_mark_sick(cur);
1399 0 : error = -EFSCORRUPTED;
1400 0 : goto done;
1401 : }
1402 76904 : error = xfs_btree_decrement(cur, 0, &i);
1403 76904 : if (error)
1404 0 : goto done;
1405 76904 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1406 0 : xfs_btree_mark_sick(cur);
1407 0 : error = -EFSCORRUPTED;
1408 0 : goto done;
1409 : }
1410 76904 : NEW = LEFT;
1411 76904 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1412 76904 : error = xfs_rmap_update(cur, &NEW);
1413 76904 : if (error)
1414 0 : goto done;
1415 : break;
1416 :
1417 564230 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1418 : /*
1419 : * Setting all of a previous oldext extent to newext.
1420 : * The left neighbor is contiguous, the right is not.
1421 : */
1422 564230 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1423 : PREV.rm_startblock, PREV.rm_blockcount,
1424 : PREV.rm_owner, PREV.rm_offset,
1425 : PREV.rm_flags);
1426 564228 : error = xfs_btree_delete(cur, &i);
1427 564230 : if (error)
1428 0 : goto done;
1429 564230 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1430 0 : xfs_btree_mark_sick(cur);
1431 0 : error = -EFSCORRUPTED;
1432 0 : goto done;
1433 : }
1434 564230 : error = xfs_btree_decrement(cur, 0, &i);
1435 564229 : if (error)
1436 0 : goto done;
1437 564229 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1438 0 : xfs_btree_mark_sick(cur);
1439 0 : error = -EFSCORRUPTED;
1440 0 : goto done;
1441 : }
1442 564229 : NEW = LEFT;
1443 564229 : NEW.rm_blockcount += PREV.rm_blockcount;
1444 564229 : error = xfs_rmap_update(cur, &NEW);
1445 564230 : if (error)
1446 0 : goto done;
1447 : break;
1448 :
1449 21089 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1450 : /*
1451 : * Setting all of a previous oldext extent to newext.
1452 : * The right neighbor is contiguous, the left is not.
1453 : */
1454 21089 : error = xfs_btree_increment(cur, 0, &i);
1455 21089 : if (error)
1456 0 : goto done;
1457 21089 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1458 0 : xfs_btree_mark_sick(cur);
1459 0 : error = -EFSCORRUPTED;
1460 0 : goto done;
1461 : }
1462 21089 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1463 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1464 : RIGHT.rm_owner, RIGHT.rm_offset,
1465 : RIGHT.rm_flags);
1466 21089 : error = xfs_btree_delete(cur, &i);
1467 21089 : if (error)
1468 0 : goto done;
1469 21089 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1470 0 : xfs_btree_mark_sick(cur);
1471 0 : error = -EFSCORRUPTED;
1472 0 : goto done;
1473 : }
1474 21089 : error = xfs_btree_decrement(cur, 0, &i);
1475 21089 : if (error)
1476 0 : goto done;
1477 21089 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1478 0 : xfs_btree_mark_sick(cur);
1479 0 : error = -EFSCORRUPTED;
1480 0 : goto done;
1481 : }
1482 21089 : NEW = PREV;
1483 21089 : NEW.rm_blockcount = len + RIGHT.rm_blockcount;
1484 21089 : NEW.rm_flags = newext;
1485 21089 : error = xfs_rmap_update(cur, &NEW);
1486 21089 : if (error)
1487 0 : goto done;
1488 : break;
1489 :
1490 10414471 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1491 : /*
1492 : * Setting all of a previous oldext extent to newext.
1493 : * Neither the left nor right neighbors are contiguous with
1494 : * the new one.
1495 : */
1496 10414471 : NEW = PREV;
1497 10414471 : NEW.rm_flags = newext;
1498 10414471 : error = xfs_rmap_update(cur, &NEW);
1499 10414543 : if (error)
1500 0 : goto done;
1501 : break;
1502 :
1503 277367 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1504 : /*
1505 : * Setting the first part of a previous oldext extent to newext.
1506 : * The left neighbor is contiguous.
1507 : */
1508 277367 : NEW = PREV;
1509 277367 : NEW.rm_offset += len;
1510 277367 : NEW.rm_startblock += len;
1511 277367 : NEW.rm_blockcount -= len;
1512 277367 : error = xfs_rmap_update(cur, &NEW);
1513 277367 : if (error)
1514 0 : goto done;
1515 277367 : error = xfs_btree_decrement(cur, 0, &i);
1516 277367 : if (error)
1517 0 : goto done;
1518 277367 : NEW = LEFT;
1519 277367 : NEW.rm_blockcount += len;
1520 277367 : error = xfs_rmap_update(cur, &NEW);
1521 277367 : if (error)
1522 0 : goto done;
1523 : break;
1524 :
1525 266102 : case RMAP_LEFT_FILLING:
1526 : /*
1527 : * Setting the first part of a previous oldext extent to newext.
1528 : * The left neighbor is not contiguous.
1529 : */
1530 266102 : NEW = PREV;
1531 266102 : NEW.rm_startblock += len;
1532 266102 : NEW.rm_offset += len;
1533 266102 : NEW.rm_blockcount -= len;
1534 266102 : error = xfs_rmap_update(cur, &NEW);
1535 266103 : if (error)
1536 0 : goto done;
1537 266103 : NEW.rm_startblock = bno;
1538 266103 : NEW.rm_owner = owner;
1539 266103 : NEW.rm_offset = offset;
1540 266103 : NEW.rm_blockcount = len;
1541 266103 : NEW.rm_flags = newext;
1542 266103 : cur->bc_rec.r = NEW;
1543 266103 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno,
1544 : len, owner, offset, newext);
1545 266101 : error = xfs_btree_insert(cur, &i);
1546 266104 : if (error)
1547 0 : goto done;
1548 266104 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1549 0 : xfs_btree_mark_sick(cur);
1550 0 : error = -EFSCORRUPTED;
1551 0 : goto done;
1552 : }
1553 : break;
1554 :
1555 73090 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1556 : /*
1557 : * Setting the last part of a previous oldext extent to newext.
1558 : * The right neighbor is contiguous with the new allocation.
1559 : */
1560 73090 : NEW = PREV;
1561 73090 : NEW.rm_blockcount -= len;
1562 73090 : error = xfs_rmap_update(cur, &NEW);
1563 73090 : if (error)
1564 0 : goto done;
1565 73090 : error = xfs_btree_increment(cur, 0, &i);
1566 73090 : if (error)
1567 0 : goto done;
1568 73090 : NEW = RIGHT;
1569 73090 : NEW.rm_offset = offset;
1570 73090 : NEW.rm_startblock = bno;
1571 73090 : NEW.rm_blockcount += len;
1572 73090 : error = xfs_rmap_update(cur, &NEW);
1573 73090 : if (error)
1574 0 : goto done;
1575 : break;
1576 :
1577 75020 : case RMAP_RIGHT_FILLING:
1578 : /*
1579 : * Setting the last part of a previous oldext extent to newext.
1580 : * The right neighbor is not contiguous.
1581 : */
1582 75020 : NEW = PREV;
1583 75020 : NEW.rm_blockcount -= len;
1584 75020 : error = xfs_rmap_update(cur, &NEW);
1585 75020 : if (error)
1586 0 : goto done;
1587 75020 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1588 : oldext, &i);
1589 75020 : if (error)
1590 0 : goto done;
1591 75020 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1592 0 : xfs_btree_mark_sick(cur);
1593 0 : error = -EFSCORRUPTED;
1594 0 : goto done;
1595 : }
1596 75020 : NEW.rm_startblock = bno;
1597 75020 : NEW.rm_owner = owner;
1598 75020 : NEW.rm_offset = offset;
1599 75020 : NEW.rm_blockcount = len;
1600 75020 : NEW.rm_flags = newext;
1601 75020 : cur->bc_rec.r = NEW;
1602 75020 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno,
1603 : len, owner, offset, newext);
1604 75020 : error = xfs_btree_insert(cur, &i);
1605 75020 : if (error)
1606 0 : goto done;
1607 75020 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1608 0 : xfs_btree_mark_sick(cur);
1609 0 : error = -EFSCORRUPTED;
1610 0 : goto done;
1611 : }
1612 : break;
1613 :
1614 1191745 : case 0:
1615 : /*
1616 : * Setting the middle part of a previous oldext extent to
1617 : * newext. Contiguity is impossible here.
1618 : * One extent becomes three extents.
1619 : */
1620 : /* new right extent - oldext */
1621 1191745 : NEW.rm_startblock = bno + len;
1622 1191745 : NEW.rm_owner = owner;
1623 1191745 : NEW.rm_offset = new_endoff;
1624 1191745 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1625 : new_endoff;
1626 1191745 : NEW.rm_flags = PREV.rm_flags;
1627 1191745 : error = xfs_rmap_update(cur, &NEW);
1628 1191745 : if (error)
1629 0 : goto done;
1630 : /* new left extent - oldext */
1631 1191745 : NEW = PREV;
1632 1191745 : NEW.rm_blockcount = offset - PREV.rm_offset;
1633 1191745 : cur->bc_rec.r = NEW;
1634 1191745 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno,
1635 : NEW.rm_startblock, NEW.rm_blockcount,
1636 : NEW.rm_owner, NEW.rm_offset,
1637 : NEW.rm_flags);
1638 1191745 : error = xfs_btree_insert(cur, &i);
1639 1191745 : if (error)
1640 0 : goto done;
1641 1191745 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1642 0 : xfs_btree_mark_sick(cur);
1643 0 : error = -EFSCORRUPTED;
1644 0 : goto done;
1645 : }
1646 : /*
1647 : * Reset the cursor to the position of the new extent
1648 : * we are about to insert as we can't trust it after
1649 : * the previous insert.
1650 : */
1651 1191745 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1652 : oldext, &i);
1653 1191745 : if (error)
1654 0 : goto done;
1655 1191745 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1656 0 : xfs_btree_mark_sick(cur);
1657 0 : error = -EFSCORRUPTED;
1658 0 : goto done;
1659 : }
1660 : /* new middle extent - newext */
1661 1191745 : cur->bc_rec.r.rm_flags &= ~XFS_RMAP_UNWRITTEN;
1662 1191745 : cur->bc_rec.r.rm_flags |= newext;
1663 1191745 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1664 : owner, offset, newext);
1665 1191745 : error = xfs_btree_insert(cur, &i);
1666 1191745 : if (error)
1667 0 : goto done;
1668 1191745 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1669 0 : xfs_btree_mark_sick(cur);
1670 0 : error = -EFSCORRUPTED;
1671 0 : goto done;
1672 : }
1673 : break;
1674 :
1675 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1676 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1677 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1678 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1679 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1680 : case RMAP_LEFT_CONTIG:
1681 : case RMAP_RIGHT_CONTIG:
1682 : /*
1683 : * These cases are all impossible.
1684 : */
1685 0 : ASSERT(0);
1686 : }
1687 :
1688 12960092 : trace_xfs_rmap_convert_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1689 : unwritten, oinfo);
1690 12959917 : done:
1691 12959917 : if (error)
1692 0 : trace_xfs_rmap_convert_error(cur->bc_mp,
1693 0 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
1694 12959917 : return error;
1695 : }
1696 :
1697 : /*
1698 : * Convert an unwritten extent to a real extent or vice versa. If there is no
1699 : * possibility of overlapping extents, delegate to the simpler convert
1700 : * function.
1701 : */
1702 : STATIC int
1703 18877934 : xfs_rmap_convert_shared(
1704 : struct xfs_btree_cur *cur,
1705 : xfs_agblock_t bno,
1706 : xfs_extlen_t len,
1707 : bool unwritten,
1708 : const struct xfs_owner_info *oinfo)
1709 : {
1710 18877934 : struct xfs_mount *mp = cur->bc_mp;
1711 18877934 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1712 : /* left is 0, right is 1, */
1713 : /* prev is 2, new is 3 */
1714 18877934 : uint64_t owner;
1715 18877934 : uint64_t offset;
1716 18877934 : uint64_t new_endoff;
1717 18877934 : unsigned int oldext;
1718 18877934 : unsigned int newext;
1719 18877934 : unsigned int flags = 0;
1720 18877934 : int i;
1721 18877934 : int state = 0;
1722 18877934 : int error;
1723 :
1724 18877934 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1725 18877934 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1726 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1727 18877934 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1728 18877934 : new_endoff = offset + len;
1729 18877934 : trace_xfs_rmap_convert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1730 : unwritten, oinfo);
1731 :
1732 : /*
1733 : * For the initial lookup, look for and exact match or the left-adjacent
1734 : * record for our insertion point. This will also give us the record for
1735 : * start block contiguity tests.
1736 : */
1737 18877928 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, oldext,
1738 : &PREV, &i);
1739 18877927 : if (error)
1740 2 : goto done;
1741 18877925 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1742 0 : xfs_btree_mark_sick(cur);
1743 0 : error = -EFSCORRUPTED;
1744 0 : goto done;
1745 : }
1746 :
1747 18877925 : ASSERT(PREV.rm_offset <= offset);
1748 18877925 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1749 18877925 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1750 18877925 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1751 :
1752 : /*
1753 : * Set flags determining what part of the previous oldext allocation
1754 : * extent is being replaced by a newext allocation.
1755 : */
1756 18877925 : if (PREV.rm_offset == offset)
1757 17367837 : state |= RMAP_LEFT_FILLING;
1758 18877925 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1759 13532345 : state |= RMAP_RIGHT_FILLING;
1760 :
1761 : /* Is there a left record that abuts our range? */
1762 18877925 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, newext,
1763 : &LEFT, &i);
1764 18877941 : if (error)
1765 0 : goto done;
1766 18877941 : if (i) {
1767 741140 : state |= RMAP_LEFT_VALID;
1768 741140 : if (XFS_IS_CORRUPT(mp,
1769 : LEFT.rm_startblock + LEFT.rm_blockcount >
1770 : bno)) {
1771 0 : xfs_btree_mark_sick(cur);
1772 0 : error = -EFSCORRUPTED;
1773 0 : goto done;
1774 : }
1775 741140 : if (xfs_rmap_is_mergeable(&LEFT, owner, newext))
1776 741135 : state |= RMAP_LEFT_CONTIG;
1777 : }
1778 :
1779 : /* Is there a right record that abuts our range? */
1780 18877941 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
1781 : newext, &i);
1782 18877940 : if (error)
1783 0 : goto done;
1784 18877940 : if (i) {
1785 467356 : state |= RMAP_RIGHT_VALID;
1786 467356 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1787 467356 : if (error)
1788 0 : goto done;
1789 467356 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1790 0 : xfs_btree_mark_sick(cur);
1791 0 : error = -EFSCORRUPTED;
1792 0 : goto done;
1793 : }
1794 467356 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1795 0 : xfs_btree_mark_sick(cur);
1796 0 : error = -EFSCORRUPTED;
1797 0 : goto done;
1798 : }
1799 467356 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1800 467356 : cur->bc_ag.pag->pag_agno, RIGHT.rm_startblock,
1801 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1802 : RIGHT.rm_offset, RIGHT.rm_flags);
1803 467356 : if (xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1804 467356 : state |= RMAP_RIGHT_CONTIG;
1805 : }
1806 :
1807 : /* check that left + prev + right is not too long */
1808 18877940 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1809 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1810 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1811 109901 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1812 109901 : (unsigned long)LEFT.rm_blockcount + len +
1813 109901 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1814 0 : state &= ~RMAP_RIGHT_CONTIG;
1815 :
1816 18877940 : trace_xfs_rmap_convert_state(mp, cur->bc_ag.pag->pag_agno, state,
1817 18877940 : _RET_IP_);
1818 : /*
1819 : * Switch out based on the FILLING and CONTIG state bits.
1820 : */
1821 18877939 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1822 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1823 109901 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1824 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1825 : /*
1826 : * Setting all of a previous oldext extent to newext.
1827 : * The left and right neighbors are both contiguous with new.
1828 : */
1829 109901 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1830 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1831 : RIGHT.rm_offset, RIGHT.rm_flags);
1832 109901 : if (error)
1833 0 : goto done;
1834 109901 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1835 : PREV.rm_blockcount, PREV.rm_owner,
1836 : PREV.rm_offset, PREV.rm_flags);
1837 109901 : if (error)
1838 0 : goto done;
1839 109901 : NEW = LEFT;
1840 109901 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1841 : NEW.rm_blockcount, NEW.rm_owner,
1842 : NEW.rm_offset, NEW.rm_flags, &i);
1843 109901 : if (error)
1844 0 : goto done;
1845 109901 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1846 0 : xfs_btree_mark_sick(cur);
1847 0 : error = -EFSCORRUPTED;
1848 0 : goto done;
1849 : }
1850 109901 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1851 109901 : error = xfs_rmap_update(cur, &NEW);
1852 109901 : if (error)
1853 0 : goto done;
1854 : break;
1855 :
1856 433189 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1857 : /*
1858 : * Setting all of a previous oldext extent to newext.
1859 : * The left neighbor is contiguous, the right is not.
1860 : */
1861 433189 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1862 : PREV.rm_blockcount, PREV.rm_owner,
1863 : PREV.rm_offset, PREV.rm_flags);
1864 433189 : if (error)
1865 0 : goto done;
1866 433189 : NEW = LEFT;
1867 433189 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1868 : NEW.rm_blockcount, NEW.rm_owner,
1869 : NEW.rm_offset, NEW.rm_flags, &i);
1870 433189 : if (error)
1871 0 : goto done;
1872 433189 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1873 0 : xfs_btree_mark_sick(cur);
1874 0 : error = -EFSCORRUPTED;
1875 0 : goto done;
1876 : }
1877 433189 : NEW.rm_blockcount += PREV.rm_blockcount;
1878 433189 : error = xfs_rmap_update(cur, &NEW);
1879 433189 : if (error)
1880 0 : goto done;
1881 : break;
1882 :
1883 263788 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1884 : /*
1885 : * Setting all of a previous oldext extent to newext.
1886 : * The right neighbor is contiguous, the left is not.
1887 : */
1888 263788 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1889 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1890 : RIGHT.rm_offset, RIGHT.rm_flags);
1891 263788 : if (error)
1892 0 : goto done;
1893 263788 : NEW = PREV;
1894 263788 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1895 : NEW.rm_blockcount, NEW.rm_owner,
1896 : NEW.rm_offset, NEW.rm_flags, &i);
1897 263788 : if (error)
1898 0 : goto done;
1899 263788 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1900 0 : xfs_btree_mark_sick(cur);
1901 0 : error = -EFSCORRUPTED;
1902 0 : goto done;
1903 : }
1904 263788 : NEW.rm_blockcount += RIGHT.rm_blockcount;
1905 263788 : NEW.rm_flags = RIGHT.rm_flags;
1906 263788 : error = xfs_rmap_update(cur, &NEW);
1907 263788 : if (error)
1908 0 : goto done;
1909 : break;
1910 :
1911 11982508 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1912 : /*
1913 : * Setting all of a previous oldext extent to newext.
1914 : * Neither the left nor right neighbors are contiguous with
1915 : * the new one.
1916 : */
1917 11982508 : NEW = PREV;
1918 11982508 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1919 : NEW.rm_blockcount, NEW.rm_owner,
1920 : NEW.rm_offset, NEW.rm_flags, &i);
1921 11982511 : if (error)
1922 0 : goto done;
1923 11982511 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1924 0 : xfs_btree_mark_sick(cur);
1925 0 : error = -EFSCORRUPTED;
1926 0 : goto done;
1927 : }
1928 11982511 : NEW.rm_flags = newext;
1929 11982511 : error = xfs_rmap_update(cur, &NEW);
1930 11982513 : if (error)
1931 0 : goto done;
1932 : break;
1933 :
1934 198045 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1935 : /*
1936 : * Setting the first part of a previous oldext extent to newext.
1937 : * The left neighbor is contiguous.
1938 : */
1939 198045 : NEW = PREV;
1940 198045 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1941 : NEW.rm_blockcount, NEW.rm_owner,
1942 : NEW.rm_offset, NEW.rm_flags);
1943 198045 : if (error)
1944 0 : goto done;
1945 198045 : NEW.rm_offset += len;
1946 198045 : NEW.rm_startblock += len;
1947 198045 : NEW.rm_blockcount -= len;
1948 198045 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1949 : NEW.rm_blockcount, NEW.rm_owner,
1950 : NEW.rm_offset, NEW.rm_flags);
1951 198045 : if (error)
1952 0 : goto done;
1953 198045 : NEW = LEFT;
1954 198045 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1955 : NEW.rm_blockcount, NEW.rm_owner,
1956 : NEW.rm_offset, NEW.rm_flags, &i);
1957 198045 : if (error)
1958 0 : goto done;
1959 198045 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1960 0 : xfs_btree_mark_sick(cur);
1961 0 : error = -EFSCORRUPTED;
1962 0 : goto done;
1963 : }
1964 198045 : NEW.rm_blockcount += len;
1965 198045 : error = xfs_rmap_update(cur, &NEW);
1966 198045 : if (error)
1967 0 : goto done;
1968 : break;
1969 :
1970 4380421 : case RMAP_LEFT_FILLING:
1971 : /*
1972 : * Setting the first part of a previous oldext extent to newext.
1973 : * The left neighbor is not contiguous.
1974 : */
1975 4380421 : NEW = PREV;
1976 4380421 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1977 : NEW.rm_blockcount, NEW.rm_owner,
1978 : NEW.rm_offset, NEW.rm_flags);
1979 4380421 : if (error)
1980 0 : goto done;
1981 4380421 : NEW.rm_offset += len;
1982 4380421 : NEW.rm_startblock += len;
1983 4380421 : NEW.rm_blockcount -= len;
1984 4380421 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1985 : NEW.rm_blockcount, NEW.rm_owner,
1986 : NEW.rm_offset, NEW.rm_flags);
1987 4380421 : if (error)
1988 0 : goto done;
1989 4380421 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1990 4380421 : if (error)
1991 0 : goto done;
1992 : break;
1993 :
1994 93667 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1995 : /*
1996 : * Setting the last part of a previous oldext extent to newext.
1997 : * The right neighbor is contiguous with the new allocation.
1998 : */
1999 93667 : NEW = PREV;
2000 93667 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2001 : NEW.rm_blockcount, NEW.rm_owner,
2002 : NEW.rm_offset, NEW.rm_flags, &i);
2003 93667 : if (error)
2004 0 : goto done;
2005 93667 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2006 0 : xfs_btree_mark_sick(cur);
2007 0 : error = -EFSCORRUPTED;
2008 0 : goto done;
2009 : }
2010 93667 : NEW.rm_blockcount = offset - NEW.rm_offset;
2011 93667 : error = xfs_rmap_update(cur, &NEW);
2012 93667 : if (error)
2013 0 : goto done;
2014 93667 : NEW = RIGHT;
2015 93667 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
2016 : NEW.rm_blockcount, NEW.rm_owner,
2017 : NEW.rm_offset, NEW.rm_flags);
2018 93667 : if (error)
2019 0 : goto done;
2020 93667 : NEW.rm_offset = offset;
2021 93667 : NEW.rm_startblock = bno;
2022 93667 : NEW.rm_blockcount += len;
2023 93667 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2024 : NEW.rm_blockcount, NEW.rm_owner,
2025 : NEW.rm_offset, NEW.rm_flags);
2026 93667 : if (error)
2027 0 : goto done;
2028 : break;
2029 :
2030 649305 : case RMAP_RIGHT_FILLING:
2031 : /*
2032 : * Setting the last part of a previous oldext extent to newext.
2033 : * The right neighbor is not contiguous.
2034 : */
2035 649305 : NEW = PREV;
2036 649305 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2037 : NEW.rm_blockcount, NEW.rm_owner,
2038 : NEW.rm_offset, NEW.rm_flags, &i);
2039 649305 : if (error)
2040 0 : goto done;
2041 649305 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2042 0 : xfs_btree_mark_sick(cur);
2043 0 : error = -EFSCORRUPTED;
2044 0 : goto done;
2045 : }
2046 649305 : NEW.rm_blockcount -= len;
2047 649305 : error = xfs_rmap_update(cur, &NEW);
2048 649305 : if (error)
2049 0 : goto done;
2050 649305 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
2051 649305 : if (error)
2052 0 : goto done;
2053 : break;
2054 :
2055 767115 : case 0:
2056 : /*
2057 : * Setting the middle part of a previous oldext extent to
2058 : * newext. Contiguity is impossible here.
2059 : * One extent becomes three extents.
2060 : */
2061 : /* new right extent - oldext */
2062 767115 : NEW.rm_startblock = bno + len;
2063 767115 : NEW.rm_owner = owner;
2064 767115 : NEW.rm_offset = new_endoff;
2065 767115 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
2066 : new_endoff;
2067 767115 : NEW.rm_flags = PREV.rm_flags;
2068 767115 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2069 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
2070 : NEW.rm_flags);
2071 767115 : if (error)
2072 0 : goto done;
2073 : /* new left extent - oldext */
2074 767115 : NEW = PREV;
2075 767115 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2076 : NEW.rm_blockcount, NEW.rm_owner,
2077 : NEW.rm_offset, NEW.rm_flags, &i);
2078 767115 : if (error)
2079 0 : goto done;
2080 767115 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2081 0 : xfs_btree_mark_sick(cur);
2082 0 : error = -EFSCORRUPTED;
2083 0 : goto done;
2084 : }
2085 767115 : NEW.rm_blockcount = offset - NEW.rm_offset;
2086 767115 : error = xfs_rmap_update(cur, &NEW);
2087 767115 : if (error)
2088 0 : goto done;
2089 : /* new middle extent - newext */
2090 767115 : NEW.rm_startblock = bno;
2091 767115 : NEW.rm_blockcount = len;
2092 767115 : NEW.rm_owner = owner;
2093 767115 : NEW.rm_offset = offset;
2094 767115 : NEW.rm_flags = newext;
2095 767115 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2096 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
2097 : NEW.rm_flags);
2098 767115 : if (error)
2099 0 : goto done;
2100 : break;
2101 :
2102 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2103 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2104 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
2105 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
2106 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2107 : case RMAP_LEFT_CONTIG:
2108 : case RMAP_RIGHT_CONTIG:
2109 : /*
2110 : * These cases are all impossible.
2111 : */
2112 0 : ASSERT(0);
2113 : }
2114 :
2115 18877944 : trace_xfs_rmap_convert_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2116 : unwritten, oinfo);
2117 18877934 : done:
2118 18877934 : if (error)
2119 4 : trace_xfs_rmap_convert_error(cur->bc_mp,
2120 2 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2121 18877934 : return error;
2122 : }
2123 :
2124 : #undef NEW
2125 : #undef LEFT
2126 : #undef RIGHT
2127 : #undef PREV
2128 :
2129 : /*
2130 : * Find an extent in the rmap btree and unmap it. For rmap extent types that
2131 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2132 : * that the prev/next records in the btree might belong to another owner.
2133 : * Therefore we must use delete+insert to alter any of the key fields.
2134 : *
2135 : * For every other situation there can only be one owner for a given extent,
2136 : * so we can call the regular _free function.
2137 : */
2138 : STATIC int
2139 111561406 : xfs_rmap_unmap_shared(
2140 : struct xfs_btree_cur *cur,
2141 : xfs_agblock_t bno,
2142 : xfs_extlen_t len,
2143 : bool unwritten,
2144 : const struct xfs_owner_info *oinfo)
2145 : {
2146 111561406 : struct xfs_mount *mp = cur->bc_mp;
2147 111561406 : struct xfs_rmap_irec ltrec;
2148 111561406 : uint64_t ltoff;
2149 111561406 : int error = 0;
2150 111561406 : int i;
2151 111561406 : uint64_t owner;
2152 111561406 : uint64_t offset;
2153 111561406 : unsigned int flags;
2154 :
2155 111561406 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2156 111561406 : if (unwritten)
2157 22661203 : flags |= XFS_RMAP_UNWRITTEN;
2158 111561406 : trace_xfs_rmap_unmap(mp, cur->bc_ag.pag->pag_agno, bno, len,
2159 : unwritten, oinfo);
2160 :
2161 : /*
2162 : * We should always have a left record because there's a static record
2163 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
2164 : * will not ever be removed from the tree.
2165 : */
2166 111559983 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, flags,
2167 : <rec, &i);
2168 111558686 : if (error)
2169 297 : goto out_error;
2170 111558389 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2171 0 : xfs_btree_mark_sick(cur);
2172 0 : error = -EFSCORRUPTED;
2173 0 : goto out_error;
2174 : }
2175 111558389 : ltoff = ltrec.rm_offset;
2176 :
2177 : /* Make sure the extent we found covers the entire freeing range. */
2178 111558389 : if (XFS_IS_CORRUPT(mp,
2179 : ltrec.rm_startblock > bno ||
2180 : ltrec.rm_startblock + ltrec.rm_blockcount <
2181 : bno + len)) {
2182 0 : xfs_btree_mark_sick(cur);
2183 0 : error = -EFSCORRUPTED;
2184 0 : goto out_error;
2185 : }
2186 :
2187 : /* Make sure the owner matches what we expect to find in the tree. */
2188 111558389 : if (XFS_IS_CORRUPT(mp, owner != ltrec.rm_owner)) {
2189 0 : xfs_btree_mark_sick(cur);
2190 0 : error = -EFSCORRUPTED;
2191 0 : goto out_error;
2192 : }
2193 :
2194 : /* Make sure the unwritten flag matches. */
2195 111558389 : if (XFS_IS_CORRUPT(mp,
2196 : (flags & XFS_RMAP_UNWRITTEN) !=
2197 : (ltrec.rm_flags & XFS_RMAP_UNWRITTEN))) {
2198 0 : xfs_btree_mark_sick(cur);
2199 0 : error = -EFSCORRUPTED;
2200 0 : goto out_error;
2201 : }
2202 :
2203 : /* Check the offset. */
2204 111558389 : if (XFS_IS_CORRUPT(mp, ltrec.rm_offset > offset)) {
2205 0 : xfs_btree_mark_sick(cur);
2206 0 : error = -EFSCORRUPTED;
2207 0 : goto out_error;
2208 : }
2209 111558389 : if (XFS_IS_CORRUPT(mp, offset > ltoff + ltrec.rm_blockcount)) {
2210 0 : xfs_btree_mark_sick(cur);
2211 0 : error = -EFSCORRUPTED;
2212 0 : goto out_error;
2213 : }
2214 :
2215 111558389 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
2216 : /* Exact match, simply remove the record from rmap tree. */
2217 82425114 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2218 : ltrec.rm_blockcount, ltrec.rm_owner,
2219 : ltrec.rm_offset, ltrec.rm_flags);
2220 82427492 : if (error)
2221 4 : goto out_error;
2222 29133275 : } else if (ltrec.rm_startblock == bno) {
2223 : /*
2224 : * Overlap left hand side of extent: move the start, trim the
2225 : * length and update the current record.
2226 : *
2227 : * ltbno ltlen
2228 : * Orig: |oooooooooooooooooooo|
2229 : * Freeing: |fffffffff|
2230 : * Result: |rrrrrrrrrr|
2231 : * bno len
2232 : */
2233 :
2234 : /* Delete prev rmap. */
2235 9453027 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2236 : ltrec.rm_blockcount, ltrec.rm_owner,
2237 : ltrec.rm_offset, ltrec.rm_flags);
2238 9453076 : if (error)
2239 1 : goto out_error;
2240 :
2241 : /* Add an rmap at the new offset. */
2242 9453075 : ltrec.rm_startblock += len;
2243 9453075 : ltrec.rm_blockcount -= len;
2244 9453075 : ltrec.rm_offset += len;
2245 9453075 : error = xfs_rmap_insert(cur, ltrec.rm_startblock,
2246 : ltrec.rm_blockcount, ltrec.rm_owner,
2247 : ltrec.rm_offset, ltrec.rm_flags);
2248 9453105 : if (error)
2249 0 : goto out_error;
2250 19680248 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
2251 : /*
2252 : * Overlap right hand side of extent: trim the length and
2253 : * update the current record.
2254 : *
2255 : * ltbno ltlen
2256 : * Orig: |oooooooooooooooooooo|
2257 : * Freeing: |fffffffff|
2258 : * Result: |rrrrrrrrrr|
2259 : * bno len
2260 : */
2261 6956884 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2262 : ltrec.rm_blockcount, ltrec.rm_owner,
2263 : ltrec.rm_offset, ltrec.rm_flags, &i);
2264 6956910 : if (error)
2265 0 : goto out_error;
2266 6956910 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2267 0 : xfs_btree_mark_sick(cur);
2268 0 : error = -EFSCORRUPTED;
2269 0 : goto out_error;
2270 : }
2271 6956910 : ltrec.rm_blockcount -= len;
2272 6956910 : error = xfs_rmap_update(cur, <rec);
2273 6956923 : if (error)
2274 0 : goto out_error;
2275 : } else {
2276 : /*
2277 : * Overlap middle of extent: trim the length of the existing
2278 : * record to the length of the new left-extent size, increment
2279 : * the insertion position so we can insert a new record
2280 : * containing the remaining right-extent space.
2281 : *
2282 : * ltbno ltlen
2283 : * Orig: |oooooooooooooooooooo|
2284 : * Freeing: |fffffffff|
2285 : * Result: |rrrrr| |rrrr|
2286 : * bno len
2287 : */
2288 12723364 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
2289 :
2290 : /* Shrink the left side of the rmap */
2291 12723364 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2292 : ltrec.rm_blockcount, ltrec.rm_owner,
2293 : ltrec.rm_offset, ltrec.rm_flags, &i);
2294 12723429 : if (error)
2295 0 : goto out_error;
2296 12723429 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2297 0 : xfs_btree_mark_sick(cur);
2298 0 : error = -EFSCORRUPTED;
2299 0 : goto out_error;
2300 : }
2301 12723429 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
2302 12723429 : error = xfs_rmap_update(cur, <rec);
2303 12723418 : if (error)
2304 0 : goto out_error;
2305 :
2306 : /* Add an rmap at the new offset */
2307 12723418 : error = xfs_rmap_insert(cur, bno + len,
2308 12723418 : orig_len - len - ltrec.rm_blockcount,
2309 : ltrec.rm_owner, offset + len,
2310 : ltrec.rm_flags);
2311 12723444 : if (error)
2312 0 : goto out_error;
2313 : }
2314 :
2315 111560960 : trace_xfs_rmap_unmap_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2316 : unwritten, oinfo);
2317 111560000 : out_error:
2318 111560000 : if (error)
2319 604 : trace_xfs_rmap_unmap_error(cur->bc_mp,
2320 302 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2321 111560000 : return error;
2322 : }
2323 :
2324 : /*
2325 : * Find an extent in the rmap btree and map it. For rmap extent types that
2326 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2327 : * that the prev/next records in the btree might belong to another owner.
2328 : * Therefore we must use delete+insert to alter any of the key fields.
2329 : *
2330 : * For every other situation there can only be one owner for a given extent,
2331 : * so we can call the regular _alloc function.
2332 : */
2333 : STATIC int
2334 164079303 : xfs_rmap_map_shared(
2335 : struct xfs_btree_cur *cur,
2336 : xfs_agblock_t bno,
2337 : xfs_extlen_t len,
2338 : bool unwritten,
2339 : const struct xfs_owner_info *oinfo)
2340 : {
2341 164079303 : struct xfs_mount *mp = cur->bc_mp;
2342 164079303 : struct xfs_rmap_irec ltrec;
2343 164079303 : struct xfs_rmap_irec gtrec;
2344 164079303 : int have_gt;
2345 164079303 : int have_lt;
2346 164079303 : int error = 0;
2347 164079303 : int i;
2348 164079303 : uint64_t owner;
2349 164079303 : uint64_t offset;
2350 164079303 : unsigned int flags = 0;
2351 :
2352 164079303 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2353 164079303 : if (unwritten)
2354 25832633 : flags |= XFS_RMAP_UNWRITTEN;
2355 164079303 : trace_xfs_rmap_map(mp, cur->bc_ag.pag->pag_agno, bno, len,
2356 : unwritten, oinfo);
2357 :
2358 : /* Is there a left record that abuts our range? */
2359 164078737 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, flags,
2360 : <rec, &have_lt);
2361 164081726 : if (error)
2362 292 : goto out_error;
2363 164081434 : if (have_lt &&
2364 32388473 : !xfs_rmap_is_mergeable(<rec, owner, flags))
2365 1737976 : have_lt = 0;
2366 :
2367 : /* Is there a right record that abuts our range? */
2368 164081434 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
2369 : flags, &have_gt);
2370 164082104 : if (error)
2371 1 : goto out_error;
2372 164082103 : if (have_gt) {
2373 2306481 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
2374 2306481 : if (error)
2375 0 : goto out_error;
2376 2306481 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
2377 0 : xfs_btree_mark_sick(cur);
2378 0 : error = -EFSCORRUPTED;
2379 0 : goto out_error;
2380 : }
2381 2306481 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
2382 2306481 : cur->bc_ag.pag->pag_agno, gtrec.rm_startblock,
2383 : gtrec.rm_blockcount, gtrec.rm_owner,
2384 : gtrec.rm_offset, gtrec.rm_flags);
2385 :
2386 2306481 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
2387 403452 : have_gt = 0;
2388 : }
2389 :
2390 164082103 : if (have_lt &&
2391 30650496 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
2392 7200726 : ltrec.rm_offset + ltrec.rm_blockcount == offset) {
2393 : /*
2394 : * Left edge contiguous, merge into left record.
2395 : *
2396 : * ltbno ltlen
2397 : * orig: |ooooooooo|
2398 : * adding: |aaaaaaaaa|
2399 : * result: |rrrrrrrrrrrrrrrrrrr|
2400 : * bno len
2401 : */
2402 7200726 : ltrec.rm_blockcount += len;
2403 7200726 : if (have_gt &&
2404 1195982 : bno + len == gtrec.rm_startblock &&
2405 1195982 : offset + len == gtrec.rm_offset) {
2406 : /*
2407 : * Right edge also contiguous, delete right record
2408 : * and merge into left record.
2409 : *
2410 : * ltbno ltlen gtbno gtlen
2411 : * orig: |ooooooooo| |ooooooooo|
2412 : * adding: |aaaaaaaaa|
2413 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
2414 : */
2415 1195982 : ltrec.rm_blockcount += gtrec.rm_blockcount;
2416 1195982 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2417 : gtrec.rm_blockcount, gtrec.rm_owner,
2418 : gtrec.rm_offset, gtrec.rm_flags);
2419 1195982 : if (error)
2420 0 : goto out_error;
2421 : }
2422 :
2423 : /* Point the cursor back to the left record and update. */
2424 7200726 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2425 : ltrec.rm_blockcount, ltrec.rm_owner,
2426 : ltrec.rm_offset, ltrec.rm_flags, &i);
2427 7200726 : if (error)
2428 0 : goto out_error;
2429 7200726 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2430 0 : xfs_btree_mark_sick(cur);
2431 0 : error = -EFSCORRUPTED;
2432 0 : goto out_error;
2433 : }
2434 :
2435 7200726 : error = xfs_rmap_update(cur, <rec);
2436 7200726 : if (error)
2437 0 : goto out_error;
2438 156881377 : } else if (have_gt &&
2439 707047 : bno + len == gtrec.rm_startblock &&
2440 707047 : offset + len == gtrec.rm_offset) {
2441 : /*
2442 : * Right edge contiguous, merge into right record.
2443 : *
2444 : * gtbno gtlen
2445 : * Orig: |ooooooooo|
2446 : * adding: |aaaaaaaaa|
2447 : * Result: |rrrrrrrrrrrrrrrrrrr|
2448 : * bno len
2449 : */
2450 : /* Delete the old record. */
2451 707047 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2452 : gtrec.rm_blockcount, gtrec.rm_owner,
2453 : gtrec.rm_offset, gtrec.rm_flags);
2454 707047 : if (error)
2455 0 : goto out_error;
2456 :
2457 : /* Move the start and re-add it. */
2458 707047 : gtrec.rm_startblock = bno;
2459 707047 : gtrec.rm_blockcount += len;
2460 707047 : gtrec.rm_offset = offset;
2461 707047 : error = xfs_rmap_insert(cur, gtrec.rm_startblock,
2462 : gtrec.rm_blockcount, gtrec.rm_owner,
2463 : gtrec.rm_offset, gtrec.rm_flags);
2464 707047 : if (error)
2465 0 : goto out_error;
2466 : } else {
2467 : /*
2468 : * No contiguous edge with identical owner, insert
2469 : * new record at current cursor position.
2470 : */
2471 156174330 : error = xfs_rmap_insert(cur, bno, len, owner, offset, flags);
2472 156173789 : if (error)
2473 175 : goto out_error;
2474 : }
2475 :
2476 164081387 : trace_xfs_rmap_map_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2477 : unwritten, oinfo);
2478 164081101 : out_error:
2479 164081101 : if (error)
2480 936 : trace_xfs_rmap_map_error(cur->bc_mp,
2481 468 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2482 164081101 : return error;
2483 : }
2484 :
2485 : /* Insert a raw rmap into the rmapbt. */
2486 : int
2487 52623668 : xfs_rmap_map_raw(
2488 : struct xfs_btree_cur *cur,
2489 : struct xfs_rmap_irec *rmap)
2490 : {
2491 52623668 : struct xfs_owner_info oinfo;
2492 :
2493 52623668 : xfs_owner_info_pack(&oinfo, rmap->rm_owner, rmap->rm_offset,
2494 : rmap->rm_flags);
2495 :
2496 52623668 : if ((rmap->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK |
2497 39904256 : XFS_RMAP_UNWRITTEN)) ||
2498 39904256 : XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2499 13665864 : return xfs_rmap_map(cur, rmap->rm_startblock,
2500 : rmap->rm_blockcount,
2501 13665864 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2502 : &oinfo);
2503 :
2504 38957804 : return xfs_rmap_map_shared(cur, rmap->rm_startblock,
2505 : rmap->rm_blockcount,
2506 38957804 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2507 : &oinfo);
2508 : }
2509 :
2510 : struct xfs_rmap_query_range_info {
2511 : xfs_rmap_query_range_fn fn;
2512 : void *priv;
2513 : };
2514 :
2515 : /* Format btree record and pass to our callback. */
2516 : STATIC int
2517 84260599656 : xfs_rmap_query_range_helper(
2518 : struct xfs_btree_cur *cur,
2519 : const union xfs_btree_rec *rec,
2520 : void *priv)
2521 : {
2522 84260599656 : struct xfs_rmap_query_range_info *query = priv;
2523 84260599656 : struct xfs_rmap_irec irec;
2524 84260599656 : xfs_failaddr_t fa;
2525 :
2526 84260599656 : fa = xfs_rmap_btrec_to_irec(rec, &irec);
2527 84002372075 : if (!fa)
2528 84025101471 : fa = xfs_rmap_check_irec(cur, &irec);
2529 83987382869 : if (fa)
2530 0 : return xfs_rmap_complain_bad_rec(cur, fa, &irec);
2531 :
2532 83987382869 : return query->fn(cur, &irec, query->priv);
2533 : }
2534 :
2535 : /* Find all rmaps between two keys. */
2536 : int
2537 657660064 : xfs_rmap_query_range(
2538 : struct xfs_btree_cur *cur,
2539 : const struct xfs_rmap_irec *low_rec,
2540 : const struct xfs_rmap_irec *high_rec,
2541 : xfs_rmap_query_range_fn fn,
2542 : void *priv)
2543 : {
2544 657660064 : union xfs_btree_irec low_brec = { .r = *low_rec };
2545 657660064 : union xfs_btree_irec high_brec = { .r = *high_rec };
2546 657660064 : struct xfs_rmap_query_range_info query = { .priv = priv, .fn = fn };
2547 :
2548 657660064 : return xfs_btree_query_range(cur, &low_brec, &high_brec,
2549 : xfs_rmap_query_range_helper, &query);
2550 : }
2551 :
2552 : /* Find all rmaps. */
2553 : int
2554 15294157 : xfs_rmap_query_all(
2555 : struct xfs_btree_cur *cur,
2556 : xfs_rmap_query_range_fn fn,
2557 : void *priv)
2558 : {
2559 15294157 : struct xfs_rmap_query_range_info query;
2560 :
2561 15294157 : query.priv = priv;
2562 15294157 : query.fn = fn;
2563 15294157 : return xfs_btree_query_all(cur, xfs_rmap_query_range_helper, &query);
2564 : }
2565 :
2566 : /* Clean up after calling xfs_rmap_finish_one. */
2567 : void
2568 406650443 : xfs_rmap_finish_one_cleanup(
2569 : struct xfs_trans *tp,
2570 : struct xfs_btree_cur *rcur,
2571 : int error)
2572 : {
2573 406652772 : struct xfs_buf *agbp;
2574 :
2575 406650443 : if (rcur == NULL)
2576 : return;
2577 406651269 : agbp = rcur->bc_ag.agbp;
2578 406648940 : xfs_btree_del_cursor(rcur, error);
2579 406659212 : if (error)
2580 1673 : xfs_trans_brelse(tp, agbp);
2581 : }
2582 :
2583 : /* Commit an rmap operation into the ondisk tree. */
2584 : int
2585 470827976 : __xfs_rmap_finish_intent(
2586 : struct xfs_btree_cur *rcur,
2587 : enum xfs_rmap_intent_type op,
2588 : xfs_agblock_t bno,
2589 : xfs_extlen_t len,
2590 : const struct xfs_owner_info *oinfo,
2591 : bool unwritten)
2592 : {
2593 470827976 : switch (op) {
2594 101348108 : case XFS_RMAP_ALLOC:
2595 : case XFS_RMAP_MAP:
2596 101348108 : return xfs_rmap_map(rcur, bno, len, unwritten, oinfo);
2597 125123335 : case XFS_RMAP_MAP_SHARED:
2598 125123335 : return xfs_rmap_map_shared(rcur, bno, len, unwritten, oinfo);
2599 100957137 : case XFS_RMAP_FREE:
2600 : case XFS_RMAP_UNMAP:
2601 100957137 : return xfs_rmap_unmap(rcur, bno, len, unwritten, oinfo);
2602 111561393 : case XFS_RMAP_UNMAP_SHARED:
2603 111561393 : return xfs_rmap_unmap_shared(rcur, bno, len, unwritten, oinfo);
2604 12960068 : case XFS_RMAP_CONVERT:
2605 12960068 : return xfs_rmap_convert(rcur, bno, len, !unwritten, oinfo);
2606 18877935 : case XFS_RMAP_CONVERT_SHARED:
2607 18877935 : return xfs_rmap_convert_shared(rcur, bno, len, !unwritten,
2608 18877935 : oinfo);
2609 0 : default:
2610 0 : ASSERT(0);
2611 0 : return -EFSCORRUPTED;
2612 : }
2613 : }
2614 :
2615 : /*
2616 : * Process one of the deferred rmap operations. We pass back the
2617 : * btree cursor to maintain our lock on the rmapbt between calls.
2618 : * This saves time and eliminates a buffer deadlock between the
2619 : * superblock and the AGF because we'll always grab them in the same
2620 : * order.
2621 : */
2622 : int
2623 470625123 : xfs_rmap_finish_one(
2624 : struct xfs_trans *tp,
2625 : struct xfs_rmap_intent *ri,
2626 : struct xfs_btree_cur **pcur)
2627 : {
2628 470625123 : struct xfs_mount *mp = tp->t_mountp;
2629 470625123 : struct xfs_btree_cur *rcur;
2630 470625123 : struct xfs_buf *agbp = NULL;
2631 470625123 : int error = 0;
2632 470625123 : struct xfs_owner_info oinfo;
2633 470625123 : xfs_agblock_t bno;
2634 470625123 : bool unwritten;
2635 :
2636 470625123 : bno = XFS_FSB_TO_AGBNO(mp, ri->ri_bmap.br_startblock);
2637 :
2638 470616016 : trace_xfs_rmap_deferred(mp, ri->ri_pag->pag_agno, ri->ri_type, bno,
2639 : ri->ri_owner, ri->ri_whichfork,
2640 : ri->ri_bmap.br_startoff, ri->ri_bmap.br_blockcount,
2641 : ri->ri_bmap.br_state);
2642 :
2643 470617546 : if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_RMAP_FINISH_ONE))
2644 : return -EIO;
2645 :
2646 : /*
2647 : * If we haven't gotten a cursor or the cursor AG doesn't match
2648 : * the startblock, get one now.
2649 : */
2650 470622623 : rcur = *pcur;
2651 470622623 : if (rcur != NULL && rcur->bc_ag.pag != ri->ri_pag) {
2652 2329 : xfs_rmap_finish_one_cleanup(tp, rcur, 0);
2653 2329 : rcur = NULL;
2654 2329 : *pcur = NULL;
2655 : }
2656 470622623 : if (rcur == NULL) {
2657 : /*
2658 : * Refresh the freelist before we start changing the
2659 : * rmapbt, because a shape change could cause us to
2660 : * allocate blocks.
2661 : */
2662 406658795 : error = xfs_free_extent_fix_freelist(tp, ri->ri_pag, &agbp);
2663 406674017 : if (error) {
2664 1486 : xfs_ag_mark_sick(ri->ri_pag, XFS_SICK_AG_AGFL);
2665 1486 : return error;
2666 : }
2667 406672531 : if (XFS_IS_CORRUPT(tp->t_mountp, !agbp)) {
2668 0 : xfs_ag_mark_sick(ri->ri_pag, XFS_SICK_AG_AGFL);
2669 0 : return -EFSCORRUPTED;
2670 : }
2671 :
2672 406672531 : rcur = xfs_rmapbt_init_cursor(mp, tp, agbp, ri->ri_pag);
2673 : }
2674 470637051 : *pcur = rcur;
2675 :
2676 470637051 : xfs_rmap_ino_owner(&oinfo, ri->ri_owner, ri->ri_whichfork,
2677 : ri->ri_bmap.br_startoff);
2678 470637051 : unwritten = ri->ri_bmap.br_state == XFS_EXT_UNWRITTEN;
2679 470637051 : bno = XFS_FSB_TO_AGBNO(rcur->bc_mp, ri->ri_bmap.br_startblock);
2680 :
2681 470633611 : error = __xfs_rmap_finish_intent(rcur, ri->ri_type, bno,
2682 470633611 : ri->ri_bmap.br_blockcount, &oinfo, unwritten);
2683 470611372 : if (error)
2684 : return error;
2685 :
2686 470608725 : xfs_rmap_update_hook(tp, ri->ri_pag, ri->ri_type, bno,
2687 470608725 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2688 470608725 : return 0;
2689 : }
2690 :
2691 : /*
2692 : * Don't defer an rmap if we aren't an rmap filesystem.
2693 : */
2694 : static bool
2695 : xfs_rmap_update_is_needed(
2696 : struct xfs_mount *mp,
2697 : int whichfork)
2698 : {
2699 464039774 : return xfs_has_rmapbt(mp) && whichfork != XFS_COW_FORK;
2700 : }
2701 :
2702 : /*
2703 : * Record a rmap intent; the list is kept sorted first by AG and then by
2704 : * increasing age.
2705 : */
2706 : static void
2707 470526634 : __xfs_rmap_add(
2708 : struct xfs_trans *tp,
2709 : enum xfs_rmap_intent_type type,
2710 : uint64_t owner,
2711 : int whichfork,
2712 : struct xfs_bmbt_irec *bmap)
2713 : {
2714 470526634 : struct xfs_rmap_intent *ri;
2715 :
2716 1411476844 : trace_xfs_rmap_defer(tp->t_mountp,
2717 470475105 : XFS_FSB_TO_AGNO(tp->t_mountp, bmap->br_startblock),
2718 : type,
2719 470526634 : XFS_FSB_TO_AGBNO(tp->t_mountp, bmap->br_startblock),
2720 : owner, whichfork,
2721 : bmap->br_startoff,
2722 : bmap->br_blockcount,
2723 : bmap->br_state);
2724 :
2725 470483914 : ri = kmem_cache_alloc(xfs_rmap_intent_cache, GFP_NOFS | __GFP_NOFAIL);
2726 470571800 : INIT_LIST_HEAD(&ri->ri_list);
2727 470571800 : ri->ri_type = type;
2728 470571800 : ri->ri_owner = owner;
2729 470571800 : ri->ri_whichfork = whichfork;
2730 470571800 : ri->ri_bmap = *bmap;
2731 :
2732 470571800 : xfs_rmap_update_get_group(tp->t_mountp, ri);
2733 470617175 : xfs_defer_add(tp, XFS_DEFER_OPS_TYPE_RMAP, &ri->ri_list);
2734 470558348 : }
2735 :
2736 : /* Map an extent into a file. */
2737 : void
2738 347769614 : xfs_rmap_map_extent(
2739 : struct xfs_trans *tp,
2740 : struct xfs_inode *ip,
2741 : int whichfork,
2742 : struct xfs_bmbt_irec *PREV)
2743 : {
2744 347769614 : enum xfs_rmap_intent_type type = XFS_RMAP_MAP;
2745 :
2746 695539228 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2747 : return;
2748 :
2749 222541511 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2750 125065864 : type = XFS_RMAP_MAP_SHARED;
2751 :
2752 222541511 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2753 : }
2754 :
2755 : /* Unmap an extent out of a file. */
2756 : void
2757 296738152 : xfs_rmap_unmap_extent(
2758 : struct xfs_trans *tp,
2759 : struct xfs_inode *ip,
2760 : int whichfork,
2761 : struct xfs_bmbt_irec *PREV)
2762 : {
2763 296738152 : enum xfs_rmap_intent_type type = XFS_RMAP_UNMAP;
2764 :
2765 593476304 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2766 : return;
2767 :
2768 196789167 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2769 111493040 : type = XFS_RMAP_UNMAP_SHARED;
2770 :
2771 196789167 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2772 : }
2773 :
2774 : /*
2775 : * Convert a data fork extent from unwritten to real or vice versa.
2776 : *
2777 : * Note that tp can be NULL here as no transaction is used for COW fork
2778 : * unwritten conversion.
2779 : */
2780 : void
2781 67529519 : xfs_rmap_convert_extent(
2782 : struct xfs_mount *mp,
2783 : struct xfs_trans *tp,
2784 : struct xfs_inode *ip,
2785 : int whichfork,
2786 : struct xfs_bmbt_irec *PREV)
2787 : {
2788 67529519 : enum xfs_rmap_intent_type type = XFS_RMAP_CONVERT;
2789 :
2790 135059038 : if (!xfs_rmap_update_is_needed(mp, whichfork))
2791 : return;
2792 :
2793 31802176 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2794 18846769 : type = XFS_RMAP_CONVERT_SHARED;
2795 :
2796 31802176 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2797 : }
2798 :
2799 : /* Schedule the creation of an rmap for non-file data. */
2800 : void
2801 3824760 : xfs_rmap_alloc_extent(
2802 : struct xfs_trans *tp,
2803 : xfs_agnumber_t agno,
2804 : xfs_agblock_t bno,
2805 : xfs_extlen_t len,
2806 : uint64_t owner)
2807 : {
2808 3824760 : struct xfs_bmbt_irec bmap;
2809 :
2810 3824760 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2811 0 : return;
2812 :
2813 3824760 : bmap.br_startblock = XFS_AGB_TO_FSB(tp->t_mountp, agno, bno);
2814 3824760 : bmap.br_blockcount = len;
2815 3824760 : bmap.br_startoff = 0;
2816 3824760 : bmap.br_state = XFS_EXT_NORM;
2817 :
2818 3824760 : __xfs_rmap_add(tp, XFS_RMAP_ALLOC, owner, XFS_DATA_FORK, &bmap);
2819 : }
2820 :
2821 : /* Schedule the deletion of an rmap for non-file data. */
2822 : void
2823 15590878 : xfs_rmap_free_extent(
2824 : struct xfs_trans *tp,
2825 : xfs_agnumber_t agno,
2826 : xfs_agblock_t bno,
2827 : xfs_extlen_t len,
2828 : uint64_t owner)
2829 : {
2830 15590878 : struct xfs_bmbt_irec bmap;
2831 :
2832 15590878 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2833 0 : return;
2834 :
2835 15590878 : bmap.br_startblock = XFS_AGB_TO_FSB(tp->t_mountp, agno, bno);
2836 15590878 : bmap.br_blockcount = len;
2837 15590878 : bmap.br_startoff = 0;
2838 15590878 : bmap.br_state = XFS_EXT_NORM;
2839 :
2840 15590878 : __xfs_rmap_add(tp, XFS_RMAP_FREE, owner, XFS_DATA_FORK, &bmap);
2841 : }
2842 :
2843 : /* Compare rmap records. Returns -1 if a < b, 1 if a > b, and 0 if equal. */
2844 : int
2845 116486649 : xfs_rmap_compare(
2846 : const struct xfs_rmap_irec *a,
2847 : const struct xfs_rmap_irec *b)
2848 : {
2849 116486649 : __u64 oa;
2850 116486649 : __u64 ob;
2851 :
2852 116486649 : oa = xfs_rmap_irec_offset_pack(a);
2853 116486649 : ob = xfs_rmap_irec_offset_pack(b);
2854 :
2855 116486649 : if (a->rm_startblock < b->rm_startblock)
2856 : return -1;
2857 116451771 : else if (a->rm_startblock > b->rm_startblock)
2858 : return 1;
2859 27438618 : else if (a->rm_owner < b->rm_owner)
2860 : return -1;
2861 26857244 : else if (a->rm_owner > b->rm_owner)
2862 : return 1;
2863 26223096 : else if (oa < ob)
2864 : return -1;
2865 26035669 : else if (oa > ob)
2866 : return 1;
2867 : else
2868 12736 : return 0;
2869 : }
2870 :
2871 : /*
2872 : * Scan the physical storage part of the keyspace of the reverse mapping index
2873 : * and tell us if the area has no records, is fully mapped by records, or is
2874 : * partially filled.
2875 : */
2876 : int
2877 441206595 : xfs_rmap_has_records(
2878 : struct xfs_btree_cur *cur,
2879 : xfs_agblock_t bno,
2880 : xfs_extlen_t len,
2881 : enum xbtree_recpacking *outcome)
2882 : {
2883 441206595 : union xfs_btree_key mask = {
2884 : .rmap.rm_startblock = cpu_to_be32(-1U),
2885 : };
2886 441206595 : union xfs_btree_irec low;
2887 441206595 : union xfs_btree_irec high;
2888 :
2889 441206595 : memset(&low, 0, sizeof(low));
2890 441206595 : low.r.rm_startblock = bno;
2891 441206595 : memset(&high, 0xFF, sizeof(high));
2892 441206595 : high.r.rm_startblock = bno + len - 1;
2893 :
2894 441206595 : return xfs_btree_has_records(cur, &low, &high, &mask, outcome);
2895 : }
2896 :
2897 : struct xfs_rmap_ownercount {
2898 : /* Owner that we're looking for. */
2899 : struct xfs_rmap_irec good;
2900 :
2901 : /* rmap search keys */
2902 : struct xfs_rmap_irec low;
2903 : struct xfs_rmap_irec high;
2904 :
2905 : struct xfs_rmap_matches *results;
2906 :
2907 : /* Stop early if we find a nonmatch? */
2908 : bool stop_on_nonmatch;
2909 : };
2910 :
2911 : /* Does this rmap represent space that can have multiple owners? */
2912 : static inline bool
2913 : xfs_rmap_shareable(
2914 : struct xfs_mount *mp,
2915 : const struct xfs_rmap_irec *rmap)
2916 : {
2917 39833339 : if (!xfs_has_reflink(mp))
2918 : return false;
2919 79665736 : if (XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2920 : return false;
2921 39116180 : if (rmap->rm_flags & (XFS_RMAP_ATTR_FORK |
2922 : XFS_RMAP_BMBT_BLOCK))
2923 1156894 : return false;
2924 : return true;
2925 : }
2926 :
2927 : static inline void
2928 417554638 : xfs_rmap_ownercount_init(
2929 : struct xfs_rmap_ownercount *roc,
2930 : xfs_agblock_t bno,
2931 : xfs_extlen_t len,
2932 : const struct xfs_owner_info *oinfo,
2933 : struct xfs_rmap_matches *results)
2934 : {
2935 417554638 : memset(roc, 0, sizeof(*roc));
2936 417554638 : roc->results = results;
2937 :
2938 417554638 : roc->low.rm_startblock = bno;
2939 417554638 : memset(&roc->high, 0xFF, sizeof(roc->high));
2940 417554638 : roc->high.rm_startblock = bno + len - 1;
2941 :
2942 417554638 : memset(results, 0, sizeof(*results));
2943 417554638 : roc->good.rm_startblock = bno;
2944 417554638 : roc->good.rm_blockcount = len;
2945 417554638 : roc->good.rm_owner = oinfo->oi_owner;
2946 417554638 : roc->good.rm_offset = oinfo->oi_offset;
2947 417554638 : if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK)
2948 1056896 : roc->good.rm_flags |= XFS_RMAP_ATTR_FORK;
2949 417554638 : if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK)
2950 1889088 : roc->good.rm_flags |= XFS_RMAP_BMBT_BLOCK;
2951 417554638 : }
2952 :
2953 : /* Figure out if this is a match for the owner. */
2954 : STATIC int
2955 444153127 : xfs_rmap_count_owners_helper(
2956 : struct xfs_btree_cur *cur,
2957 : const struct xfs_rmap_irec *rec,
2958 : void *priv)
2959 : {
2960 444153127 : struct xfs_rmap_ownercount *roc = priv;
2961 444153127 : struct xfs_rmap_irec check = *rec;
2962 444153127 : unsigned int keyflags;
2963 444153127 : bool filedata;
2964 444153127 : int64_t delta;
2965 :
2966 444153127 : filedata = !XFS_RMAP_NON_INODE_OWNER(check.rm_owner) &&
2967 174925718 : !(check.rm_flags & XFS_RMAP_BMBT_BLOCK);
2968 :
2969 : /* Trim the part of check that comes before the comparison range. */
2970 444153127 : delta = (int64_t)roc->good.rm_startblock - check.rm_startblock;
2971 444153127 : if (delta > 0) {
2972 193363779 : check.rm_startblock += delta;
2973 193363779 : check.rm_blockcount -= delta;
2974 193363779 : if (filedata)
2975 1703751 : check.rm_offset += delta;
2976 : }
2977 :
2978 : /* Trim the part of check that comes after the comparison range. */
2979 444153127 : delta = (check.rm_startblock + check.rm_blockcount) -
2980 444153127 : (roc->good.rm_startblock + roc->good.rm_blockcount);
2981 444153127 : if (delta > 0)
2982 223172722 : check.rm_blockcount -= delta;
2983 :
2984 : /* Don't care about unwritten status for establishing ownership. */
2985 444153127 : keyflags = check.rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK);
2986 :
2987 444153127 : if (check.rm_startblock == roc->good.rm_startblock &&
2988 417019945 : check.rm_blockcount == roc->good.rm_blockcount &&
2989 417019945 : check.rm_owner == roc->good.rm_owner &&
2990 404309863 : check.rm_offset == roc->good.rm_offset &&
2991 404314488 : keyflags == roc->good.rm_flags) {
2992 404319788 : roc->results->matches++;
2993 : } else {
2994 39833339 : roc->results->non_owner_matches++;
2995 79666678 : if (xfs_rmap_shareable(cur->bc_mp, &roc->good) ^
2996 : xfs_rmap_shareable(cur->bc_mp, &check))
2997 37959311 : roc->results->bad_non_owner_matches++;
2998 : }
2999 :
3000 444153127 : if (roc->results->non_owner_matches && roc->stop_on_nonmatch)
3001 0 : return -ECANCELED;
3002 :
3003 : return 0;
3004 : }
3005 :
3006 : /* Count the number of owners and non-owners of this range of blocks. */
3007 : int
3008 415512619 : xfs_rmap_count_owners(
3009 : struct xfs_btree_cur *cur,
3010 : xfs_agblock_t bno,
3011 : xfs_extlen_t len,
3012 : const struct xfs_owner_info *oinfo,
3013 : struct xfs_rmap_matches *results)
3014 : {
3015 415512619 : struct xfs_rmap_ownercount roc;
3016 415512619 : int error;
3017 :
3018 415512619 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, results);
3019 415358565 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
3020 : xfs_rmap_count_owners_helper, &roc);
3021 415867473 : if (error)
3022 : return error;
3023 :
3024 : /*
3025 : * There can't be any non-owner rmaps that conflict with the given
3026 : * owner if we didn't find any rmaps matching the owner.
3027 : */
3028 415867473 : if (!results->matches)
3029 13343448 : results->bad_non_owner_matches = 0;
3030 :
3031 : return 0;
3032 : }
3033 :
3034 : /*
3035 : * Given an extent and some owner info, can we find records overlapping
3036 : * the extent whose owner info does not match the given owner?
3037 : */
3038 : int
3039 2188666 : xfs_rmap_has_other_keys(
3040 : struct xfs_btree_cur *cur,
3041 : xfs_agblock_t bno,
3042 : xfs_extlen_t len,
3043 : const struct xfs_owner_info *oinfo,
3044 : bool *has_other)
3045 : {
3046 2188666 : struct xfs_rmap_matches res;
3047 2188666 : struct xfs_rmap_ownercount roc;
3048 2188666 : int error;
3049 :
3050 2188666 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, &res);
3051 2188418 : roc.stop_on_nonmatch = true;
3052 :
3053 2188418 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
3054 : xfs_rmap_count_owners_helper, &roc);
3055 2189045 : if (error == -ECANCELED) {
3056 0 : *has_other = true;
3057 0 : return 0;
3058 : }
3059 2189045 : if (error)
3060 : return error;
3061 :
3062 2189045 : *has_other = false;
3063 2189045 : return 0;
3064 : }
3065 :
3066 : const struct xfs_owner_info XFS_RMAP_OINFO_SKIP_UPDATE = {
3067 : .oi_owner = XFS_RMAP_OWN_NULL,
3068 : };
3069 : const struct xfs_owner_info XFS_RMAP_OINFO_ANY_OWNER = {
3070 : .oi_owner = XFS_RMAP_OWN_UNKNOWN,
3071 : };
3072 : const struct xfs_owner_info XFS_RMAP_OINFO_FS = {
3073 : .oi_owner = XFS_RMAP_OWN_FS,
3074 : };
3075 : const struct xfs_owner_info XFS_RMAP_OINFO_LOG = {
3076 : .oi_owner = XFS_RMAP_OWN_LOG,
3077 : };
3078 : const struct xfs_owner_info XFS_RMAP_OINFO_AG = {
3079 : .oi_owner = XFS_RMAP_OWN_AG,
3080 : };
3081 : const struct xfs_owner_info XFS_RMAP_OINFO_INOBT = {
3082 : .oi_owner = XFS_RMAP_OWN_INOBT,
3083 : };
3084 : const struct xfs_owner_info XFS_RMAP_OINFO_INODES = {
3085 : .oi_owner = XFS_RMAP_OWN_INODES,
3086 : };
3087 : const struct xfs_owner_info XFS_RMAP_OINFO_REFC = {
3088 : .oi_owner = XFS_RMAP_OWN_REFC,
3089 : };
3090 : const struct xfs_owner_info XFS_RMAP_OINFO_COW = {
3091 : .oi_owner = XFS_RMAP_OWN_COW,
3092 : };
3093 :
3094 : int __init
3095 50 : xfs_rmap_intent_init_cache(void)
3096 : {
3097 50 : xfs_rmap_intent_cache = kmem_cache_create("xfs_rmap_intent",
3098 : sizeof(struct xfs_rmap_intent),
3099 : 0, 0, NULL);
3100 :
3101 50 : return xfs_rmap_intent_cache != NULL ? 0 : -ENOMEM;
3102 : }
3103 :
3104 : void
3105 49 : xfs_rmap_intent_destroy_cache(void)
3106 : {
3107 49 : kmem_cache_destroy(xfs_rmap_intent_cache);
3108 49 : xfs_rmap_intent_cache = NULL;
3109 49 : }
|
