Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2014 Red Hat, Inc.
4 : * All Rights Reserved.
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_bit.h"
13 : #include "xfs_mount.h"
14 : #include "xfs_sb.h"
15 : #include "xfs_defer.h"
16 : #include "xfs_btree.h"
17 : #include "xfs_trans.h"
18 : #include "xfs_alloc.h"
19 : #include "xfs_rmap.h"
20 : #include "xfs_rmap_btree.h"
21 : #include "xfs_trace.h"
22 : #include "xfs_errortag.h"
23 : #include "xfs_error.h"
24 : #include "xfs_inode.h"
25 : #include "xfs_ag.h"
26 :
27 : struct kmem_cache *xfs_rmap_intent_cache;
28 :
29 : /*
30 : * Lookup the first record less than or equal to [bno, len, owner, offset]
31 : * in the btree given by cur.
32 : */
33 : int
34 400864765 : xfs_rmap_lookup_le(
35 : struct xfs_btree_cur *cur,
36 : xfs_agblock_t bno,
37 : uint64_t owner,
38 : uint64_t offset,
39 : unsigned int flags,
40 : struct xfs_rmap_irec *irec,
41 : int *stat)
42 : {
43 400864765 : int get_stat = 0;
44 400864765 : int error;
45 :
46 400864765 : cur->bc_rec.r.rm_startblock = bno;
47 400864765 : cur->bc_rec.r.rm_blockcount = 0;
48 400864765 : cur->bc_rec.r.rm_owner = owner;
49 400864765 : cur->bc_rec.r.rm_offset = offset;
50 400864765 : cur->bc_rec.r.rm_flags = flags;
51 :
52 400864765 : error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat);
53 400866895 : if (error || !(*stat) || !irec)
54 : return error;
55 :
56 395442013 : error = xfs_rmap_get_rec(cur, irec, &get_stat);
57 395444208 : if (error)
58 : return error;
59 395444208 : if (!get_stat)
60 0 : return -EFSCORRUPTED;
61 :
62 : return 0;
63 : }
64 :
65 : /*
66 : * Lookup the record exactly matching [bno, len, owner, offset]
67 : * in the btree given by cur.
68 : */
69 : int
70 0 : xfs_rmap_lookup_eq(
71 : struct xfs_btree_cur *cur,
72 : xfs_agblock_t bno,
73 : xfs_extlen_t len,
74 : uint64_t owner,
75 : uint64_t offset,
76 : unsigned int flags,
77 : int *stat)
78 : {
79 208907256 : cur->bc_rec.r.rm_startblock = bno;
80 208907256 : cur->bc_rec.r.rm_blockcount = len;
81 208907256 : cur->bc_rec.r.rm_owner = owner;
82 208907256 : cur->bc_rec.r.rm_offset = offset;
83 208907256 : cur->bc_rec.r.rm_flags = flags;
84 0 : return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat);
85 : }
86 :
87 : /*
88 : * Update the record referred to by cur to the value given
89 : * by [bno, len, owner, offset].
90 : * This either works (return 0) or gets an EFSCORRUPTED error.
91 : */
92 : STATIC int
93 28855959 : xfs_rmap_update(
94 : struct xfs_btree_cur *cur,
95 : struct xfs_rmap_irec *irec)
96 : {
97 28855959 : union xfs_btree_rec rec;
98 28855959 : int error;
99 :
100 28855959 : trace_xfs_rmap_update(cur->bc_mp, cur->bc_ag.pag->pag_agno,
101 : irec->rm_startblock, irec->rm_blockcount,
102 : irec->rm_owner, irec->rm_offset, irec->rm_flags);
103 :
104 28855989 : rec.rmap.rm_startblock = cpu_to_be32(irec->rm_startblock);
105 28855989 : rec.rmap.rm_blockcount = cpu_to_be32(irec->rm_blockcount);
106 28855989 : rec.rmap.rm_owner = cpu_to_be64(irec->rm_owner);
107 28855989 : rec.rmap.rm_offset = cpu_to_be64(
108 : xfs_rmap_irec_offset_pack(irec));
109 28855989 : error = xfs_btree_update(cur, &rec);
110 28855986 : if (error)
111 0 : trace_xfs_rmap_update_error(cur->bc_mp,
112 0 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
113 28855986 : return error;
114 : }
115 :
116 : int
117 71351853 : xfs_rmap_insert(
118 : struct xfs_btree_cur *rcur,
119 : xfs_agblock_t agbno,
120 : xfs_extlen_t len,
121 : uint64_t owner,
122 : uint64_t offset,
123 : unsigned int flags)
124 : {
125 71351853 : int i;
126 71351853 : int error;
127 :
128 71351853 : trace_xfs_rmap_insert(rcur->bc_mp, rcur->bc_ag.pag->pag_agno, agbno,
129 : len, owner, offset, flags);
130 :
131 71351870 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
132 71351940 : if (error)
133 0 : goto done;
134 71351940 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 0)) {
135 0 : error = -EFSCORRUPTED;
136 0 : goto done;
137 : }
138 :
139 71351940 : rcur->bc_rec.r.rm_startblock = agbno;
140 71351940 : rcur->bc_rec.r.rm_blockcount = len;
141 71351940 : rcur->bc_rec.r.rm_owner = owner;
142 71351940 : rcur->bc_rec.r.rm_offset = offset;
143 71351940 : rcur->bc_rec.r.rm_flags = flags;
144 71351940 : error = xfs_btree_insert(rcur, &i);
145 71351939 : if (error)
146 162 : goto done;
147 71351777 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
148 0 : error = -EFSCORRUPTED;
149 0 : goto done;
150 : }
151 71351939 : done:
152 71351939 : if (error)
153 162 : trace_xfs_rmap_insert_error(rcur->bc_mp,
154 162 : rcur->bc_ag.pag->pag_agno, error, _RET_IP_);
155 71351939 : return error;
156 : }
157 :
158 : STATIC int
159 49387047 : xfs_rmap_delete(
160 : struct xfs_btree_cur *rcur,
161 : xfs_agblock_t agbno,
162 : xfs_extlen_t len,
163 : uint64_t owner,
164 : uint64_t offset,
165 : unsigned int flags)
166 : {
167 49387047 : int i;
168 49387047 : int error;
169 :
170 49387047 : trace_xfs_rmap_delete(rcur->bc_mp, rcur->bc_ag.pag->pag_agno, agbno,
171 : len, owner, offset, flags);
172 :
173 49387048 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
174 49387078 : if (error)
175 0 : goto done;
176 49387078 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
177 0 : error = -EFSCORRUPTED;
178 0 : goto done;
179 : }
180 :
181 49387078 : error = xfs_btree_delete(rcur, &i);
182 49387041 : if (error)
183 1 : goto done;
184 49387040 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
185 0 : error = -EFSCORRUPTED;
186 0 : goto done;
187 : }
188 49387041 : done:
189 49387041 : if (error)
190 1 : trace_xfs_rmap_delete_error(rcur->bc_mp,
191 1 : rcur->bc_ag.pag->pag_agno, error, _RET_IP_);
192 49387041 : return error;
193 : }
194 :
195 : /* Convert an internal btree record to an rmap record. */
196 : xfs_failaddr_t
197 51532729622 : xfs_rmap_btrec_to_irec(
198 : const union xfs_btree_rec *rec,
199 : struct xfs_rmap_irec *irec)
200 : {
201 51532729622 : irec->rm_startblock = be32_to_cpu(rec->rmap.rm_startblock);
202 51532729622 : irec->rm_blockcount = be32_to_cpu(rec->rmap.rm_blockcount);
203 51532729622 : irec->rm_owner = be64_to_cpu(rec->rmap.rm_owner);
204 51532729622 : return xfs_rmap_irec_offset_unpack(be64_to_cpu(rec->rmap.rm_offset),
205 : irec);
206 : }
207 :
208 : /* Simple checks for rmap records. */
209 : xfs_failaddr_t
210 51588283030 : xfs_rmap_check_irec(
211 : struct xfs_btree_cur *cur,
212 : const struct xfs_rmap_irec *irec)
213 : {
214 51588283030 : struct xfs_mount *mp = cur->bc_mp;
215 51588283030 : bool is_inode;
216 51588283030 : bool is_unwritten;
217 51588283030 : bool is_bmbt;
218 51588283030 : bool is_attr;
219 :
220 51588283030 : if (irec->rm_blockcount == 0)
221 0 : return __this_address;
222 51588283030 : if (irec->rm_startblock <= XFS_AGFL_BLOCK(mp)) {
223 8676576 : if (irec->rm_owner != XFS_RMAP_OWN_FS)
224 0 : return __this_address;
225 8676576 : if (irec->rm_blockcount != XFS_AGFL_BLOCK(mp) + 1)
226 0 : return __this_address;
227 : } else {
228 : /* check for valid extent range, including overflow */
229 51579606454 : if (!xfs_verify_agbext(cur->bc_ag.pag, irec->rm_startblock,
230 51579606454 : irec->rm_blockcount))
231 0 : return __this_address;
232 : }
233 :
234 51588283030 : if (!(xfs_verify_ino(mp, irec->rm_owner) ||
235 1235044134 : (irec->rm_owner <= XFS_RMAP_OWN_FS &&
236 : irec->rm_owner >= XFS_RMAP_OWN_MIN)))
237 0 : return __this_address;
238 :
239 : /* Check flags. */
240 51917227221 : is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner);
241 51917227221 : is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK;
242 51917227221 : is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK;
243 51917227221 : is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN;
244 :
245 51917227221 : if (is_bmbt && irec->rm_offset != 0)
246 0 : return __this_address;
247 :
248 51917227221 : if (!is_inode && irec->rm_offset != 0)
249 0 : return __this_address;
250 :
251 51917227221 : if (is_unwritten && (is_bmbt || !is_inode || is_attr))
252 0 : return __this_address;
253 :
254 51917227221 : if (!is_inode && (is_bmbt || is_unwritten || is_attr))
255 0 : return __this_address;
256 :
257 : /* Check for a valid fork offset, if applicable. */
258 >10223*10^7 : if (is_inode && !is_bmbt &&
259 50253784387 : !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
260 0 : return __this_address;
261 :
262 : return NULL;
263 : }
264 :
265 : static inline int
266 0 : xfs_rmap_complain_bad_rec(
267 : struct xfs_btree_cur *cur,
268 : xfs_failaddr_t fa,
269 : const struct xfs_rmap_irec *irec)
270 : {
271 0 : struct xfs_mount *mp = cur->bc_mp;
272 :
273 0 : xfs_warn(mp,
274 : "Reverse Mapping BTree record corruption in AG %d detected at %pS!",
275 : cur->bc_ag.pag->pag_agno, fa);
276 0 : xfs_warn(mp,
277 : "Owner 0x%llx, flags 0x%x, start block 0x%x block count 0x%x",
278 : irec->rm_owner, irec->rm_flags, irec->rm_startblock,
279 : irec->rm_blockcount);
280 0 : return -EFSCORRUPTED;
281 : }
282 :
283 : /*
284 : * Get the data from the pointed-to record.
285 : */
286 : int
287 443717149 : xfs_rmap_get_rec(
288 : struct xfs_btree_cur *cur,
289 : struct xfs_rmap_irec *irec,
290 : int *stat)
291 : {
292 443717149 : union xfs_btree_rec *rec;
293 443717149 : xfs_failaddr_t fa;
294 443717149 : int error;
295 :
296 443717149 : error = xfs_btree_get_rec(cur, &rec, stat);
297 443723895 : if (error || !*stat)
298 : return error;
299 :
300 443726724 : fa = xfs_rmap_btrec_to_irec(rec, irec);
301 443727573 : if (!fa)
302 443727362 : fa = xfs_rmap_check_irec(cur, irec);
303 443725616 : if (fa)
304 0 : return xfs_rmap_complain_bad_rec(cur, fa, irec);
305 :
306 : return 0;
307 : }
308 :
309 : struct xfs_find_left_neighbor_info {
310 : struct xfs_rmap_irec high;
311 : struct xfs_rmap_irec *irec;
312 : };
313 :
314 : /* For each rmap given, figure out if it matches the key we want. */
315 : STATIC int
316 86143694 : xfs_rmap_find_left_neighbor_helper(
317 : struct xfs_btree_cur *cur,
318 : const struct xfs_rmap_irec *rec,
319 : void *priv)
320 : {
321 86143694 : struct xfs_find_left_neighbor_info *info = priv;
322 :
323 86143694 : trace_xfs_rmap_find_left_neighbor_candidate(cur->bc_mp,
324 86143694 : cur->bc_ag.pag->pag_agno, rec->rm_startblock,
325 86143694 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
326 86143694 : rec->rm_flags);
327 :
328 86143886 : if (rec->rm_owner != info->high.rm_owner)
329 : return 0;
330 23526779 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
331 23526779 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
332 23522557 : rec->rm_offset + rec->rm_blockcount - 1 != info->high.rm_offset)
333 : return 0;
334 :
335 9596248 : *info->irec = *rec;
336 9596248 : return -ECANCELED;
337 : }
338 :
339 : /*
340 : * Find the record to the left of the given extent, being careful only to
341 : * return a match with the same owner and adjacent physical and logical
342 : * block ranges.
343 : */
344 : STATIC int
345 73525210 : xfs_rmap_find_left_neighbor(
346 : struct xfs_btree_cur *cur,
347 : xfs_agblock_t bno,
348 : uint64_t owner,
349 : uint64_t offset,
350 : unsigned int flags,
351 : struct xfs_rmap_irec *irec,
352 : int *stat)
353 : {
354 73525210 : struct xfs_find_left_neighbor_info info;
355 73525210 : int found = 0;
356 73525210 : int error;
357 :
358 73525210 : *stat = 0;
359 73525210 : if (bno == 0)
360 : return 0;
361 73525210 : info.high.rm_startblock = bno - 1;
362 73525210 : info.high.rm_owner = owner;
363 73525210 : if (!XFS_RMAP_NON_INODE_OWNER(owner) &&
364 73525117 : !(flags & XFS_RMAP_BMBT_BLOCK)) {
365 73525210 : if (offset == 0)
366 : return 0;
367 73326675 : info.high.rm_offset = offset - 1;
368 : } else
369 0 : info.high.rm_offset = 0;
370 73326675 : info.high.rm_flags = flags;
371 73326675 : info.high.rm_blockcount = 0;
372 73326675 : info.irec = irec;
373 :
374 73326675 : trace_xfs_rmap_find_left_neighbor_query(cur->bc_mp,
375 73326675 : cur->bc_ag.pag->pag_agno, bno, 0, owner, offset, flags);
376 :
377 : /*
378 : * Historically, we always used the range query to walk every reverse
379 : * mapping that could possibly overlap the key that the caller asked
380 : * for, and filter out the ones that don't. That is very slow when
381 : * there are a lot of records.
382 : *
383 : * However, there are two scenarios where the classic btree search can
384 : * produce correct results -- if the index contains a record that is an
385 : * exact match for the lookup key; and if there are no other records
386 : * between the record we want and the key we supplied.
387 : *
388 : * As an optimization, try a non-overlapped lookup first. This makes
389 : * extent conversion and remap operations run a bit faster if the
390 : * physical extents aren't being shared. If we don't find what we
391 : * want, we fall back to the overlapped query.
392 : */
393 73326680 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
394 : &found);
395 73326789 : if (error)
396 : return error;
397 73326621 : if (found)
398 73326622 : error = xfs_rmap_find_left_neighbor_helper(cur, irec, &info);
399 73326667 : if (!error)
400 64097531 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
401 : xfs_rmap_find_left_neighbor_helper, &info);
402 73326666 : if (error != -ECANCELED)
403 : return error;
404 :
405 9596248 : *stat = 1;
406 9596248 : trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
407 9596248 : cur->bc_ag.pag->pag_agno, irec->rm_startblock,
408 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
409 : irec->rm_flags);
410 9596248 : return 0;
411 : }
412 :
413 : /* For each rmap given, figure out if it matches the key we want. */
414 : STATIC int
415 121979360 : xfs_rmap_lookup_le_range_helper(
416 : struct xfs_btree_cur *cur,
417 : const struct xfs_rmap_irec *rec,
418 : void *priv)
419 : {
420 121979360 : struct xfs_find_left_neighbor_info *info = priv;
421 :
422 121979360 : trace_xfs_rmap_lookup_le_range_candidate(cur->bc_mp,
423 121979360 : cur->bc_ag.pag->pag_agno, rec->rm_startblock,
424 121979360 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
425 121979360 : rec->rm_flags);
426 :
427 121979567 : if (rec->rm_owner != info->high.rm_owner)
428 : return 0;
429 114904016 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
430 114904016 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
431 114904016 : (rec->rm_offset > info->high.rm_offset ||
432 114879529 : rec->rm_offset + rec->rm_blockcount <= info->high.rm_offset))
433 : return 0;
434 :
435 114871383 : *info->irec = *rec;
436 114871383 : return -ECANCELED;
437 : }
438 :
439 : /*
440 : * Find the record to the left of the given extent, being careful only to
441 : * return a match with the same owner and overlapping physical and logical
442 : * block ranges. This is the overlapping-interval version of
443 : * xfs_rmap_lookup_le.
444 : */
445 : int
446 114870576 : xfs_rmap_lookup_le_range(
447 : struct xfs_btree_cur *cur,
448 : xfs_agblock_t bno,
449 : uint64_t owner,
450 : uint64_t offset,
451 : unsigned int flags,
452 : struct xfs_rmap_irec *irec,
453 : int *stat)
454 : {
455 114870576 : struct xfs_find_left_neighbor_info info;
456 114870576 : int found = 0;
457 114870576 : int error;
458 :
459 114870576 : info.high.rm_startblock = bno;
460 114870576 : info.high.rm_owner = owner;
461 114870576 : if (!XFS_RMAP_NON_INODE_OWNER(owner) && !(flags & XFS_RMAP_BMBT_BLOCK))
462 114870576 : info.high.rm_offset = offset;
463 : else
464 0 : info.high.rm_offset = 0;
465 114870576 : info.high.rm_flags = flags;
466 114870576 : info.high.rm_blockcount = 0;
467 114870576 : *stat = 0;
468 114870576 : info.irec = irec;
469 :
470 114870576 : trace_xfs_rmap_lookup_le_range(cur->bc_mp, cur->bc_ag.pag->pag_agno,
471 : bno, 0, owner, offset, flags);
472 :
473 : /*
474 : * Historically, we always used the range query to walk every reverse
475 : * mapping that could possibly overlap the key that the caller asked
476 : * for, and filter out the ones that don't. That is very slow when
477 : * there are a lot of records.
478 : *
479 : * However, there are two scenarios where the classic btree search can
480 : * produce correct results -- if the index contains a record that is an
481 : * exact match for the lookup key; and if there are no other records
482 : * between the record we want and the key we supplied.
483 : *
484 : * As an optimization, try a non-overlapped lookup first. This makes
485 : * scrub run much faster on most filesystems because bmbt records are
486 : * usually an exact match for rmap records. If we don't find what we
487 : * want, we fall back to the overlapped query.
488 : */
489 114870729 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
490 : &found);
491 114871448 : if (error)
492 : return error;
493 114871144 : if (found)
494 114871255 : error = xfs_rmap_lookup_le_range_helper(cur, irec, &info);
495 114871428 : if (!error)
496 87177 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
497 : xfs_rmap_lookup_le_range_helper, &info);
498 114871428 : if (error != -ECANCELED)
499 : return error;
500 :
501 114870925 : *stat = 1;
502 114870925 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
503 114870925 : cur->bc_ag.pag->pag_agno, irec->rm_startblock,
504 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
505 : irec->rm_flags);
506 114870925 : return 0;
507 : }
508 :
509 : /*
510 : * Perform all the relevant owner checks for a removal op. If we're doing an
511 : * unknown-owner removal then we have no owner information to check.
512 : */
513 : static int
514 37924708 : xfs_rmap_free_check_owner(
515 : struct xfs_mount *mp,
516 : uint64_t ltoff,
517 : struct xfs_rmap_irec *rec,
518 : xfs_filblks_t len,
519 : uint64_t owner,
520 : uint64_t offset,
521 : unsigned int flags)
522 : {
523 37924708 : int error = 0;
524 :
525 37924708 : if (owner == XFS_RMAP_OWN_UNKNOWN)
526 : return 0;
527 :
528 : /* Make sure the unwritten flag matches. */
529 37924319 : if (XFS_IS_CORRUPT(mp,
530 : (flags & XFS_RMAP_UNWRITTEN) !=
531 : (rec->rm_flags & XFS_RMAP_UNWRITTEN))) {
532 0 : error = -EFSCORRUPTED;
533 0 : goto out;
534 : }
535 :
536 : /* Make sure the owner matches what we expect to find in the tree. */
537 37924319 : if (XFS_IS_CORRUPT(mp, owner != rec->rm_owner)) {
538 0 : error = -EFSCORRUPTED;
539 0 : goto out;
540 : }
541 :
542 : /* Check the offset, if necessary. */
543 37924319 : if (XFS_RMAP_NON_INODE_OWNER(owner))
544 2570463 : goto out;
545 :
546 35353856 : if (flags & XFS_RMAP_BMBT_BLOCK) {
547 229442 : if (XFS_IS_CORRUPT(mp,
548 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK))) {
549 0 : error = -EFSCORRUPTED;
550 0 : goto out;
551 : }
552 : } else {
553 35124414 : if (XFS_IS_CORRUPT(mp, rec->rm_offset > offset)) {
554 0 : error = -EFSCORRUPTED;
555 0 : goto out;
556 : }
557 35124414 : if (XFS_IS_CORRUPT(mp,
558 : offset + len > ltoff + rec->rm_blockcount)) {
559 0 : error = -EFSCORRUPTED;
560 0 : goto out;
561 : }
562 : }
563 :
564 : out:
565 : return error;
566 : }
567 :
568 : /*
569 : * Find the extent in the rmap btree and remove it.
570 : *
571 : * The record we find should always be an exact match for the extent that we're
572 : * looking for, since we insert them into the btree without modification.
573 : *
574 : * Special Case #1: when growing the filesystem, we "free" an extent when
575 : * growing the last AG. This extent is new space and so it is not tracked as
576 : * used space in the btree. The growfs code will pass in an owner of
577 : * XFS_RMAP_OWN_NULL to indicate that it expected that there is no owner of this
578 : * extent. We verify that - the extent lookup result in a record that does not
579 : * overlap.
580 : *
581 : * Special Case #2: EFIs do not record the owner of the extent, so when
582 : * recovering EFIs from the log we pass in XFS_RMAP_OWN_UNKNOWN to tell the rmap
583 : * btree to ignore the owner (i.e. wildcard match) so we don't trigger
584 : * corruption checks during log recovery.
585 : */
586 : STATIC int
587 37924924 : xfs_rmap_unmap(
588 : struct xfs_btree_cur *cur,
589 : xfs_agblock_t bno,
590 : xfs_extlen_t len,
591 : bool unwritten,
592 : const struct xfs_owner_info *oinfo)
593 : {
594 37924924 : struct xfs_mount *mp = cur->bc_mp;
595 37924924 : struct xfs_rmap_irec ltrec;
596 37924924 : uint64_t ltoff;
597 37924924 : int error = 0;
598 37924924 : int i;
599 37924924 : uint64_t owner;
600 37924924 : uint64_t offset;
601 37924924 : unsigned int flags;
602 37924924 : bool ignore_off;
603 :
604 37924924 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
605 37924924 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
606 35353936 : (flags & XFS_RMAP_BMBT_BLOCK);
607 37924924 : if (unwritten)
608 1277380 : flags |= XFS_RMAP_UNWRITTEN;
609 37924924 : trace_xfs_rmap_unmap(mp, cur->bc_ag.pag->pag_agno, bno, len,
610 : unwritten, oinfo);
611 :
612 : /*
613 : * We should always have a left record because there's a static record
614 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
615 : * will not ever be removed from the tree.
616 : */
617 37924992 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec, &i);
618 37924920 : if (error)
619 69 : goto out_error;
620 37924851 : if (XFS_IS_CORRUPT(mp, i != 1)) {
621 0 : error = -EFSCORRUPTED;
622 0 : goto out_error;
623 : }
624 :
625 37924851 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
626 37924851 : cur->bc_ag.pag->pag_agno, ltrec.rm_startblock,
627 : ltrec.rm_blockcount, ltrec.rm_owner,
628 : ltrec.rm_offset, ltrec.rm_flags);
629 37924872 : ltoff = ltrec.rm_offset;
630 :
631 : /*
632 : * For growfs, the incoming extent must be beyond the left record we
633 : * just found as it is new space and won't be used by anyone. This is
634 : * just a corruption check as we don't actually do anything with this
635 : * extent. Note that we need to use >= instead of > because it might
636 : * be the case that the "left" extent goes all the way to EOFS.
637 : */
638 37924872 : if (owner == XFS_RMAP_OWN_NULL) {
639 28 : if (XFS_IS_CORRUPT(mp,
640 : bno <
641 : ltrec.rm_startblock + ltrec.rm_blockcount)) {
642 0 : error = -EFSCORRUPTED;
643 0 : goto out_error;
644 : }
645 28 : goto out_done;
646 : }
647 :
648 : /*
649 : * If we're doing an unknown-owner removal for EFI recovery, we expect
650 : * to find the full range in the rmapbt or nothing at all. If we
651 : * don't find any rmaps overlapping either end of the range, we're
652 : * done. Hopefully this means that the EFI creator already queued
653 : * (and finished) a RUI to remove the rmap.
654 : */
655 37924844 : if (owner == XFS_RMAP_OWN_UNKNOWN &&
656 489 : ltrec.rm_startblock + ltrec.rm_blockcount <= bno) {
657 100 : struct xfs_rmap_irec rtrec;
658 :
659 100 : error = xfs_btree_increment(cur, 0, &i);
660 100 : if (error)
661 0 : goto out_error;
662 100 : if (i == 0)
663 100 : goto out_done;
664 94 : error = xfs_rmap_get_rec(cur, &rtrec, &i);
665 94 : if (error)
666 0 : goto out_error;
667 94 : if (XFS_IS_CORRUPT(mp, i != 1)) {
668 0 : error = -EFSCORRUPTED;
669 0 : goto out_error;
670 : }
671 94 : if (rtrec.rm_startblock >= bno + len)
672 94 : goto out_done;
673 : }
674 :
675 : /* Make sure the extent we found covers the entire freeing range. */
676 37924744 : if (XFS_IS_CORRUPT(mp,
677 : ltrec.rm_startblock > bno ||
678 : ltrec.rm_startblock + ltrec.rm_blockcount <
679 : bno + len)) {
680 0 : error = -EFSCORRUPTED;
681 0 : goto out_error;
682 : }
683 :
684 : /* Check owner information. */
685 37924744 : error = xfs_rmap_free_check_owner(mp, ltoff, <rec, len, owner,
686 : offset, flags);
687 37924749 : if (error)
688 0 : goto out_error;
689 :
690 37924749 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
691 : /* exact match, simply remove the record from rmap tree */
692 32844986 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
693 : ltrec.rm_startblock, ltrec.rm_blockcount,
694 : ltrec.rm_owner, ltrec.rm_offset,
695 : ltrec.rm_flags);
696 32844949 : error = xfs_btree_delete(cur, &i);
697 32844879 : if (error)
698 1 : goto out_error;
699 32844878 : if (XFS_IS_CORRUPT(mp, i != 1)) {
700 0 : error = -EFSCORRUPTED;
701 0 : goto out_error;
702 : }
703 5079763 : } else if (ltrec.rm_startblock == bno) {
704 : /*
705 : * overlap left hand side of extent: move the start, trim the
706 : * length and update the current record.
707 : *
708 : * ltbno ltlen
709 : * Orig: |oooooooooooooooooooo|
710 : * Freeing: |fffffffff|
711 : * Result: |rrrrrrrrrr|
712 : * bno len
713 : */
714 924699 : ltrec.rm_startblock += len;
715 924699 : ltrec.rm_blockcount -= len;
716 924699 : if (!ignore_off)
717 130221 : ltrec.rm_offset += len;
718 924699 : error = xfs_rmap_update(cur, <rec);
719 924699 : if (error)
720 0 : goto out_error;
721 4155064 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
722 : /*
723 : * overlap right hand side of extent: trim the length and update
724 : * the current record.
725 : *
726 : * ltbno ltlen
727 : * Orig: |oooooooooooooooooooo|
728 : * Freeing: |fffffffff|
729 : * Result: |rrrrrrrrrr|
730 : * bno len
731 : */
732 390807 : ltrec.rm_blockcount -= len;
733 390807 : error = xfs_rmap_update(cur, <rec);
734 390807 : if (error)
735 0 : goto out_error;
736 : } else {
737 :
738 : /*
739 : * overlap middle of extent: trim the length of the existing
740 : * record to the length of the new left-extent size, increment
741 : * the insertion position so we can insert a new record
742 : * containing the remaining right-extent space.
743 : *
744 : * ltbno ltlen
745 : * Orig: |oooooooooooooooooooo|
746 : * Freeing: |fffffffff|
747 : * Result: |rrrrr| |rrrr|
748 : * bno len
749 : */
750 3764257 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
751 :
752 3764257 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
753 3764257 : error = xfs_rmap_update(cur, <rec);
754 3764257 : if (error)
755 0 : goto out_error;
756 :
757 3764257 : error = xfs_btree_increment(cur, 0, &i);
758 3764257 : if (error)
759 0 : goto out_error;
760 :
761 3764257 : cur->bc_rec.r.rm_startblock = bno + len;
762 3764257 : cur->bc_rec.r.rm_blockcount = orig_len - len -
763 3764257 : ltrec.rm_blockcount;
764 3764257 : cur->bc_rec.r.rm_owner = ltrec.rm_owner;
765 3764257 : if (ignore_off)
766 657709 : cur->bc_rec.r.rm_offset = 0;
767 : else
768 3106548 : cur->bc_rec.r.rm_offset = offset + len;
769 3764257 : cur->bc_rec.r.rm_flags = flags;
770 3764257 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno,
771 : cur->bc_rec.r.rm_startblock,
772 : cur->bc_rec.r.rm_blockcount,
773 : cur->bc_rec.r.rm_owner,
774 : cur->bc_rec.r.rm_offset,
775 : cur->bc_rec.r.rm_flags);
776 3764257 : error = xfs_btree_insert(cur, &i);
777 3764257 : if (error)
778 0 : goto out_error;
779 : }
780 :
781 3764257 : out_done:
782 37924769 : trace_xfs_rmap_unmap_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
783 : unwritten, oinfo);
784 37924800 : out_error:
785 37924800 : if (error)
786 70 : trace_xfs_rmap_unmap_error(mp, cur->bc_ag.pag->pag_agno,
787 70 : error, _RET_IP_);
788 37924800 : return error;
789 : }
790 :
791 : /*
792 : * Remove a reference to an extent in the rmap btree.
793 : */
794 : int
795 548510 : xfs_rmap_free(
796 : struct xfs_trans *tp,
797 : struct xfs_buf *agbp,
798 : struct xfs_perag *pag,
799 : xfs_agblock_t bno,
800 : xfs_extlen_t len,
801 : const struct xfs_owner_info *oinfo)
802 : {
803 548510 : struct xfs_mount *mp = tp->t_mountp;
804 548510 : struct xfs_btree_cur *cur;
805 548510 : int error;
806 :
807 548510 : if (!xfs_has_rmapbt(mp))
808 : return 0;
809 :
810 512221 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
811 :
812 512221 : error = xfs_rmap_unmap(cur, bno, len, false, oinfo);
813 :
814 512221 : xfs_btree_del_cursor(cur, error);
815 512221 : return error;
816 : }
817 :
818 : /*
819 : * A mergeable rmap must have the same owner and the same values for
820 : * the unwritten, attr_fork, and bmbt flags. The startblock and
821 : * offset are checked separately.
822 : */
823 : static bool
824 89524953 : xfs_rmap_is_mergeable(
825 : struct xfs_rmap_irec *irec,
826 : uint64_t owner,
827 : unsigned int flags)
828 : {
829 89524953 : if (irec->rm_owner == XFS_RMAP_OWN_NULL)
830 : return false;
831 89524953 : if (irec->rm_owner != owner)
832 : return false;
833 53938261 : if ((flags & XFS_RMAP_UNWRITTEN) ^
834 53938261 : (irec->rm_flags & XFS_RMAP_UNWRITTEN))
835 : return false;
836 52786051 : if ((flags & XFS_RMAP_ATTR_FORK) ^
837 : (irec->rm_flags & XFS_RMAP_ATTR_FORK))
838 : return false;
839 52783824 : if ((flags & XFS_RMAP_BMBT_BLOCK) ^
840 : (irec->rm_flags & XFS_RMAP_BMBT_BLOCK))
841 89931 : return false;
842 : return true;
843 : }
844 :
845 : /*
846 : * When we allocate a new block, the first thing we do is add a reference to
847 : * the extent in the rmap btree. This takes the form of a [agbno, length,
848 : * owner, offset] record. Flags are encoded in the high bits of the offset
849 : * field.
850 : */
851 : STATIC int
852 41725526 : xfs_rmap_map(
853 : struct xfs_btree_cur *cur,
854 : xfs_agblock_t bno,
855 : xfs_extlen_t len,
856 : bool unwritten,
857 : const struct xfs_owner_info *oinfo)
858 : {
859 41725526 : struct xfs_mount *mp = cur->bc_mp;
860 41725526 : struct xfs_rmap_irec ltrec;
861 41725526 : struct xfs_rmap_irec gtrec;
862 41725526 : int have_gt;
863 41725526 : int have_lt;
864 41725526 : int error = 0;
865 41725526 : int i;
866 41725526 : uint64_t owner;
867 41725526 : uint64_t offset;
868 41725526 : unsigned int flags = 0;
869 41725526 : bool ignore_off;
870 :
871 41725526 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
872 41725526 : ASSERT(owner != 0);
873 41725526 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
874 40755101 : (flags & XFS_RMAP_BMBT_BLOCK);
875 41725526 : if (unwritten)
876 9068869 : flags |= XFS_RMAP_UNWRITTEN;
877 41725526 : trace_xfs_rmap_map(mp, cur->bc_ag.pag->pag_agno, bno, len,
878 : unwritten, oinfo);
879 41725701 : ASSERT(!xfs_rmap_should_skip_owner_update(oinfo));
880 :
881 : /*
882 : * For the initial lookup, look for an exact match or the left-adjacent
883 : * record for our insertion point. This will also give us the record for
884 : * start block contiguity tests.
885 : */
886 41725701 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec,
887 : &have_lt);
888 41725247 : if (error)
889 468 : goto out_error;
890 41724779 : if (have_lt) {
891 41725137 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
892 41725137 : cur->bc_ag.pag->pag_agno, ltrec.rm_startblock,
893 : ltrec.rm_blockcount, ltrec.rm_owner,
894 : ltrec.rm_offset, ltrec.rm_flags);
895 :
896 41725208 : if (!xfs_rmap_is_mergeable(<rec, owner, flags))
897 19101622 : have_lt = 0;
898 : }
899 :
900 41724850 : if (XFS_IS_CORRUPT(mp,
901 : have_lt != 0 &&
902 : ltrec.rm_startblock + ltrec.rm_blockcount > bno)) {
903 0 : error = -EFSCORRUPTED;
904 0 : goto out_error;
905 : }
906 :
907 : /*
908 : * Increment the cursor to see if we have a right-adjacent record to our
909 : * insertion point. This will give us the record for end block
910 : * contiguity tests.
911 : */
912 41724850 : error = xfs_btree_increment(cur, 0, &have_gt);
913 41724735 : if (error)
914 1 : goto out_error;
915 41724734 : if (have_gt) {
916 37008045 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
917 37008386 : if (error)
918 0 : goto out_error;
919 37008386 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
920 0 : error = -EFSCORRUPTED;
921 0 : goto out_error;
922 : }
923 37008386 : if (XFS_IS_CORRUPT(mp, bno + len > gtrec.rm_startblock)) {
924 0 : error = -EFSCORRUPTED;
925 0 : goto out_error;
926 : }
927 37008386 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
928 37008386 : cur->bc_ag.pag->pag_agno, gtrec.rm_startblock,
929 : gtrec.rm_blockcount, gtrec.rm_owner,
930 : gtrec.rm_offset, gtrec.rm_flags);
931 37008474 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
932 17290282 : have_gt = 0;
933 : }
934 :
935 : /*
936 : * Note: cursor currently points one record to the right of ltrec, even
937 : * if there is no record in the tree to the right.
938 : */
939 41725163 : if (have_lt &&
940 22623271 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
941 4548026 : (ignore_off || ltrec.rm_offset + ltrec.rm_blockcount == offset)) {
942 : /*
943 : * left edge contiguous, merge into left record.
944 : *
945 : * ltbno ltlen
946 : * orig: |ooooooooo|
947 : * adding: |aaaaaaaaa|
948 : * result: |rrrrrrrrrrrrrrrrrrr|
949 : * bno len
950 : */
951 3660918 : ltrec.rm_blockcount += len;
952 3660918 : if (have_gt &&
953 1323205 : bno + len == gtrec.rm_startblock &&
954 2249 : (ignore_off || offset + len == gtrec.rm_offset) &&
955 35392 : (unsigned long)ltrec.rm_blockcount + len +
956 35392 : gtrec.rm_blockcount <= XFS_RMAP_LEN_MAX) {
957 : /*
958 : * right edge also contiguous, delete right record
959 : * and merge into left record.
960 : *
961 : * ltbno ltlen gtbno gtlen
962 : * orig: |ooooooooo| |ooooooooo|
963 : * adding: |aaaaaaaaa|
964 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
965 : */
966 35392 : ltrec.rm_blockcount += gtrec.rm_blockcount;
967 35392 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
968 : gtrec.rm_startblock,
969 : gtrec.rm_blockcount,
970 : gtrec.rm_owner,
971 : gtrec.rm_offset,
972 : gtrec.rm_flags);
973 35392 : error = xfs_btree_delete(cur, &i);
974 35392 : if (error)
975 0 : goto out_error;
976 35392 : if (XFS_IS_CORRUPT(mp, i != 1)) {
977 0 : error = -EFSCORRUPTED;
978 0 : goto out_error;
979 : }
980 : }
981 :
982 : /* point the cursor back to the left record and update */
983 3660918 : error = xfs_btree_decrement(cur, 0, &have_gt);
984 3660923 : if (error)
985 0 : goto out_error;
986 3660923 : error = xfs_rmap_update(cur, <rec);
987 3660923 : if (error)
988 0 : goto out_error;
989 38064245 : } else if (have_gt &&
990 18395009 : bno + len == gtrec.rm_startblock &&
991 600132 : (ignore_off || offset + len == gtrec.rm_offset)) {
992 : /*
993 : * right edge contiguous, merge into right record.
994 : *
995 : * gtbno gtlen
996 : * Orig: |ooooooooo|
997 : * adding: |aaaaaaaaa|
998 : * Result: |rrrrrrrrrrrrrrrrrrr|
999 : * bno len
1000 : */
1001 115680 : gtrec.rm_startblock = bno;
1002 115680 : gtrec.rm_blockcount += len;
1003 115680 : if (!ignore_off)
1004 3811 : gtrec.rm_offset = offset;
1005 115680 : error = xfs_rmap_update(cur, >rec);
1006 115680 : if (error)
1007 0 : goto out_error;
1008 : } else {
1009 : /*
1010 : * no contiguous edge with identical owner, insert
1011 : * new record at current cursor position.
1012 : */
1013 37948565 : cur->bc_rec.r.rm_startblock = bno;
1014 37948565 : cur->bc_rec.r.rm_blockcount = len;
1015 37948565 : cur->bc_rec.r.rm_owner = owner;
1016 37948565 : cur->bc_rec.r.rm_offset = offset;
1017 37948565 : cur->bc_rec.r.rm_flags = flags;
1018 37948565 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1019 : owner, offset, flags);
1020 37948517 : error = xfs_btree_insert(cur, &i);
1021 37948563 : if (error)
1022 26 : goto out_error;
1023 37948537 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1024 0 : error = -EFSCORRUPTED;
1025 0 : goto out_error;
1026 : }
1027 : }
1028 :
1029 41725140 : trace_xfs_rmap_map_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1030 : unwritten, oinfo);
1031 41725683 : out_error:
1032 41725683 : if (error)
1033 495 : trace_xfs_rmap_map_error(mp, cur->bc_ag.pag->pag_agno,
1034 495 : error, _RET_IP_);
1035 41725683 : return error;
1036 : }
1037 :
1038 : /*
1039 : * Add a reference to an extent in the rmap btree.
1040 : */
1041 : int
1042 1223337 : xfs_rmap_alloc(
1043 : struct xfs_trans *tp,
1044 : struct xfs_buf *agbp,
1045 : struct xfs_perag *pag,
1046 : xfs_agblock_t bno,
1047 : xfs_extlen_t len,
1048 : const struct xfs_owner_info *oinfo)
1049 : {
1050 1223337 : struct xfs_mount *mp = tp->t_mountp;
1051 1223337 : struct xfs_btree_cur *cur;
1052 1223337 : int error;
1053 :
1054 1223337 : if (!xfs_has_rmapbt(mp))
1055 : return 0;
1056 :
1057 1031999 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
1058 1031997 : error = xfs_rmap_map(cur, bno, len, false, oinfo);
1059 :
1060 1031998 : xfs_btree_del_cursor(cur, error);
1061 1031998 : return error;
1062 : }
1063 :
1064 : #define RMAP_LEFT_CONTIG (1 << 0)
1065 : #define RMAP_RIGHT_CONTIG (1 << 1)
1066 : #define RMAP_LEFT_FILLING (1 << 2)
1067 : #define RMAP_RIGHT_FILLING (1 << 3)
1068 : #define RMAP_LEFT_VALID (1 << 6)
1069 : #define RMAP_RIGHT_VALID (1 << 7)
1070 :
1071 : #define LEFT r[0]
1072 : #define RIGHT r[1]
1073 : #define PREV r[2]
1074 : #define NEW r[3]
1075 :
1076 : /*
1077 : * Convert an unwritten extent to a real extent or vice versa.
1078 : * Does not handle overlapping extents.
1079 : */
1080 : STATIC int
1081 5423872 : xfs_rmap_convert(
1082 : struct xfs_btree_cur *cur,
1083 : xfs_agblock_t bno,
1084 : xfs_extlen_t len,
1085 : bool unwritten,
1086 : const struct xfs_owner_info *oinfo)
1087 : {
1088 5423872 : struct xfs_mount *mp = cur->bc_mp;
1089 5423872 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1090 : /* left is 0, right is 1, */
1091 : /* prev is 2, new is 3 */
1092 5423872 : uint64_t owner;
1093 5423872 : uint64_t offset;
1094 5423872 : uint64_t new_endoff;
1095 5423872 : unsigned int oldext;
1096 5423872 : unsigned int newext;
1097 5423872 : unsigned int flags = 0;
1098 5423872 : int i;
1099 5423872 : int state = 0;
1100 5423872 : int error;
1101 :
1102 5423872 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1103 5423872 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1104 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1105 5423872 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1106 5423872 : new_endoff = offset + len;
1107 5423872 : trace_xfs_rmap_convert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1108 : unwritten, oinfo);
1109 :
1110 : /*
1111 : * For the initial lookup, look for an exact match or the left-adjacent
1112 : * record for our insertion point. This will also give us the record for
1113 : * start block contiguity tests.
1114 : */
1115 5423873 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, &PREV, &i);
1116 5423865 : if (error)
1117 0 : goto done;
1118 5423865 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1119 0 : error = -EFSCORRUPTED;
1120 0 : goto done;
1121 : }
1122 :
1123 5423865 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
1124 5423865 : cur->bc_ag.pag->pag_agno, PREV.rm_startblock,
1125 : PREV.rm_blockcount, PREV.rm_owner,
1126 : PREV.rm_offset, PREV.rm_flags);
1127 :
1128 5423864 : ASSERT(PREV.rm_offset <= offset);
1129 5423864 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1130 5423864 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1131 5423864 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1132 :
1133 : /*
1134 : * Set flags determining what part of the previous oldext allocation
1135 : * extent is being replaced by a newext allocation.
1136 : */
1137 5423864 : if (PREV.rm_offset == offset)
1138 5220688 : state |= RMAP_LEFT_FILLING;
1139 5423864 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1140 5074353 : state |= RMAP_RIGHT_FILLING;
1141 :
1142 : /*
1143 : * Decrement the cursor to see if we have a left-adjacent record to our
1144 : * insertion point. This will give us the record for end block
1145 : * contiguity tests.
1146 : */
1147 5423864 : error = xfs_btree_decrement(cur, 0, &i);
1148 5423874 : if (error)
1149 0 : goto done;
1150 5423874 : if (i) {
1151 5423871 : state |= RMAP_LEFT_VALID;
1152 5423871 : error = xfs_rmap_get_rec(cur, &LEFT, &i);
1153 5423872 : if (error)
1154 0 : goto done;
1155 5423872 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1156 0 : error = -EFSCORRUPTED;
1157 0 : goto done;
1158 : }
1159 5423872 : if (XFS_IS_CORRUPT(mp,
1160 : LEFT.rm_startblock + LEFT.rm_blockcount >
1161 : bno)) {
1162 0 : error = -EFSCORRUPTED;
1163 0 : goto done;
1164 : }
1165 5423872 : trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
1166 5423872 : cur->bc_ag.pag->pag_agno, LEFT.rm_startblock,
1167 : LEFT.rm_blockcount, LEFT.rm_owner,
1168 : LEFT.rm_offset, LEFT.rm_flags);
1169 5423872 : if (LEFT.rm_startblock + LEFT.rm_blockcount == bno &&
1170 5049871 : LEFT.rm_offset + LEFT.rm_blockcount == offset &&
1171 330083 : xfs_rmap_is_mergeable(&LEFT, owner, newext))
1172 327250 : state |= RMAP_LEFT_CONTIG;
1173 : }
1174 :
1175 : /*
1176 : * Increment the cursor to see if we have a right-adjacent record to our
1177 : * insertion point. This will give us the record for end block
1178 : * contiguity tests.
1179 : */
1180 5423875 : error = xfs_btree_increment(cur, 0, &i);
1181 5423874 : if (error)
1182 0 : goto done;
1183 5423874 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1184 0 : error = -EFSCORRUPTED;
1185 0 : goto done;
1186 : }
1187 5423874 : error = xfs_btree_increment(cur, 0, &i);
1188 5423874 : if (error)
1189 0 : goto done;
1190 5423874 : if (i) {
1191 5049751 : state |= RMAP_RIGHT_VALID;
1192 5049751 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1193 5049749 : if (error)
1194 0 : goto done;
1195 5049749 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1196 0 : error = -EFSCORRUPTED;
1197 0 : goto done;
1198 : }
1199 5049749 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1200 0 : error = -EFSCORRUPTED;
1201 0 : goto done;
1202 : }
1203 5049749 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1204 5049749 : cur->bc_ag.pag->pag_agno, RIGHT.rm_startblock,
1205 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1206 : RIGHT.rm_offset, RIGHT.rm_flags);
1207 5049750 : if (bno + len == RIGHT.rm_startblock &&
1208 3591292 : offset + len == RIGHT.rm_offset &&
1209 64790 : xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1210 63219 : state |= RMAP_RIGHT_CONTIG;
1211 : }
1212 :
1213 : /* check that left + prev + right is not too long */
1214 5423873 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1215 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1216 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1217 17186 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1218 17186 : (unsigned long)LEFT.rm_blockcount + len +
1219 17186 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1220 0 : state &= ~RMAP_RIGHT_CONTIG;
1221 :
1222 5423872 : trace_xfs_rmap_convert_state(mp, cur->bc_ag.pag->pag_agno, state,
1223 5423873 : _RET_IP_);
1224 :
1225 : /* reset the cursor back to PREV */
1226 5423872 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, NULL, &i);
1227 5423872 : if (error)
1228 0 : goto done;
1229 5423872 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1230 0 : error = -EFSCORRUPTED;
1231 0 : goto done;
1232 : }
1233 :
1234 : /*
1235 : * Switch out based on the FILLING and CONTIG state bits.
1236 : */
1237 5423872 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1238 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1239 17186 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1240 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1241 : /*
1242 : * Setting all of a previous oldext extent to newext.
1243 : * The left and right neighbors are both contiguous with new.
1244 : */
1245 17186 : error = xfs_btree_increment(cur, 0, &i);
1246 17186 : if (error)
1247 0 : goto done;
1248 17186 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1249 0 : error = -EFSCORRUPTED;
1250 0 : goto done;
1251 : }
1252 17186 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1253 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1254 : RIGHT.rm_owner, RIGHT.rm_offset,
1255 : RIGHT.rm_flags);
1256 17186 : error = xfs_btree_delete(cur, &i);
1257 17186 : if (error)
1258 0 : goto done;
1259 17186 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1260 0 : error = -EFSCORRUPTED;
1261 0 : goto done;
1262 : }
1263 17186 : error = xfs_btree_decrement(cur, 0, &i);
1264 17186 : if (error)
1265 0 : goto done;
1266 17186 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1267 0 : error = -EFSCORRUPTED;
1268 0 : goto done;
1269 : }
1270 17186 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1271 : PREV.rm_startblock, PREV.rm_blockcount,
1272 : PREV.rm_owner, PREV.rm_offset,
1273 : PREV.rm_flags);
1274 17186 : error = xfs_btree_delete(cur, &i);
1275 17186 : if (error)
1276 0 : goto done;
1277 17186 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1278 0 : error = -EFSCORRUPTED;
1279 0 : goto done;
1280 : }
1281 17186 : error = xfs_btree_decrement(cur, 0, &i);
1282 17186 : if (error)
1283 0 : goto done;
1284 17186 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1285 0 : error = -EFSCORRUPTED;
1286 0 : goto done;
1287 : }
1288 17186 : NEW = LEFT;
1289 17186 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1290 17186 : error = xfs_rmap_update(cur, &NEW);
1291 17186 : if (error)
1292 0 : goto done;
1293 : break;
1294 :
1295 228035 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1296 : /*
1297 : * Setting all of a previous oldext extent to newext.
1298 : * The left neighbor is contiguous, the right is not.
1299 : */
1300 228035 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1301 : PREV.rm_startblock, PREV.rm_blockcount,
1302 : PREV.rm_owner, PREV.rm_offset,
1303 : PREV.rm_flags);
1304 228035 : error = xfs_btree_delete(cur, &i);
1305 228035 : if (error)
1306 0 : goto done;
1307 228035 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1308 0 : error = -EFSCORRUPTED;
1309 0 : goto done;
1310 : }
1311 228035 : error = xfs_btree_decrement(cur, 0, &i);
1312 228035 : if (error)
1313 0 : goto done;
1314 228035 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1315 0 : error = -EFSCORRUPTED;
1316 0 : goto done;
1317 : }
1318 228035 : NEW = LEFT;
1319 228035 : NEW.rm_blockcount += PREV.rm_blockcount;
1320 228035 : error = xfs_rmap_update(cur, &NEW);
1321 228035 : if (error)
1322 0 : goto done;
1323 : break;
1324 :
1325 19119 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1326 : /*
1327 : * Setting all of a previous oldext extent to newext.
1328 : * The right neighbor is contiguous, the left is not.
1329 : */
1330 19119 : error = xfs_btree_increment(cur, 0, &i);
1331 19119 : if (error)
1332 0 : goto done;
1333 19119 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1334 0 : error = -EFSCORRUPTED;
1335 0 : goto done;
1336 : }
1337 19119 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1338 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1339 : RIGHT.rm_owner, RIGHT.rm_offset,
1340 : RIGHT.rm_flags);
1341 19119 : error = xfs_btree_delete(cur, &i);
1342 19119 : if (error)
1343 0 : goto done;
1344 19119 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1345 0 : error = -EFSCORRUPTED;
1346 0 : goto done;
1347 : }
1348 19119 : error = xfs_btree_decrement(cur, 0, &i);
1349 19119 : if (error)
1350 0 : goto done;
1351 19119 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1352 0 : error = -EFSCORRUPTED;
1353 0 : goto done;
1354 : }
1355 19119 : NEW = PREV;
1356 19119 : NEW.rm_blockcount = len + RIGHT.rm_blockcount;
1357 19119 : NEW.rm_flags = newext;
1358 19119 : error = xfs_rmap_update(cur, &NEW);
1359 19119 : if (error)
1360 0 : goto done;
1361 : break;
1362 :
1363 4732963 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1364 : /*
1365 : * Setting all of a previous oldext extent to newext.
1366 : * Neither the left nor right neighbors are contiguous with
1367 : * the new one.
1368 : */
1369 4732963 : NEW = PREV;
1370 4732963 : NEW.rm_flags = newext;
1371 4732963 : error = xfs_rmap_update(cur, &NEW);
1372 4732961 : if (error)
1373 0 : goto done;
1374 : break;
1375 :
1376 82029 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1377 : /*
1378 : * Setting the first part of a previous oldext extent to newext.
1379 : * The left neighbor is contiguous.
1380 : */
1381 82029 : NEW = PREV;
1382 82029 : NEW.rm_offset += len;
1383 82029 : NEW.rm_startblock += len;
1384 82029 : NEW.rm_blockcount -= len;
1385 82029 : error = xfs_rmap_update(cur, &NEW);
1386 82029 : if (error)
1387 0 : goto done;
1388 82029 : error = xfs_btree_decrement(cur, 0, &i);
1389 82029 : if (error)
1390 0 : goto done;
1391 82029 : NEW = LEFT;
1392 82029 : NEW.rm_blockcount += len;
1393 82029 : error = xfs_rmap_update(cur, &NEW);
1394 82029 : if (error)
1395 0 : goto done;
1396 : break;
1397 :
1398 141359 : case RMAP_LEFT_FILLING:
1399 : /*
1400 : * Setting the first part of a previous oldext extent to newext.
1401 : * The left neighbor is not contiguous.
1402 : */
1403 141359 : NEW = PREV;
1404 141359 : NEW.rm_startblock += len;
1405 141359 : NEW.rm_offset += len;
1406 141359 : NEW.rm_blockcount -= len;
1407 141359 : error = xfs_rmap_update(cur, &NEW);
1408 141359 : if (error)
1409 0 : goto done;
1410 141359 : NEW.rm_startblock = bno;
1411 141359 : NEW.rm_owner = owner;
1412 141359 : NEW.rm_offset = offset;
1413 141359 : NEW.rm_blockcount = len;
1414 141359 : NEW.rm_flags = newext;
1415 141359 : cur->bc_rec.r = NEW;
1416 141359 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno,
1417 : len, owner, offset, newext);
1418 141359 : error = xfs_btree_insert(cur, &i);
1419 141359 : if (error)
1420 0 : goto done;
1421 141359 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1422 0 : error = -EFSCORRUPTED;
1423 0 : goto done;
1424 : }
1425 : break;
1426 :
1427 26914 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1428 : /*
1429 : * Setting the last part of a previous oldext extent to newext.
1430 : * The right neighbor is contiguous with the new allocation.
1431 : */
1432 26914 : NEW = PREV;
1433 26914 : NEW.rm_blockcount -= len;
1434 26914 : error = xfs_rmap_update(cur, &NEW);
1435 26914 : if (error)
1436 0 : goto done;
1437 26914 : error = xfs_btree_increment(cur, 0, &i);
1438 26914 : if (error)
1439 0 : goto done;
1440 26914 : NEW = RIGHT;
1441 26914 : NEW.rm_offset = offset;
1442 26914 : NEW.rm_startblock = bno;
1443 26914 : NEW.rm_blockcount += len;
1444 26914 : error = xfs_rmap_update(cur, &NEW);
1445 26914 : if (error)
1446 0 : goto done;
1447 : break;
1448 :
1449 50142 : case RMAP_RIGHT_FILLING:
1450 : /*
1451 : * Setting the last part of a previous oldext extent to newext.
1452 : * The right neighbor is not contiguous.
1453 : */
1454 50142 : NEW = PREV;
1455 50142 : NEW.rm_blockcount -= len;
1456 50142 : error = xfs_rmap_update(cur, &NEW);
1457 50142 : if (error)
1458 0 : goto done;
1459 50142 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1460 : oldext, &i);
1461 50142 : if (error)
1462 0 : goto done;
1463 50142 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1464 0 : error = -EFSCORRUPTED;
1465 0 : goto done;
1466 : }
1467 50142 : NEW.rm_startblock = bno;
1468 50142 : NEW.rm_owner = owner;
1469 50142 : NEW.rm_offset = offset;
1470 50142 : NEW.rm_blockcount = len;
1471 50142 : NEW.rm_flags = newext;
1472 50142 : cur->bc_rec.r = NEW;
1473 50142 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno,
1474 : len, owner, offset, newext);
1475 50142 : error = xfs_btree_insert(cur, &i);
1476 50142 : if (error)
1477 0 : goto done;
1478 50142 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1479 0 : error = -EFSCORRUPTED;
1480 0 : goto done;
1481 : }
1482 : break;
1483 :
1484 126125 : case 0:
1485 : /*
1486 : * Setting the middle part of a previous oldext extent to
1487 : * newext. Contiguity is impossible here.
1488 : * One extent becomes three extents.
1489 : */
1490 : /* new right extent - oldext */
1491 126125 : NEW.rm_startblock = bno + len;
1492 126125 : NEW.rm_owner = owner;
1493 126125 : NEW.rm_offset = new_endoff;
1494 126125 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1495 : new_endoff;
1496 126125 : NEW.rm_flags = PREV.rm_flags;
1497 126125 : error = xfs_rmap_update(cur, &NEW);
1498 126125 : if (error)
1499 0 : goto done;
1500 : /* new left extent - oldext */
1501 126125 : NEW = PREV;
1502 126125 : NEW.rm_blockcount = offset - PREV.rm_offset;
1503 126125 : cur->bc_rec.r = NEW;
1504 126125 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno,
1505 : NEW.rm_startblock, NEW.rm_blockcount,
1506 : NEW.rm_owner, NEW.rm_offset,
1507 : NEW.rm_flags);
1508 126125 : error = xfs_btree_insert(cur, &i);
1509 126125 : if (error)
1510 0 : goto done;
1511 126125 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1512 0 : error = -EFSCORRUPTED;
1513 0 : goto done;
1514 : }
1515 : /*
1516 : * Reset the cursor to the position of the new extent
1517 : * we are about to insert as we can't trust it after
1518 : * the previous insert.
1519 : */
1520 126125 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1521 : oldext, &i);
1522 126125 : if (error)
1523 0 : goto done;
1524 126125 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1525 0 : error = -EFSCORRUPTED;
1526 0 : goto done;
1527 : }
1528 : /* new middle extent - newext */
1529 126125 : cur->bc_rec.r.rm_flags &= ~XFS_RMAP_UNWRITTEN;
1530 126125 : cur->bc_rec.r.rm_flags |= newext;
1531 126125 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1532 : owner, offset, newext);
1533 126125 : error = xfs_btree_insert(cur, &i);
1534 126125 : if (error)
1535 0 : goto done;
1536 126125 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1537 0 : error = -EFSCORRUPTED;
1538 0 : goto done;
1539 : }
1540 : break;
1541 :
1542 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1543 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1544 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1545 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1546 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1547 : case RMAP_LEFT_CONTIG:
1548 : case RMAP_RIGHT_CONTIG:
1549 : /*
1550 : * These cases are all impossible.
1551 : */
1552 0 : ASSERT(0);
1553 : }
1554 :
1555 5423870 : trace_xfs_rmap_convert_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1556 : unwritten, oinfo);
1557 5423869 : done:
1558 5423869 : if (error)
1559 0 : trace_xfs_rmap_convert_error(cur->bc_mp,
1560 0 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
1561 5423869 : return error;
1562 : }
1563 :
1564 : /*
1565 : * Convert an unwritten extent to a real extent or vice versa. If there is no
1566 : * possibility of overlapping extents, delegate to the simpler convert
1567 : * function.
1568 : */
1569 : STATIC int
1570 10004031 : xfs_rmap_convert_shared(
1571 : struct xfs_btree_cur *cur,
1572 : xfs_agblock_t bno,
1573 : xfs_extlen_t len,
1574 : bool unwritten,
1575 : const struct xfs_owner_info *oinfo)
1576 : {
1577 10004031 : struct xfs_mount *mp = cur->bc_mp;
1578 10004031 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1579 : /* left is 0, right is 1, */
1580 : /* prev is 2, new is 3 */
1581 10004031 : uint64_t owner;
1582 10004031 : uint64_t offset;
1583 10004031 : uint64_t new_endoff;
1584 10004031 : unsigned int oldext;
1585 10004031 : unsigned int newext;
1586 10004031 : unsigned int flags = 0;
1587 10004031 : int i;
1588 10004031 : int state = 0;
1589 10004031 : int error;
1590 :
1591 10004031 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1592 10004031 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1593 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1594 10004031 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1595 10004031 : new_endoff = offset + len;
1596 10004031 : trace_xfs_rmap_convert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1597 : unwritten, oinfo);
1598 :
1599 : /*
1600 : * For the initial lookup, look for and exact match or the left-adjacent
1601 : * record for our insertion point. This will also give us the record for
1602 : * start block contiguity tests.
1603 : */
1604 10004031 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, oldext,
1605 : &PREV, &i);
1606 10004030 : if (error)
1607 2 : goto done;
1608 10004028 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1609 0 : error = -EFSCORRUPTED;
1610 0 : goto done;
1611 : }
1612 :
1613 10004028 : ASSERT(PREV.rm_offset <= offset);
1614 10004028 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1615 10004028 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1616 10004028 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1617 :
1618 : /*
1619 : * Set flags determining what part of the previous oldext allocation
1620 : * extent is being replaced by a newext allocation.
1621 : */
1622 10004028 : if (PREV.rm_offset == offset)
1623 9651017 : state |= RMAP_LEFT_FILLING;
1624 10004028 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1625 8111086 : state |= RMAP_RIGHT_FILLING;
1626 :
1627 : /* Is there a left record that abuts our range? */
1628 10004028 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, newext,
1629 : &LEFT, &i);
1630 10004028 : if (error)
1631 0 : goto done;
1632 10004028 : if (i) {
1633 253535 : state |= RMAP_LEFT_VALID;
1634 253535 : if (XFS_IS_CORRUPT(mp,
1635 : LEFT.rm_startblock + LEFT.rm_blockcount >
1636 : bno)) {
1637 0 : error = -EFSCORRUPTED;
1638 0 : goto done;
1639 : }
1640 253535 : if (xfs_rmap_is_mergeable(&LEFT, owner, newext))
1641 253533 : state |= RMAP_LEFT_CONTIG;
1642 : }
1643 :
1644 : /* Is there a right record that abuts our range? */
1645 10004028 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
1646 : newext, &i);
1647 10004029 : if (error)
1648 0 : goto done;
1649 10004029 : if (i) {
1650 147135 : state |= RMAP_RIGHT_VALID;
1651 147135 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1652 147135 : if (error)
1653 0 : goto done;
1654 147135 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1655 0 : error = -EFSCORRUPTED;
1656 0 : goto done;
1657 : }
1658 147135 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1659 0 : error = -EFSCORRUPTED;
1660 0 : goto done;
1661 : }
1662 147135 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1663 147135 : cur->bc_ag.pag->pag_agno, RIGHT.rm_startblock,
1664 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1665 : RIGHT.rm_offset, RIGHT.rm_flags);
1666 147135 : if (xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1667 147135 : state |= RMAP_RIGHT_CONTIG;
1668 : }
1669 :
1670 : /* check that left + prev + right is not too long */
1671 10004029 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1672 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1673 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1674 33097 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1675 33097 : (unsigned long)LEFT.rm_blockcount + len +
1676 33097 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1677 0 : state &= ~RMAP_RIGHT_CONTIG;
1678 :
1679 10004029 : trace_xfs_rmap_convert_state(mp, cur->bc_ag.pag->pag_agno, state,
1680 10004029 : _RET_IP_);
1681 : /*
1682 : * Switch out based on the FILLING and CONTIG state bits.
1683 : */
1684 10004029 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1685 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1686 33097 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1687 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1688 : /*
1689 : * Setting all of a previous oldext extent to newext.
1690 : * The left and right neighbors are both contiguous with new.
1691 : */
1692 33097 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1693 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1694 : RIGHT.rm_offset, RIGHT.rm_flags);
1695 33097 : if (error)
1696 0 : goto done;
1697 33097 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1698 : PREV.rm_blockcount, PREV.rm_owner,
1699 : PREV.rm_offset, PREV.rm_flags);
1700 33097 : if (error)
1701 0 : goto done;
1702 33097 : NEW = LEFT;
1703 33097 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1704 : NEW.rm_blockcount, NEW.rm_owner,
1705 : NEW.rm_offset, NEW.rm_flags, &i);
1706 33097 : if (error)
1707 0 : goto done;
1708 33097 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1709 0 : error = -EFSCORRUPTED;
1710 0 : goto done;
1711 : }
1712 33097 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1713 33097 : error = xfs_rmap_update(cur, &NEW);
1714 33097 : if (error)
1715 0 : goto done;
1716 : break;
1717 :
1718 163100 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1719 : /*
1720 : * Setting all of a previous oldext extent to newext.
1721 : * The left neighbor is contiguous, the right is not.
1722 : */
1723 163100 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1724 : PREV.rm_blockcount, PREV.rm_owner,
1725 : PREV.rm_offset, PREV.rm_flags);
1726 163100 : if (error)
1727 0 : goto done;
1728 163100 : NEW = LEFT;
1729 163100 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1730 : NEW.rm_blockcount, NEW.rm_owner,
1731 : NEW.rm_offset, NEW.rm_flags, &i);
1732 163100 : if (error)
1733 0 : goto done;
1734 163100 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1735 0 : error = -EFSCORRUPTED;
1736 0 : goto done;
1737 : }
1738 163100 : NEW.rm_blockcount += PREV.rm_blockcount;
1739 163100 : error = xfs_rmap_update(cur, &NEW);
1740 163100 : if (error)
1741 0 : goto done;
1742 : break;
1743 :
1744 95527 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1745 : /*
1746 : * Setting all of a previous oldext extent to newext.
1747 : * The right neighbor is contiguous, the left is not.
1748 : */
1749 95527 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1750 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1751 : RIGHT.rm_offset, RIGHT.rm_flags);
1752 95527 : if (error)
1753 0 : goto done;
1754 95527 : NEW = PREV;
1755 95527 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1756 : NEW.rm_blockcount, NEW.rm_owner,
1757 : NEW.rm_offset, NEW.rm_flags, &i);
1758 95527 : if (error)
1759 0 : goto done;
1760 95527 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1761 0 : error = -EFSCORRUPTED;
1762 0 : goto done;
1763 : }
1764 95527 : NEW.rm_blockcount += RIGHT.rm_blockcount;
1765 95527 : NEW.rm_flags = RIGHT.rm_flags;
1766 95527 : error = xfs_rmap_update(cur, &NEW);
1767 95527 : if (error)
1768 0 : goto done;
1769 : break;
1770 :
1771 7627688 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1772 : /*
1773 : * Setting all of a previous oldext extent to newext.
1774 : * Neither the left nor right neighbors are contiguous with
1775 : * the new one.
1776 : */
1777 7627688 : NEW = PREV;
1778 7627688 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1779 : NEW.rm_blockcount, NEW.rm_owner,
1780 : NEW.rm_offset, NEW.rm_flags, &i);
1781 7627682 : if (error)
1782 0 : goto done;
1783 7627682 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1784 0 : error = -EFSCORRUPTED;
1785 0 : goto done;
1786 : }
1787 7627682 : NEW.rm_flags = newext;
1788 7627682 : error = xfs_rmap_update(cur, &NEW);
1789 7627688 : if (error)
1790 0 : goto done;
1791 : break;
1792 :
1793 57336 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1794 : /*
1795 : * Setting the first part of a previous oldext extent to newext.
1796 : * The left neighbor is contiguous.
1797 : */
1798 57336 : NEW = PREV;
1799 57336 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1800 : NEW.rm_blockcount, NEW.rm_owner,
1801 : NEW.rm_offset, NEW.rm_flags);
1802 57336 : if (error)
1803 0 : goto done;
1804 57336 : NEW.rm_offset += len;
1805 57336 : NEW.rm_startblock += len;
1806 57336 : NEW.rm_blockcount -= len;
1807 57336 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1808 : NEW.rm_blockcount, NEW.rm_owner,
1809 : NEW.rm_offset, NEW.rm_flags);
1810 57336 : if (error)
1811 0 : goto done;
1812 57336 : NEW = LEFT;
1813 57336 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1814 : NEW.rm_blockcount, NEW.rm_owner,
1815 : NEW.rm_offset, NEW.rm_flags, &i);
1816 57336 : if (error)
1817 0 : goto done;
1818 57336 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1819 0 : error = -EFSCORRUPTED;
1820 0 : goto done;
1821 : }
1822 57336 : NEW.rm_blockcount += len;
1823 57336 : error = xfs_rmap_update(cur, &NEW);
1824 57336 : if (error)
1825 0 : goto done;
1826 : break;
1827 :
1828 1674269 : case RMAP_LEFT_FILLING:
1829 : /*
1830 : * Setting the first part of a previous oldext extent to newext.
1831 : * The left neighbor is not contiguous.
1832 : */
1833 1674269 : NEW = PREV;
1834 1674269 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1835 : NEW.rm_blockcount, NEW.rm_owner,
1836 : NEW.rm_offset, NEW.rm_flags);
1837 1674269 : if (error)
1838 0 : goto done;
1839 1674269 : NEW.rm_offset += len;
1840 1674269 : NEW.rm_startblock += len;
1841 1674269 : NEW.rm_blockcount -= len;
1842 1674269 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1843 : NEW.rm_blockcount, NEW.rm_owner,
1844 : NEW.rm_offset, NEW.rm_flags);
1845 1674269 : if (error)
1846 0 : goto done;
1847 1674269 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1848 1674269 : if (error)
1849 0 : goto done;
1850 : break;
1851 :
1852 18511 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1853 : /*
1854 : * Setting the last part of a previous oldext extent to newext.
1855 : * The right neighbor is contiguous with the new allocation.
1856 : */
1857 18511 : NEW = PREV;
1858 18511 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1859 : NEW.rm_blockcount, NEW.rm_owner,
1860 : NEW.rm_offset, NEW.rm_flags, &i);
1861 18511 : if (error)
1862 0 : goto done;
1863 18511 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1864 0 : error = -EFSCORRUPTED;
1865 0 : goto done;
1866 : }
1867 18511 : NEW.rm_blockcount = offset - NEW.rm_offset;
1868 18511 : error = xfs_rmap_update(cur, &NEW);
1869 18511 : if (error)
1870 0 : goto done;
1871 18511 : NEW = RIGHT;
1872 18511 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1873 : NEW.rm_blockcount, NEW.rm_owner,
1874 : NEW.rm_offset, NEW.rm_flags);
1875 18511 : if (error)
1876 0 : goto done;
1877 18511 : NEW.rm_offset = offset;
1878 18511 : NEW.rm_startblock = bno;
1879 18511 : NEW.rm_blockcount += len;
1880 18511 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1881 : NEW.rm_blockcount, NEW.rm_owner,
1882 : NEW.rm_offset, NEW.rm_flags);
1883 18511 : if (error)
1884 0 : goto done;
1885 : break;
1886 :
1887 173163 : case RMAP_RIGHT_FILLING:
1888 : /*
1889 : * Setting the last part of a previous oldext extent to newext.
1890 : * The right neighbor is not contiguous.
1891 : */
1892 173163 : NEW = PREV;
1893 173163 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1894 : NEW.rm_blockcount, NEW.rm_owner,
1895 : NEW.rm_offset, NEW.rm_flags, &i);
1896 173163 : if (error)
1897 0 : goto done;
1898 173163 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1899 0 : error = -EFSCORRUPTED;
1900 0 : goto done;
1901 : }
1902 173163 : NEW.rm_blockcount -= len;
1903 173163 : error = xfs_rmap_update(cur, &NEW);
1904 173163 : if (error)
1905 0 : goto done;
1906 173163 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1907 173163 : if (error)
1908 0 : goto done;
1909 : break;
1910 :
1911 161338 : case 0:
1912 : /*
1913 : * Setting the middle part of a previous oldext extent to
1914 : * newext. Contiguity is impossible here.
1915 : * One extent becomes three extents.
1916 : */
1917 : /* new right extent - oldext */
1918 161338 : NEW.rm_startblock = bno + len;
1919 161338 : NEW.rm_owner = owner;
1920 161338 : NEW.rm_offset = new_endoff;
1921 161338 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1922 : new_endoff;
1923 161338 : NEW.rm_flags = PREV.rm_flags;
1924 161338 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1925 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
1926 : NEW.rm_flags);
1927 161338 : if (error)
1928 0 : goto done;
1929 : /* new left extent - oldext */
1930 161338 : NEW = PREV;
1931 161338 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1932 : NEW.rm_blockcount, NEW.rm_owner,
1933 : NEW.rm_offset, NEW.rm_flags, &i);
1934 161338 : if (error)
1935 0 : goto done;
1936 161338 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1937 0 : error = -EFSCORRUPTED;
1938 0 : goto done;
1939 : }
1940 161338 : NEW.rm_blockcount = offset - NEW.rm_offset;
1941 161338 : error = xfs_rmap_update(cur, &NEW);
1942 161338 : if (error)
1943 0 : goto done;
1944 : /* new middle extent - newext */
1945 161338 : NEW.rm_startblock = bno;
1946 161338 : NEW.rm_blockcount = len;
1947 161338 : NEW.rm_owner = owner;
1948 161338 : NEW.rm_offset = offset;
1949 161338 : NEW.rm_flags = newext;
1950 161338 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1951 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
1952 : NEW.rm_flags);
1953 161338 : if (error)
1954 0 : goto done;
1955 : break;
1956 :
1957 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1958 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1959 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1960 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1961 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1962 : case RMAP_LEFT_CONTIG:
1963 : case RMAP_RIGHT_CONTIG:
1964 : /*
1965 : * These cases are all impossible.
1966 : */
1967 0 : ASSERT(0);
1968 : }
1969 :
1970 10004029 : trace_xfs_rmap_convert_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1971 : unwritten, oinfo);
1972 10004031 : done:
1973 10004031 : if (error)
1974 2 : trace_xfs_rmap_convert_error(cur->bc_mp,
1975 2 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
1976 10004031 : return error;
1977 : }
1978 :
1979 : #undef NEW
1980 : #undef LEFT
1981 : #undef RIGHT
1982 : #undef PREV
1983 :
1984 : /*
1985 : * Find an extent in the rmap btree and unmap it. For rmap extent types that
1986 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
1987 : * that the prev/next records in the btree might belong to another owner.
1988 : * Therefore we must use delete+insert to alter any of the key fields.
1989 : *
1990 : * For every other situation there can only be one owner for a given extent,
1991 : * so we can call the regular _free function.
1992 : */
1993 : STATIC int
1994 51608772 : xfs_rmap_unmap_shared(
1995 : struct xfs_btree_cur *cur,
1996 : xfs_agblock_t bno,
1997 : xfs_extlen_t len,
1998 : bool unwritten,
1999 : const struct xfs_owner_info *oinfo)
2000 : {
2001 51608772 : struct xfs_mount *mp = cur->bc_mp;
2002 51608772 : struct xfs_rmap_irec ltrec;
2003 51608772 : uint64_t ltoff;
2004 51608772 : int error = 0;
2005 51608772 : int i;
2006 51608772 : uint64_t owner;
2007 51608772 : uint64_t offset;
2008 51608772 : unsigned int flags;
2009 :
2010 51608772 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2011 51608772 : if (unwritten)
2012 5270973 : flags |= XFS_RMAP_UNWRITTEN;
2013 51608772 : trace_xfs_rmap_unmap(mp, cur->bc_ag.pag->pag_agno, bno, len,
2014 : unwritten, oinfo);
2015 :
2016 : /*
2017 : * We should always have a left record because there's a static record
2018 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
2019 : * will not ever be removed from the tree.
2020 : */
2021 51608790 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, flags,
2022 : <rec, &i);
2023 51608755 : if (error)
2024 302 : goto out_error;
2025 51608453 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2026 0 : error = -EFSCORRUPTED;
2027 0 : goto out_error;
2028 : }
2029 51608453 : ltoff = ltrec.rm_offset;
2030 :
2031 : /* Make sure the extent we found covers the entire freeing range. */
2032 51608453 : if (XFS_IS_CORRUPT(mp,
2033 : ltrec.rm_startblock > bno ||
2034 : ltrec.rm_startblock + ltrec.rm_blockcount <
2035 : bno + len)) {
2036 0 : error = -EFSCORRUPTED;
2037 0 : goto out_error;
2038 : }
2039 :
2040 : /* Make sure the owner matches what we expect to find in the tree. */
2041 51608453 : if (XFS_IS_CORRUPT(mp, owner != ltrec.rm_owner)) {
2042 0 : error = -EFSCORRUPTED;
2043 0 : goto out_error;
2044 : }
2045 :
2046 : /* Make sure the unwritten flag matches. */
2047 51608453 : if (XFS_IS_CORRUPT(mp,
2048 : (flags & XFS_RMAP_UNWRITTEN) !=
2049 : (ltrec.rm_flags & XFS_RMAP_UNWRITTEN))) {
2050 0 : error = -EFSCORRUPTED;
2051 0 : goto out_error;
2052 : }
2053 :
2054 : /* Check the offset. */
2055 51608453 : if (XFS_IS_CORRUPT(mp, ltrec.rm_offset > offset)) {
2056 0 : error = -EFSCORRUPTED;
2057 0 : goto out_error;
2058 : }
2059 51608453 : if (XFS_IS_CORRUPT(mp, offset > ltoff + ltrec.rm_blockcount)) {
2060 0 : error = -EFSCORRUPTED;
2061 0 : goto out_error;
2062 : }
2063 :
2064 51608453 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
2065 : /* Exact match, simply remove the record from rmap tree. */
2066 45260205 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2067 : ltrec.rm_blockcount, ltrec.rm_owner,
2068 : ltrec.rm_offset, ltrec.rm_flags);
2069 45260230 : if (error)
2070 1 : goto out_error;
2071 6348248 : } else if (ltrec.rm_startblock == bno) {
2072 : /*
2073 : * Overlap left hand side of extent: move the start, trim the
2074 : * length and update the current record.
2075 : *
2076 : * ltbno ltlen
2077 : * Orig: |oooooooooooooooooooo|
2078 : * Freeing: |fffffffff|
2079 : * Result: |rrrrrrrrrr|
2080 : * bno len
2081 : */
2082 :
2083 : /* Delete prev rmap. */
2084 1522720 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2085 : ltrec.rm_blockcount, ltrec.rm_owner,
2086 : ltrec.rm_offset, ltrec.rm_flags);
2087 1522720 : if (error)
2088 0 : goto out_error;
2089 :
2090 : /* Add an rmap at the new offset. */
2091 1522720 : ltrec.rm_startblock += len;
2092 1522720 : ltrec.rm_blockcount -= len;
2093 1522720 : ltrec.rm_offset += len;
2094 1522720 : error = xfs_rmap_insert(cur, ltrec.rm_startblock,
2095 : ltrec.rm_blockcount, ltrec.rm_owner,
2096 : ltrec.rm_offset, ltrec.rm_flags);
2097 1522720 : if (error)
2098 0 : goto out_error;
2099 4825528 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
2100 : /*
2101 : * Overlap right hand side of extent: trim the length and
2102 : * update the current record.
2103 : *
2104 : * ltbno ltlen
2105 : * Orig: |oooooooooooooooooooo|
2106 : * Freeing: |fffffffff|
2107 : * Result: |rrrrrrrrrr|
2108 : * bno len
2109 : */
2110 1126205 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2111 : ltrec.rm_blockcount, ltrec.rm_owner,
2112 : ltrec.rm_offset, ltrec.rm_flags, &i);
2113 1126205 : if (error)
2114 0 : goto out_error;
2115 1126205 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2116 0 : error = -EFSCORRUPTED;
2117 0 : goto out_error;
2118 : }
2119 1126205 : ltrec.rm_blockcount -= len;
2120 1126205 : error = xfs_rmap_update(cur, <rec);
2121 1126205 : if (error)
2122 0 : goto out_error;
2123 : } else {
2124 : /*
2125 : * Overlap middle of extent: trim the length of the existing
2126 : * record to the length of the new left-extent size, increment
2127 : * the insertion position so we can insert a new record
2128 : * containing the remaining right-extent space.
2129 : *
2130 : * ltbno ltlen
2131 : * Orig: |oooooooooooooooooooo|
2132 : * Freeing: |fffffffff|
2133 : * Result: |rrrrr| |rrrr|
2134 : * bno len
2135 : */
2136 3699323 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
2137 :
2138 : /* Shrink the left side of the rmap */
2139 3699323 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2140 : ltrec.rm_blockcount, ltrec.rm_owner,
2141 : ltrec.rm_offset, ltrec.rm_flags, &i);
2142 3699319 : if (error)
2143 0 : goto out_error;
2144 3699319 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2145 0 : error = -EFSCORRUPTED;
2146 0 : goto out_error;
2147 : }
2148 3699319 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
2149 3699319 : error = xfs_rmap_update(cur, <rec);
2150 3699322 : if (error)
2151 0 : goto out_error;
2152 :
2153 : /* Add an rmap at the new offset */
2154 3699322 : error = xfs_rmap_insert(cur, bno + len,
2155 3699322 : orig_len - len - ltrec.rm_blockcount,
2156 : ltrec.rm_owner, offset + len,
2157 : ltrec.rm_flags);
2158 3699323 : if (error)
2159 0 : goto out_error;
2160 : }
2161 :
2162 51608477 : trace_xfs_rmap_unmap_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2163 : unwritten, oinfo);
2164 51608749 : out_error:
2165 51608749 : if (error)
2166 303 : trace_xfs_rmap_unmap_error(cur->bc_mp,
2167 303 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2168 51608749 : return error;
2169 : }
2170 :
2171 : /*
2172 : * Find an extent in the rmap btree and map it. For rmap extent types that
2173 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2174 : * that the prev/next records in the btree might belong to another owner.
2175 : * Therefore we must use delete+insert to alter any of the key fields.
2176 : *
2177 : * For every other situation there can only be one owner for a given extent,
2178 : * so we can call the regular _alloc function.
2179 : */
2180 : STATIC int
2181 63521379 : xfs_rmap_map_shared(
2182 : struct xfs_btree_cur *cur,
2183 : xfs_agblock_t bno,
2184 : xfs_extlen_t len,
2185 : bool unwritten,
2186 : const struct xfs_owner_info *oinfo)
2187 : {
2188 63521379 : struct xfs_mount *mp = cur->bc_mp;
2189 63521379 : struct xfs_rmap_irec ltrec;
2190 63521379 : struct xfs_rmap_irec gtrec;
2191 63521379 : int have_gt;
2192 63521379 : int have_lt;
2193 63521379 : int error = 0;
2194 63521379 : int i;
2195 63521379 : uint64_t owner;
2196 63521379 : uint64_t offset;
2197 63521379 : unsigned int flags = 0;
2198 :
2199 63521379 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2200 63521379 : if (unwritten)
2201 12204218 : flags |= XFS_RMAP_UNWRITTEN;
2202 63521379 : trace_xfs_rmap_map(mp, cur->bc_ag.pag->pag_agno, bno, len,
2203 : unwritten, oinfo);
2204 :
2205 : /* Is there a left record that abuts our range? */
2206 63521401 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, flags,
2207 : <rec, &have_lt);
2208 63521394 : if (error)
2209 168 : goto out_error;
2210 63521226 : if (have_lt &&
2211 9342713 : !xfs_rmap_is_mergeable(<rec, owner, flags))
2212 310621 : have_lt = 0;
2213 :
2214 : /* Is there a right record that abuts our range? */
2215 63521226 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
2216 : flags, &have_gt);
2217 63521171 : if (error)
2218 0 : goto out_error;
2219 63521171 : if (have_gt) {
2220 654089 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
2221 654089 : if (error)
2222 0 : goto out_error;
2223 654089 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
2224 0 : error = -EFSCORRUPTED;
2225 0 : goto out_error;
2226 : }
2227 654089 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
2228 654089 : cur->bc_ag.pag->pag_agno, gtrec.rm_startblock,
2229 : gtrec.rm_blockcount, gtrec.rm_owner,
2230 : gtrec.rm_offset, gtrec.rm_flags);
2231 :
2232 654089 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
2233 124912 : have_gt = 0;
2234 : }
2235 :
2236 63521171 : if (have_lt &&
2237 9032092 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
2238 1311529 : ltrec.rm_offset + ltrec.rm_blockcount == offset) {
2239 : /*
2240 : * Left edge contiguous, merge into left record.
2241 : *
2242 : * ltbno ltlen
2243 : * orig: |ooooooooo|
2244 : * adding: |aaaaaaaaa|
2245 : * result: |rrrrrrrrrrrrrrrrrrr|
2246 : * bno len
2247 : */
2248 1311529 : ltrec.rm_blockcount += len;
2249 1311529 : if (have_gt &&
2250 338454 : bno + len == gtrec.rm_startblock &&
2251 338454 : offset + len == gtrec.rm_offset) {
2252 : /*
2253 : * Right edge also contiguous, delete right record
2254 : * and merge into left record.
2255 : *
2256 : * ltbno ltlen gtbno gtlen
2257 : * orig: |ooooooooo| |ooooooooo|
2258 : * adding: |aaaaaaaaa|
2259 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
2260 : */
2261 338454 : ltrec.rm_blockcount += gtrec.rm_blockcount;
2262 338454 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2263 : gtrec.rm_blockcount, gtrec.rm_owner,
2264 : gtrec.rm_offset, gtrec.rm_flags);
2265 338454 : if (error)
2266 0 : goto out_error;
2267 : }
2268 :
2269 : /* Point the cursor back to the left record and update. */
2270 1311529 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2271 : ltrec.rm_blockcount, ltrec.rm_owner,
2272 : ltrec.rm_offset, ltrec.rm_flags, &i);
2273 1311529 : if (error)
2274 0 : goto out_error;
2275 1311529 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2276 0 : error = -EFSCORRUPTED;
2277 0 : goto out_error;
2278 : }
2279 :
2280 1311529 : error = xfs_rmap_update(cur, <rec);
2281 1311529 : if (error)
2282 0 : goto out_error;
2283 62209642 : } else if (have_gt &&
2284 190723 : bno + len == gtrec.rm_startblock &&
2285 190723 : offset + len == gtrec.rm_offset) {
2286 : /*
2287 : * Right edge contiguous, merge into right record.
2288 : *
2289 : * gtbno gtlen
2290 : * Orig: |ooooooooo|
2291 : * adding: |aaaaaaaaa|
2292 : * Result: |rrrrrrrrrrrrrrrrrrr|
2293 : * bno len
2294 : */
2295 : /* Delete the old record. */
2296 190723 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2297 : gtrec.rm_blockcount, gtrec.rm_owner,
2298 : gtrec.rm_offset, gtrec.rm_flags);
2299 190723 : if (error)
2300 0 : goto out_error;
2301 :
2302 : /* Move the start and re-add it. */
2303 190723 : gtrec.rm_startblock = bno;
2304 190723 : gtrec.rm_blockcount += len;
2305 190723 : gtrec.rm_offset = offset;
2306 190723 : error = xfs_rmap_insert(cur, gtrec.rm_startblock,
2307 : gtrec.rm_blockcount, gtrec.rm_owner,
2308 : gtrec.rm_offset, gtrec.rm_flags);
2309 190723 : if (error)
2310 0 : goto out_error;
2311 : } else {
2312 : /*
2313 : * No contiguous edge with identical owner, insert
2314 : * new record at current cursor position.
2315 : */
2316 62018919 : error = xfs_rmap_insert(cur, bno, len, owner, offset, flags);
2317 62018950 : if (error)
2318 162 : goto out_error;
2319 : }
2320 :
2321 63521040 : trace_xfs_rmap_map_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2322 : unwritten, oinfo);
2323 63521386 : out_error:
2324 63521386 : if (error)
2325 330 : trace_xfs_rmap_map_error(cur->bc_mp,
2326 330 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2327 63521386 : return error;
2328 : }
2329 :
2330 : /* Insert a raw rmap into the rmapbt. */
2331 : int
2332 0 : xfs_rmap_map_raw(
2333 : struct xfs_btree_cur *cur,
2334 : struct xfs_rmap_irec *rmap)
2335 : {
2336 0 : struct xfs_owner_info oinfo;
2337 :
2338 0 : oinfo.oi_owner = rmap->rm_owner;
2339 0 : oinfo.oi_offset = rmap->rm_offset;
2340 0 : oinfo.oi_flags = 0;
2341 0 : if (rmap->rm_flags & XFS_RMAP_ATTR_FORK)
2342 0 : oinfo.oi_flags |= XFS_OWNER_INFO_ATTR_FORK;
2343 0 : if (rmap->rm_flags & XFS_RMAP_BMBT_BLOCK)
2344 0 : oinfo.oi_flags |= XFS_OWNER_INFO_BMBT_BLOCK;
2345 :
2346 0 : if (rmap->rm_flags || XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2347 0 : return xfs_rmap_map(cur, rmap->rm_startblock,
2348 : rmap->rm_blockcount,
2349 0 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2350 : &oinfo);
2351 :
2352 0 : return xfs_rmap_map_shared(cur, rmap->rm_startblock,
2353 : rmap->rm_blockcount,
2354 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2355 : &oinfo);
2356 : }
2357 :
2358 : struct xfs_rmap_query_range_info {
2359 : xfs_rmap_query_range_fn fn;
2360 : void *priv;
2361 : };
2362 :
2363 : /* Format btree record and pass to our callback. */
2364 : STATIC int
2365 50505533397 : xfs_rmap_query_range_helper(
2366 : struct xfs_btree_cur *cur,
2367 : const union xfs_btree_rec *rec,
2368 : void *priv)
2369 : {
2370 50505533397 : struct xfs_rmap_query_range_info *query = priv;
2371 50505533397 : struct xfs_rmap_irec irec;
2372 50505533397 : xfs_failaddr_t fa;
2373 :
2374 50505533397 : fa = xfs_rmap_btrec_to_irec(rec, &irec);
2375 51101250378 : if (!fa)
2376 50777672480 : fa = xfs_rmap_check_irec(cur, &irec);
2377 50839973003 : if (fa)
2378 0 : return xfs_rmap_complain_bad_rec(cur, fa, &irec);
2379 :
2380 50839973003 : return query->fn(cur, &irec, query->priv);
2381 : }
2382 :
2383 : /* Find all rmaps between two keys. */
2384 : int
2385 503127603 : xfs_rmap_query_range(
2386 : struct xfs_btree_cur *cur,
2387 : const struct xfs_rmap_irec *low_rec,
2388 : const struct xfs_rmap_irec *high_rec,
2389 : xfs_rmap_query_range_fn fn,
2390 : void *priv)
2391 : {
2392 503127603 : union xfs_btree_irec low_brec = { .r = *low_rec };
2393 503127603 : union xfs_btree_irec high_brec = { .r = *high_rec };
2394 503127603 : struct xfs_rmap_query_range_info query = { .priv = priv, .fn = fn };
2395 :
2396 503127603 : return xfs_btree_query_range(cur, &low_brec, &high_brec,
2397 : xfs_rmap_query_range_helper, &query);
2398 : }
2399 :
2400 : /* Find all rmaps. */
2401 : int
2402 4212110 : xfs_rmap_query_all(
2403 : struct xfs_btree_cur *cur,
2404 : xfs_rmap_query_range_fn fn,
2405 : void *priv)
2406 : {
2407 4212110 : struct xfs_rmap_query_range_info query;
2408 :
2409 4212110 : query.priv = priv;
2410 4212110 : query.fn = fn;
2411 4212110 : return xfs_btree_query_all(cur, xfs_rmap_query_range_helper, &query);
2412 : }
2413 :
2414 : /* Clean up after calling xfs_rmap_finish_one. */
2415 : void
2416 189057938 : xfs_rmap_finish_one_cleanup(
2417 : struct xfs_trans *tp,
2418 : struct xfs_btree_cur *rcur,
2419 : int error)
2420 : {
2421 189059467 : struct xfs_buf *agbp;
2422 :
2423 189057938 : if (rcur == NULL)
2424 : return;
2425 189058595 : agbp = rcur->bc_ag.agbp;
2426 189057066 : xfs_btree_del_cursor(rcur, error);
2427 189058938 : if (error)
2428 1144 : xfs_trans_brelse(tp, agbp);
2429 : }
2430 :
2431 : /*
2432 : * Process one of the deferred rmap operations. We pass back the
2433 : * btree cursor to maintain our lock on the rmapbt between calls.
2434 : * This saves time and eliminates a buffer deadlock between the
2435 : * superblock and the AGF because we'll always grab them in the same
2436 : * order.
2437 : */
2438 : int
2439 208663274 : xfs_rmap_finish_one(
2440 : struct xfs_trans *tp,
2441 : struct xfs_rmap_intent *ri,
2442 : struct xfs_btree_cur **pcur)
2443 : {
2444 208663274 : struct xfs_mount *mp = tp->t_mountp;
2445 208663274 : struct xfs_btree_cur *rcur;
2446 208663274 : struct xfs_buf *agbp = NULL;
2447 208663274 : int error = 0;
2448 208663274 : struct xfs_owner_info oinfo;
2449 208663274 : xfs_agblock_t bno;
2450 208663274 : bool unwritten;
2451 :
2452 208663274 : bno = XFS_FSB_TO_AGBNO(mp, ri->ri_bmap.br_startblock);
2453 :
2454 208664428 : trace_xfs_rmap_deferred(mp, ri->ri_pag->pag_agno, ri->ri_type, bno,
2455 : ri->ri_owner, ri->ri_whichfork,
2456 : ri->ri_bmap.br_startoff, ri->ri_bmap.br_blockcount,
2457 : ri->ri_bmap.br_state);
2458 :
2459 208664947 : if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_RMAP_FINISH_ONE))
2460 : return -EIO;
2461 :
2462 : /*
2463 : * If we haven't gotten a cursor or the cursor AG doesn't match
2464 : * the startblock, get one now.
2465 : */
2466 208663810 : rcur = *pcur;
2467 208663810 : if (rcur != NULL && rcur->bc_ag.pag != ri->ri_pag) {
2468 1529 : xfs_rmap_finish_one_cleanup(tp, rcur, 0);
2469 1529 : rcur = NULL;
2470 1529 : *pcur = NULL;
2471 : }
2472 208663810 : if (rcur == NULL) {
2473 : /*
2474 : * Refresh the freelist before we start changing the
2475 : * rmapbt, because a shape change could cause us to
2476 : * allocate blocks.
2477 : */
2478 189060596 : error = xfs_free_extent_fix_freelist(tp, ri->ri_pag, &agbp);
2479 189061607 : if (error)
2480 : return error;
2481 189060741 : if (XFS_IS_CORRUPT(tp->t_mountp, !agbp))
2482 0 : return -EFSCORRUPTED;
2483 :
2484 189060741 : rcur = xfs_rmapbt_init_cursor(mp, tp, agbp, ri->ri_pag);
2485 : }
2486 208663877 : *pcur = rcur;
2487 :
2488 208663877 : xfs_rmap_ino_owner(&oinfo, ri->ri_owner, ri->ri_whichfork,
2489 : ri->ri_bmap.br_startoff);
2490 208663877 : unwritten = ri->ri_bmap.br_state == XFS_EXT_UNWRITTEN;
2491 208663877 : bno = XFS_FSB_TO_AGBNO(rcur->bc_mp, ri->ri_bmap.br_startblock);
2492 :
2493 208664015 : switch (ri->ri_type) {
2494 40693200 : case XFS_RMAP_ALLOC:
2495 : case XFS_RMAP_MAP:
2496 40693200 : error = xfs_rmap_map(rcur, bno, ri->ri_bmap.br_blockcount,
2497 : unwritten, &oinfo);
2498 40693200 : break;
2499 63521377 : case XFS_RMAP_MAP_SHARED:
2500 63521377 : error = xfs_rmap_map_shared(rcur, bno,
2501 63521377 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2502 63521377 : break;
2503 37412750 : case XFS_RMAP_FREE:
2504 : case XFS_RMAP_UNMAP:
2505 37412750 : error = xfs_rmap_unmap(rcur, bno, ri->ri_bmap.br_blockcount,
2506 : unwritten, &oinfo);
2507 37412750 : break;
2508 51608785 : case XFS_RMAP_UNMAP_SHARED:
2509 51608785 : error = xfs_rmap_unmap_shared(rcur, bno,
2510 51608785 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2511 51608785 : break;
2512 5423873 : case XFS_RMAP_CONVERT:
2513 5423873 : error = xfs_rmap_convert(rcur, bno, ri->ri_bmap.br_blockcount,
2514 5423873 : !unwritten, &oinfo);
2515 5423873 : break;
2516 10004030 : case XFS_RMAP_CONVERT_SHARED:
2517 10004030 : error = xfs_rmap_convert_shared(rcur, bno,
2518 10004030 : ri->ri_bmap.br_blockcount, !unwritten, &oinfo);
2519 10004030 : break;
2520 0 : default:
2521 0 : ASSERT(0);
2522 0 : error = -EFSCORRUPTED;
2523 : }
2524 :
2525 : return error;
2526 : }
2527 :
2528 : /*
2529 : * Don't defer an rmap if we aren't an rmap filesystem.
2530 : */
2531 : static bool
2532 : xfs_rmap_update_is_needed(
2533 : struct xfs_mount *mp,
2534 : int whichfork)
2535 : {
2536 208067309 : return xfs_has_rmapbt(mp) && whichfork != XFS_COW_FORK;
2537 : }
2538 :
2539 : /*
2540 : * Record a rmap intent; the list is kept sorted first by AG and then by
2541 : * increasing age.
2542 : */
2543 : static void
2544 208657980 : __xfs_rmap_add(
2545 : struct xfs_trans *tp,
2546 : enum xfs_rmap_intent_type type,
2547 : uint64_t owner,
2548 : int whichfork,
2549 : struct xfs_bmbt_irec *bmap)
2550 : {
2551 208657980 : struct xfs_rmap_intent *ri;
2552 :
2553 208658914 : trace_xfs_rmap_defer(tp->t_mountp,
2554 208657980 : XFS_FSB_TO_AGNO(tp->t_mountp, bmap->br_startblock),
2555 : type,
2556 208657980 : XFS_FSB_TO_AGBNO(tp->t_mountp, bmap->br_startblock),
2557 : owner, whichfork,
2558 : bmap->br_startoff,
2559 : bmap->br_blockcount,
2560 : bmap->br_state);
2561 :
2562 208657719 : ri = kmem_cache_alloc(xfs_rmap_intent_cache, GFP_NOFS | __GFP_NOFAIL);
2563 208657054 : INIT_LIST_HEAD(&ri->ri_list);
2564 208657054 : ri->ri_type = type;
2565 208657054 : ri->ri_owner = owner;
2566 208657054 : ri->ri_whichfork = whichfork;
2567 208657054 : ri->ri_bmap = *bmap;
2568 :
2569 208657054 : xfs_rmap_update_get_group(tp->t_mountp, ri);
2570 208655622 : xfs_defer_add(tp, XFS_DEFER_OPS_TYPE_RMAP, &ri->ri_list);
2571 208658875 : }
2572 :
2573 : /* Map an extent into a file. */
2574 : void
2575 121141591 : xfs_rmap_map_extent(
2576 : struct xfs_trans *tp,
2577 : struct xfs_inode *ip,
2578 : int whichfork,
2579 : struct xfs_bmbt_irec *PREV)
2580 : {
2581 121141591 : enum xfs_rmap_intent_type type = XFS_RMAP_MAP;
2582 :
2583 242283182 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2584 : return;
2585 :
2586 103888696 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2587 63521171 : type = XFS_RMAP_MAP_SHARED;
2588 :
2589 103888696 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2590 : }
2591 :
2592 : /* Unmap an extent out of a file. */
2593 : void
2594 99285933 : xfs_rmap_unmap_extent(
2595 : struct xfs_trans *tp,
2596 : struct xfs_inode *ip,
2597 : int whichfork,
2598 : struct xfs_bmbt_irec *PREV)
2599 : {
2600 99285933 : enum xfs_rmap_intent_type type = XFS_RMAP_UNMAP;
2601 :
2602 198571866 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2603 : return;
2604 :
2605 86731823 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2606 51608672 : type = XFS_RMAP_UNMAP_SHARED;
2607 :
2608 86731823 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2609 : }
2610 :
2611 : /*
2612 : * Convert a data fork extent from unwritten to real or vice versa.
2613 : *
2614 : * Note that tp can be NULL here as no transaction is used for COW fork
2615 : * unwritten conversion.
2616 : */
2617 : void
2618 22556467 : xfs_rmap_convert_extent(
2619 : struct xfs_mount *mp,
2620 : struct xfs_trans *tp,
2621 : struct xfs_inode *ip,
2622 : int whichfork,
2623 : struct xfs_bmbt_irec *PREV)
2624 : {
2625 22556467 : enum xfs_rmap_intent_type type = XFS_RMAP_CONVERT;
2626 :
2627 45112934 : if (!xfs_rmap_update_is_needed(mp, whichfork))
2628 : return;
2629 :
2630 15427980 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2631 10004253 : type = XFS_RMAP_CONVERT_SHARED;
2632 :
2633 15427980 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2634 : }
2635 :
2636 : /* Schedule the creation of an rmap for non-file data. */
2637 : void
2638 324783 : xfs_rmap_alloc_extent(
2639 : struct xfs_trans *tp,
2640 : xfs_agnumber_t agno,
2641 : xfs_agblock_t bno,
2642 : xfs_extlen_t len,
2643 : uint64_t owner)
2644 : {
2645 324783 : struct xfs_bmbt_irec bmap;
2646 :
2647 324783 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2648 0 : return;
2649 :
2650 324783 : bmap.br_startblock = XFS_AGB_TO_FSB(tp->t_mountp, agno, bno);
2651 324783 : bmap.br_blockcount = len;
2652 324783 : bmap.br_startoff = 0;
2653 324783 : bmap.br_state = XFS_EXT_NORM;
2654 :
2655 324783 : __xfs_rmap_add(tp, XFS_RMAP_ALLOC, owner, XFS_DATA_FORK, &bmap);
2656 : }
2657 :
2658 : /* Schedule the deletion of an rmap for non-file data. */
2659 : void
2660 2288221 : xfs_rmap_free_extent(
2661 : struct xfs_trans *tp,
2662 : xfs_agnumber_t agno,
2663 : xfs_agblock_t bno,
2664 : xfs_extlen_t len,
2665 : uint64_t owner)
2666 : {
2667 2288221 : struct xfs_bmbt_irec bmap;
2668 :
2669 2288221 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2670 0 : return;
2671 :
2672 2288221 : bmap.br_startblock = XFS_AGB_TO_FSB(tp->t_mountp, agno, bno);
2673 2288221 : bmap.br_blockcount = len;
2674 2288221 : bmap.br_startoff = 0;
2675 2288221 : bmap.br_state = XFS_EXT_NORM;
2676 :
2677 2288221 : __xfs_rmap_add(tp, XFS_RMAP_FREE, owner, XFS_DATA_FORK, &bmap);
2678 : }
2679 :
2680 : /* Compare rmap records. Returns -1 if a < b, 1 if a > b, and 0 if equal. */
2681 : int
2682 53847067 : xfs_rmap_compare(
2683 : const struct xfs_rmap_irec *a,
2684 : const struct xfs_rmap_irec *b)
2685 : {
2686 53847067 : __u64 oa;
2687 53847067 : __u64 ob;
2688 :
2689 53847067 : oa = xfs_rmap_irec_offset_pack(a);
2690 53847067 : ob = xfs_rmap_irec_offset_pack(b);
2691 :
2692 53847067 : if (a->rm_startblock < b->rm_startblock)
2693 : return -1;
2694 53844075 : else if (a->rm_startblock > b->rm_startblock)
2695 : return 1;
2696 7860930 : else if (a->rm_owner < b->rm_owner)
2697 : return -1;
2698 7763367 : else if (a->rm_owner > b->rm_owner)
2699 : return 1;
2700 7658280 : else if (oa < ob)
2701 : return -1;
2702 7602139 : else if (oa > ob)
2703 : return 1;
2704 : else
2705 3711 : return 0;
2706 : }
2707 :
2708 : /*
2709 : * Scan the physical storage part of the keyspace of the reverse mapping index
2710 : * and tell us if the area has no records, is fully mapped by records, or is
2711 : * partially filled.
2712 : */
2713 : int
2714 241369633 : xfs_rmap_has_records(
2715 : struct xfs_btree_cur *cur,
2716 : xfs_agblock_t bno,
2717 : xfs_extlen_t len,
2718 : enum xbtree_recpacking *outcome)
2719 : {
2720 241369633 : union xfs_btree_key mask = {
2721 : .rmap.rm_startblock = cpu_to_be32(-1U),
2722 : };
2723 241369633 : union xfs_btree_irec low;
2724 241369633 : union xfs_btree_irec high;
2725 :
2726 241369633 : memset(&low, 0, sizeof(low));
2727 241369633 : low.r.rm_startblock = bno;
2728 241369633 : memset(&high, 0xFF, sizeof(high));
2729 241369633 : high.r.rm_startblock = bno + len - 1;
2730 :
2731 241369633 : return xfs_btree_has_records(cur, &low, &high, &mask, outcome);
2732 : }
2733 :
2734 : struct xfs_rmap_ownercount {
2735 : /* Owner that we're looking for. */
2736 : struct xfs_rmap_irec good;
2737 :
2738 : /* rmap search keys */
2739 : struct xfs_rmap_irec low;
2740 : struct xfs_rmap_irec high;
2741 :
2742 : struct xfs_rmap_matches *results;
2743 :
2744 : /* Stop early if we find a nonmatch? */
2745 : bool stop_on_nonmatch;
2746 : };
2747 :
2748 : /* Does this rmap represent space that can have multiple owners? */
2749 : static inline bool
2750 : xfs_rmap_shareable(
2751 : struct xfs_mount *mp,
2752 : const struct xfs_rmap_irec *rmap)
2753 : {
2754 25509067 : if (!xfs_has_reflink(mp))
2755 : return false;
2756 51018137 : if (XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2757 : return false;
2758 25236840 : if (rmap->rm_flags & (XFS_RMAP_ATTR_FORK |
2759 : XFS_RMAP_BMBT_BLOCK))
2760 41303 : return false;
2761 : return true;
2762 : }
2763 :
2764 : static inline void
2765 419136333 : xfs_rmap_ownercount_init(
2766 : struct xfs_rmap_ownercount *roc,
2767 : xfs_agblock_t bno,
2768 : xfs_extlen_t len,
2769 : const struct xfs_owner_info *oinfo,
2770 : struct xfs_rmap_matches *results)
2771 : {
2772 419136333 : memset(roc, 0, sizeof(*roc));
2773 419136333 : roc->results = results;
2774 :
2775 419136333 : roc->low.rm_startblock = bno;
2776 419136333 : memset(&roc->high, 0xFF, sizeof(roc->high));
2777 419136333 : roc->high.rm_startblock = bno + len - 1;
2778 :
2779 419136333 : memset(results, 0, sizeof(*results));
2780 419136333 : roc->good.rm_startblock = bno;
2781 419136333 : roc->good.rm_blockcount = len;
2782 419136333 : roc->good.rm_owner = oinfo->oi_owner;
2783 419136333 : roc->good.rm_offset = oinfo->oi_offset;
2784 419136333 : if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK)
2785 449391 : roc->good.rm_flags |= XFS_RMAP_ATTR_FORK;
2786 419136333 : if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK)
2787 938891 : roc->good.rm_flags |= XFS_RMAP_BMBT_BLOCK;
2788 419136333 : }
2789 :
2790 : /* Figure out if this is a match for the owner. */
2791 : STATIC int
2792 435588317 : xfs_rmap_count_owners_helper(
2793 : struct xfs_btree_cur *cur,
2794 : const struct xfs_rmap_irec *rec,
2795 : void *priv)
2796 : {
2797 435588317 : struct xfs_rmap_ownercount *roc = priv;
2798 435588317 : struct xfs_rmap_irec check = *rec;
2799 435588317 : unsigned int keyflags;
2800 435588317 : bool filedata;
2801 435588317 : int64_t delta;
2802 :
2803 435588317 : filedata = !XFS_RMAP_NON_INODE_OWNER(check.rm_owner) &&
2804 148235221 : !(check.rm_flags & XFS_RMAP_BMBT_BLOCK);
2805 :
2806 : /* Trim the part of check that comes before the comparison range. */
2807 435588317 : delta = (int64_t)roc->good.rm_startblock - check.rm_startblock;
2808 435588317 : if (delta > 0) {
2809 230646815 : check.rm_startblock += delta;
2810 230646815 : check.rm_blockcount -= delta;
2811 230646815 : if (filedata)
2812 85768 : check.rm_offset += delta;
2813 : }
2814 :
2815 : /* Trim the part of check that comes after the comparison range. */
2816 435588317 : delta = (check.rm_startblock + check.rm_blockcount) -
2817 435588317 : (roc->good.rm_startblock + roc->good.rm_blockcount);
2818 435588317 : if (delta > 0)
2819 252580056 : check.rm_blockcount -= delta;
2820 :
2821 : /* Don't care about unwritten status for establishing ownership. */
2822 435588317 : keyflags = check.rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK);
2823 :
2824 435588317 : if (check.rm_startblock == roc->good.rm_startblock &&
2825 417866412 : check.rm_blockcount == roc->good.rm_blockcount &&
2826 417866412 : check.rm_owner == roc->good.rm_owner &&
2827 410081609 : check.rm_offset == roc->good.rm_offset &&
2828 410081722 : keyflags == roc->good.rm_flags) {
2829 410079250 : roc->results->matches++;
2830 : } else {
2831 25509067 : roc->results->non_owner_matches++;
2832 51018134 : if (xfs_rmap_shareable(cur->bc_mp, &roc->good) ^
2833 : xfs_rmap_shareable(cur->bc_mp, &check))
2834 25195532 : roc->results->bad_non_owner_matches++;
2835 : }
2836 :
2837 435588317 : if (roc->results->non_owner_matches && roc->stop_on_nonmatch)
2838 0 : return -ECANCELED;
2839 :
2840 : return 0;
2841 : }
2842 :
2843 : /* Count the number of owners and non-owners of this range of blocks. */
2844 : int
2845 417831666 : xfs_rmap_count_owners(
2846 : struct xfs_btree_cur *cur,
2847 : xfs_agblock_t bno,
2848 : xfs_extlen_t len,
2849 : const struct xfs_owner_info *oinfo,
2850 : struct xfs_rmap_matches *results)
2851 : {
2852 417831666 : struct xfs_rmap_ownercount roc;
2853 417831666 : int error;
2854 :
2855 417831666 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, results);
2856 417837396 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
2857 : xfs_rmap_count_owners_helper, &roc);
2858 417838937 : if (error)
2859 : return error;
2860 :
2861 : /*
2862 : * There can't be any non-owner rmaps that conflict with the given
2863 : * owner if we didn't find any rmaps matching the owner.
2864 : */
2865 417838937 : if (!results->matches)
2866 9060636 : results->bad_non_owner_matches = 0;
2867 :
2868 : return 0;
2869 : }
2870 :
2871 : /*
2872 : * Given an extent and some owner info, can we find records overlapping
2873 : * the extent whose owner info does not match the given owner?
2874 : */
2875 : int
2876 1303302 : xfs_rmap_has_other_keys(
2877 : struct xfs_btree_cur *cur,
2878 : xfs_agblock_t bno,
2879 : xfs_extlen_t len,
2880 : const struct xfs_owner_info *oinfo,
2881 : bool *has_other)
2882 : {
2883 1303302 : struct xfs_rmap_matches res;
2884 1303302 : struct xfs_rmap_ownercount roc;
2885 1303302 : int error;
2886 :
2887 1303302 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, &res);
2888 1303364 : roc.stop_on_nonmatch = true;
2889 :
2890 1303364 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
2891 : xfs_rmap_count_owners_helper, &roc);
2892 1303379 : if (error == -ECANCELED) {
2893 0 : *has_other = true;
2894 0 : return 0;
2895 : }
2896 1303379 : if (error)
2897 : return error;
2898 :
2899 1303379 : *has_other = false;
2900 1303379 : return 0;
2901 : }
2902 :
2903 : const struct xfs_owner_info XFS_RMAP_OINFO_SKIP_UPDATE = {
2904 : .oi_owner = XFS_RMAP_OWN_NULL,
2905 : };
2906 : const struct xfs_owner_info XFS_RMAP_OINFO_ANY_OWNER = {
2907 : .oi_owner = XFS_RMAP_OWN_UNKNOWN,
2908 : };
2909 : const struct xfs_owner_info XFS_RMAP_OINFO_FS = {
2910 : .oi_owner = XFS_RMAP_OWN_FS,
2911 : };
2912 : const struct xfs_owner_info XFS_RMAP_OINFO_LOG = {
2913 : .oi_owner = XFS_RMAP_OWN_LOG,
2914 : };
2915 : const struct xfs_owner_info XFS_RMAP_OINFO_AG = {
2916 : .oi_owner = XFS_RMAP_OWN_AG,
2917 : };
2918 : const struct xfs_owner_info XFS_RMAP_OINFO_INOBT = {
2919 : .oi_owner = XFS_RMAP_OWN_INOBT,
2920 : };
2921 : const struct xfs_owner_info XFS_RMAP_OINFO_INODES = {
2922 : .oi_owner = XFS_RMAP_OWN_INODES,
2923 : };
2924 : const struct xfs_owner_info XFS_RMAP_OINFO_REFC = {
2925 : .oi_owner = XFS_RMAP_OWN_REFC,
2926 : };
2927 : const struct xfs_owner_info XFS_RMAP_OINFO_COW = {
2928 : .oi_owner = XFS_RMAP_OWN_COW,
2929 : };
2930 :
2931 : int __init
2932 12 : xfs_rmap_intent_init_cache(void)
2933 : {
2934 12 : xfs_rmap_intent_cache = kmem_cache_create("xfs_rmap_intent",
2935 : sizeof(struct xfs_rmap_intent),
2936 : 0, 0, NULL);
2937 :
2938 12 : return xfs_rmap_intent_cache != NULL ? 0 : -ENOMEM;
2939 : }
2940 :
2941 : void
2942 12 : xfs_rmap_intent_destroy_cache(void)
2943 : {
2944 12 : kmem_cache_destroy(xfs_rmap_intent_cache);
2945 12 : xfs_rmap_intent_cache = NULL;
2946 12 : }
|
