Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2014 Red Hat, Inc.
4 : * All Rights Reserved.
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_bit.h"
13 : #include "xfs_mount.h"
14 : #include "xfs_sb.h"
15 : #include "xfs_defer.h"
16 : #include "xfs_btree.h"
17 : #include "xfs_trans.h"
18 : #include "xfs_alloc.h"
19 : #include "xfs_rmap.h"
20 : #include "xfs_rmap_btree.h"
21 : #include "xfs_trace.h"
22 : #include "xfs_errortag.h"
23 : #include "xfs_error.h"
24 : #include "xfs_inode.h"
25 : #include "xfs_ag.h"
26 :
27 : struct kmem_cache *xfs_rmap_intent_cache;
28 :
29 : /*
30 : * Lookup the first record less than or equal to [bno, len, owner, offset]
31 : * in the btree given by cur.
32 : */
33 : int
34 693712693 : xfs_rmap_lookup_le(
35 : struct xfs_btree_cur *cur,
36 : xfs_agblock_t bno,
37 : uint64_t owner,
38 : uint64_t offset,
39 : unsigned int flags,
40 : struct xfs_rmap_irec *irec,
41 : int *stat)
42 : {
43 693712693 : int get_stat = 0;
44 693712693 : int error;
45 :
46 693712693 : cur->bc_rec.r.rm_startblock = bno;
47 693712693 : cur->bc_rec.r.rm_blockcount = 0;
48 693712693 : cur->bc_rec.r.rm_owner = owner;
49 693712693 : cur->bc_rec.r.rm_offset = offset;
50 693712693 : cur->bc_rec.r.rm_flags = flags;
51 :
52 693712693 : error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat);
53 693748470 : if (error || !(*stat) || !irec)
54 : return error;
55 :
56 683843039 : error = xfs_rmap_get_rec(cur, irec, &get_stat);
57 683693500 : if (error)
58 : return error;
59 683693500 : if (!get_stat)
60 0 : return -EFSCORRUPTED;
61 :
62 : return 0;
63 : }
64 :
65 : /*
66 : * Lookup the record exactly matching [bno, len, owner, offset]
67 : * in the btree given by cur.
68 : */
69 : int
70 0 : xfs_rmap_lookup_eq(
71 : struct xfs_btree_cur *cur,
72 : xfs_agblock_t bno,
73 : xfs_extlen_t len,
74 : uint64_t owner,
75 : uint64_t offset,
76 : unsigned int flags,
77 : int *stat)
78 : {
79 371311418 : cur->bc_rec.r.rm_startblock = bno;
80 371311418 : cur->bc_rec.r.rm_blockcount = len;
81 371311418 : cur->bc_rec.r.rm_owner = owner;
82 371311418 : cur->bc_rec.r.rm_offset = offset;
83 371311418 : cur->bc_rec.r.rm_flags = flags;
84 0 : return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat);
85 : }
86 :
87 : /*
88 : * Update the record referred to by cur to the value given
89 : * by [bno, len, owner, offset].
90 : * This either works (return 0) or gets an EFSCORRUPTED error.
91 : */
92 : STATIC int
93 74250608 : xfs_rmap_update(
94 : struct xfs_btree_cur *cur,
95 : struct xfs_rmap_irec *irec)
96 : {
97 74250608 : union xfs_btree_rec rec;
98 74250608 : int error;
99 :
100 74250608 : trace_xfs_rmap_update(cur->bc_mp, cur->bc_ag.pag->pag_agno,
101 : irec->rm_startblock, irec->rm_blockcount,
102 : irec->rm_owner, irec->rm_offset, irec->rm_flags);
103 :
104 74250290 : rec.rmap.rm_startblock = cpu_to_be32(irec->rm_startblock);
105 74250290 : rec.rmap.rm_blockcount = cpu_to_be32(irec->rm_blockcount);
106 74250290 : rec.rmap.rm_owner = cpu_to_be64(irec->rm_owner);
107 74250290 : rec.rmap.rm_offset = cpu_to_be64(
108 : xfs_rmap_irec_offset_pack(irec));
109 74250290 : error = xfs_btree_update(cur, &rec);
110 74251439 : if (error)
111 0 : trace_xfs_rmap_update_error(cur->bc_mp,
112 0 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
113 74251439 : return error;
114 : }
115 :
116 : int
117 130669768 : xfs_rmap_insert(
118 : struct xfs_btree_cur *rcur,
119 : xfs_agblock_t agbno,
120 : xfs_extlen_t len,
121 : uint64_t owner,
122 : uint64_t offset,
123 : unsigned int flags)
124 : {
125 130669768 : int i;
126 130669768 : int error;
127 :
128 130669768 : trace_xfs_rmap_insert(rcur->bc_mp, rcur->bc_ag.pag->pag_agno, agbno,
129 : len, owner, offset, flags);
130 :
131 130669399 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
132 130670022 : if (error)
133 0 : goto done;
134 130670022 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 0)) {
135 0 : error = -EFSCORRUPTED;
136 0 : goto done;
137 : }
138 :
139 130670022 : rcur->bc_rec.r.rm_startblock = agbno;
140 130670022 : rcur->bc_rec.r.rm_blockcount = len;
141 130670022 : rcur->bc_rec.r.rm_owner = owner;
142 130670022 : rcur->bc_rec.r.rm_offset = offset;
143 130670022 : rcur->bc_rec.r.rm_flags = flags;
144 130670022 : error = xfs_btree_insert(rcur, &i);
145 130669549 : if (error)
146 169 : goto done;
147 130669380 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
148 0 : error = -EFSCORRUPTED;
149 0 : goto done;
150 : }
151 130669549 : done:
152 130669549 : if (error)
153 338 : trace_xfs_rmap_insert_error(rcur->bc_mp,
154 169 : rcur->bc_ag.pag->pag_agno, error, _RET_IP_);
155 130669549 : return error;
156 : }
157 :
158 : STATIC int
159 82244829 : xfs_rmap_delete(
160 : struct xfs_btree_cur *rcur,
161 : xfs_agblock_t agbno,
162 : xfs_extlen_t len,
163 : uint64_t owner,
164 : uint64_t offset,
165 : unsigned int flags)
166 : {
167 82244829 : int i;
168 82244829 : int error;
169 :
170 82244829 : trace_xfs_rmap_delete(rcur->bc_mp, rcur->bc_ag.pag->pag_agno, agbno,
171 : len, owner, offset, flags);
172 :
173 82243623 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
174 82247283 : if (error)
175 0 : goto done;
176 82247283 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
177 0 : error = -EFSCORRUPTED;
178 0 : goto done;
179 : }
180 :
181 82247283 : error = xfs_btree_delete(rcur, &i);
182 82246685 : if (error)
183 4 : goto done;
184 82246681 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
185 0 : error = -EFSCORRUPTED;
186 0 : goto done;
187 : }
188 82246685 : done:
189 82246685 : if (error)
190 8 : trace_xfs_rmap_delete_error(rcur->bc_mp,
191 4 : rcur->bc_ag.pag->pag_agno, error, _RET_IP_);
192 82246685 : return error;
193 : }
194 :
195 : /* Convert an internal btree record to an rmap record. */
196 : xfs_failaddr_t
197 50378029225 : xfs_rmap_btrec_to_irec(
198 : const union xfs_btree_rec *rec,
199 : struct xfs_rmap_irec *irec)
200 : {
201 50378029225 : irec->rm_startblock = be32_to_cpu(rec->rmap.rm_startblock);
202 50378029225 : irec->rm_blockcount = be32_to_cpu(rec->rmap.rm_blockcount);
203 50378029225 : irec->rm_owner = be64_to_cpu(rec->rmap.rm_owner);
204 50378029225 : return xfs_rmap_irec_offset_unpack(be64_to_cpu(rec->rmap.rm_offset),
205 : irec);
206 : }
207 :
208 : /* Simple checks for rmap records. */
209 : xfs_failaddr_t
210 50478522948 : xfs_rmap_check_irec(
211 : struct xfs_btree_cur *cur,
212 : const struct xfs_rmap_irec *irec)
213 : {
214 50478522948 : struct xfs_mount *mp = cur->bc_mp;
215 50478522948 : bool is_inode;
216 50478522948 : bool is_unwritten;
217 50478522948 : bool is_bmbt;
218 50478522948 : bool is_attr;
219 :
220 50478522948 : if (irec->rm_blockcount == 0)
221 0 : return __this_address;
222 50478522948 : if (irec->rm_startblock <= XFS_AGFL_BLOCK(mp)) {
223 10231569 : if (irec->rm_owner != XFS_RMAP_OWN_FS)
224 0 : return __this_address;
225 10231569 : if (irec->rm_blockcount != XFS_AGFL_BLOCK(mp) + 1)
226 0 : return __this_address;
227 : } else {
228 : /* check for valid extent range, including overflow */
229 50468291379 : if (!xfs_verify_agbext(cur->bc_ag.pag, irec->rm_startblock,
230 50468291379 : irec->rm_blockcount))
231 0 : return __this_address;
232 : }
233 :
234 50478522948 : if (!(xfs_verify_ino(mp, irec->rm_owner) ||
235 1555621135 : (irec->rm_owner <= XFS_RMAP_OWN_FS &&
236 : irec->rm_owner >= XFS_RMAP_OWN_MIN)))
237 0 : return __this_address;
238 :
239 : /* Check flags. */
240 50516347004 : is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner);
241 50516347004 : is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK;
242 50516347004 : is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK;
243 50516347004 : is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN;
244 :
245 50516347004 : if (is_bmbt && irec->rm_offset != 0)
246 0 : return __this_address;
247 :
248 50516347004 : if (!is_inode && irec->rm_offset != 0)
249 0 : return __this_address;
250 :
251 50516347004 : if (is_unwritten && (is_bmbt || !is_inode || is_attr))
252 0 : return __this_address;
253 :
254 50516347004 : if (!is_inode && (is_bmbt || is_unwritten || is_attr))
255 0 : return __this_address;
256 :
257 : /* Check for a valid fork offset, if applicable. */
258 98837571908 : if (is_inode && !is_bmbt &&
259 48449849744 : !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
260 0 : return __this_address;
261 :
262 : return NULL;
263 : }
264 :
265 : static inline int
266 0 : xfs_rmap_complain_bad_rec(
267 : struct xfs_btree_cur *cur,
268 : xfs_failaddr_t fa,
269 : const struct xfs_rmap_irec *irec)
270 : {
271 0 : struct xfs_mount *mp = cur->bc_mp;
272 :
273 0 : xfs_warn(mp,
274 : "Reverse Mapping BTree record corruption in AG %d detected at %pS!",
275 : cur->bc_ag.pag->pag_agno, fa);
276 0 : xfs_warn(mp,
277 : "Owner 0x%llx, flags 0x%x, start block 0x%x block count 0x%x",
278 : irec->rm_owner, irec->rm_flags, irec->rm_startblock,
279 : irec->rm_blockcount);
280 0 : return -EFSCORRUPTED;
281 : }
282 :
283 : /*
284 : * Get the data from the pointed-to record.
285 : */
286 : int
287 786564153 : xfs_rmap_get_rec(
288 : struct xfs_btree_cur *cur,
289 : struct xfs_rmap_irec *irec,
290 : int *stat)
291 : {
292 786564153 : union xfs_btree_rec *rec;
293 786564153 : xfs_failaddr_t fa;
294 786564153 : int error;
295 :
296 786564153 : error = xfs_btree_get_rec(cur, &rec, stat);
297 786569531 : if (error || !*stat)
298 : return error;
299 :
300 786572772 : fa = xfs_rmap_btrec_to_irec(rec, irec);
301 786513773 : if (!fa)
302 786515559 : fa = xfs_rmap_check_irec(cur, irec);
303 786437966 : if (fa)
304 0 : return xfs_rmap_complain_bad_rec(cur, fa, irec);
305 :
306 : return 0;
307 : }
308 :
309 : struct xfs_find_left_neighbor_info {
310 : struct xfs_rmap_irec high;
311 : struct xfs_rmap_irec *irec;
312 : };
313 :
314 : /* For each rmap given, figure out if it matches the key we want. */
315 : STATIC int
316 202755182 : xfs_rmap_find_left_neighbor_helper(
317 : struct xfs_btree_cur *cur,
318 : const struct xfs_rmap_irec *rec,
319 : void *priv)
320 : {
321 202755182 : struct xfs_find_left_neighbor_info *info = priv;
322 :
323 202755182 : trace_xfs_rmap_find_left_neighbor_candidate(cur->bc_mp,
324 202755182 : cur->bc_ag.pag->pag_agno, rec->rm_startblock,
325 202755182 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
326 202755182 : rec->rm_flags);
327 :
328 202754676 : if (rec->rm_owner != info->high.rm_owner)
329 : return 0;
330 56368297 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
331 56368297 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
332 56360034 : rec->rm_offset + rec->rm_blockcount - 1 != info->high.rm_offset)
333 : return 0;
334 :
335 26682205 : *info->irec = *rec;
336 26682205 : return -ECANCELED;
337 : }
338 :
339 : /*
340 : * Find the record to the left of the given extent, being careful only to
341 : * return a match with the same owner and adjacent physical and logical
342 : * block ranges.
343 : */
344 : STATIC int
345 124391098 : xfs_rmap_find_left_neighbor(
346 : struct xfs_btree_cur *cur,
347 : xfs_agblock_t bno,
348 : uint64_t owner,
349 : uint64_t offset,
350 : unsigned int flags,
351 : struct xfs_rmap_irec *irec,
352 : int *stat)
353 : {
354 124391098 : struct xfs_find_left_neighbor_info info;
355 124391098 : int found = 0;
356 124391098 : int error;
357 :
358 124391098 : *stat = 0;
359 124391098 : if (bno == 0)
360 : return 0;
361 124391098 : info.high.rm_startblock = bno - 1;
362 124391098 : info.high.rm_owner = owner;
363 124391098 : if (!XFS_RMAP_NON_INODE_OWNER(owner) &&
364 124391301 : !(flags & XFS_RMAP_BMBT_BLOCK)) {
365 124391098 : if (offset == 0)
366 : return 0;
367 124200574 : info.high.rm_offset = offset - 1;
368 : } else
369 0 : info.high.rm_offset = 0;
370 124200574 : info.high.rm_flags = flags;
371 124200574 : info.high.rm_blockcount = 0;
372 124200574 : info.irec = irec;
373 :
374 124200574 : trace_xfs_rmap_find_left_neighbor_query(cur->bc_mp,
375 124200574 : cur->bc_ag.pag->pag_agno, bno, 0, owner, offset, flags);
376 :
377 : /*
378 : * Historically, we always used the range query to walk every reverse
379 : * mapping that could possibly overlap the key that the caller asked
380 : * for, and filter out the ones that don't. That is very slow when
381 : * there are a lot of records.
382 : *
383 : * However, there are two scenarios where the classic btree search can
384 : * produce correct results -- if the index contains a record that is an
385 : * exact match for the lookup key; and if there are no other records
386 : * between the record we want and the key we supplied.
387 : *
388 : * As an optimization, try a non-overlapped lookup first. This makes
389 : * extent conversion and remap operations run a bit faster if the
390 : * physical extents aren't being shared. If we don't find what we
391 : * want, we fall back to the overlapped query.
392 : */
393 124199947 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
394 : &found);
395 124200469 : if (error)
396 : return error;
397 124200030 : if (found)
398 124199559 : error = xfs_rmap_find_left_neighbor_helper(cur, irec, &info);
399 124199888 : if (!error)
400 98852234 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
401 : xfs_rmap_find_left_neighbor_helper, &info);
402 124201277 : if (error != -ECANCELED)
403 : return error;
404 :
405 26682205 : *stat = 1;
406 26682205 : trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
407 26682205 : cur->bc_ag.pag->pag_agno, irec->rm_startblock,
408 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
409 : irec->rm_flags);
410 26682205 : return 0;
411 : }
412 :
413 : /* For each rmap given, figure out if it matches the key we want. */
414 : STATIC int
415 246667184 : xfs_rmap_lookup_le_range_helper(
416 : struct xfs_btree_cur *cur,
417 : const struct xfs_rmap_irec *rec,
418 : void *priv)
419 : {
420 246667184 : struct xfs_find_left_neighbor_info *info = priv;
421 :
422 246667184 : trace_xfs_rmap_lookup_le_range_candidate(cur->bc_mp,
423 246667184 : cur->bc_ag.pag->pag_agno, rec->rm_startblock,
424 246667184 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
425 246667184 : rec->rm_flags);
426 :
427 246646486 : if (rec->rm_owner != info->high.rm_owner)
428 : return 0;
429 225627162 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
430 225627162 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
431 225627162 : (rec->rm_offset > info->high.rm_offset ||
432 225460102 : rec->rm_offset + rec->rm_blockcount <= info->high.rm_offset))
433 : return 0;
434 :
435 225356982 : *info->irec = *rec;
436 225356982 : return -ECANCELED;
437 : }
438 :
439 : /*
440 : * Find the record to the left of the given extent, being careful only to
441 : * return a match with the same owner and overlapping physical and logical
442 : * block ranges. This is the overlapping-interval version of
443 : * xfs_rmap_lookup_le.
444 : */
445 : int
446 225390923 : xfs_rmap_lookup_le_range(
447 : struct xfs_btree_cur *cur,
448 : xfs_agblock_t bno,
449 : uint64_t owner,
450 : uint64_t offset,
451 : unsigned int flags,
452 : struct xfs_rmap_irec *irec,
453 : int *stat)
454 : {
455 225390923 : struct xfs_find_left_neighbor_info info;
456 225390923 : int found = 0;
457 225390923 : int error;
458 :
459 225390923 : info.high.rm_startblock = bno;
460 225390923 : info.high.rm_owner = owner;
461 225390923 : if (!XFS_RMAP_NON_INODE_OWNER(owner) && !(flags & XFS_RMAP_BMBT_BLOCK))
462 225390923 : info.high.rm_offset = offset;
463 : else
464 0 : info.high.rm_offset = 0;
465 225390923 : info.high.rm_flags = flags;
466 225390923 : info.high.rm_blockcount = 0;
467 225390923 : *stat = 0;
468 225390923 : info.irec = irec;
469 :
470 225390923 : trace_xfs_rmap_lookup_le_range(cur->bc_mp, cur->bc_ag.pag->pag_agno,
471 : bno, 0, owner, offset, flags);
472 :
473 : /*
474 : * Historically, we always used the range query to walk every reverse
475 : * mapping that could possibly overlap the key that the caller asked
476 : * for, and filter out the ones that don't. That is very slow when
477 : * there are a lot of records.
478 : *
479 : * However, there are two scenarios where the classic btree search can
480 : * produce correct results -- if the index contains a record that is an
481 : * exact match for the lookup key; and if there are no other records
482 : * between the record we want and the key we supplied.
483 : *
484 : * As an optimization, try a non-overlapped lookup first. This makes
485 : * scrub run much faster on most filesystems because bmbt records are
486 : * usually an exact match for rmap records. If we don't find what we
487 : * want, we fall back to the overlapped query.
488 : */
489 225383824 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
490 : &found);
491 225371567 : if (error)
492 : return error;
493 225371045 : if (found)
494 225361392 : error = xfs_rmap_lookup_le_range_helper(cur, irec, &info);
495 225380103 : if (!error)
496 859192 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
497 : xfs_rmap_lookup_le_range_helper, &info);
498 225380107 : if (error != -ECANCELED)
499 : return error;
500 :
501 225367879 : *stat = 1;
502 225367879 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
503 225367879 : cur->bc_ag.pag->pag_agno, irec->rm_startblock,
504 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
505 : irec->rm_flags);
506 225367879 : return 0;
507 : }
508 :
509 : /*
510 : * Perform all the relevant owner checks for a removal op. If we're doing an
511 : * unknown-owner removal then we have no owner information to check.
512 : */
513 : static int
514 91687326 : xfs_rmap_free_check_owner(
515 : struct xfs_mount *mp,
516 : uint64_t ltoff,
517 : struct xfs_rmap_irec *rec,
518 : xfs_filblks_t len,
519 : uint64_t owner,
520 : uint64_t offset,
521 : unsigned int flags)
522 : {
523 91687326 : int error = 0;
524 :
525 91687326 : if (owner == XFS_RMAP_OWN_UNKNOWN)
526 : return 0;
527 :
528 : /* Make sure the unwritten flag matches. */
529 91686920 : if (XFS_IS_CORRUPT(mp,
530 : (flags & XFS_RMAP_UNWRITTEN) !=
531 : (rec->rm_flags & XFS_RMAP_UNWRITTEN))) {
532 0 : error = -EFSCORRUPTED;
533 0 : goto out;
534 : }
535 :
536 : /* Make sure the owner matches what we expect to find in the tree. */
537 91686920 : if (XFS_IS_CORRUPT(mp, owner != rec->rm_owner)) {
538 0 : error = -EFSCORRUPTED;
539 0 : goto out;
540 : }
541 :
542 : /* Check the offset, if necessary. */
543 91686920 : if (XFS_RMAP_NON_INODE_OWNER(owner))
544 14357198 : goto out;
545 :
546 77329722 : if (flags & XFS_RMAP_BMBT_BLOCK) {
547 510102 : if (XFS_IS_CORRUPT(mp,
548 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK))) {
549 0 : error = -EFSCORRUPTED;
550 0 : goto out;
551 : }
552 : } else {
553 76819620 : if (XFS_IS_CORRUPT(mp, rec->rm_offset > offset)) {
554 0 : error = -EFSCORRUPTED;
555 0 : goto out;
556 : }
557 76819620 : if (XFS_IS_CORRUPT(mp,
558 : offset + len > ltoff + rec->rm_blockcount)) {
559 0 : error = -EFSCORRUPTED;
560 0 : goto out;
561 : }
562 : }
563 :
564 : out:
565 : return error;
566 : }
567 :
568 : /*
569 : * Find the extent in the rmap btree and remove it.
570 : *
571 : * The record we find should always be an exact match for the extent that we're
572 : * looking for, since we insert them into the btree without modification.
573 : *
574 : * Special Case #1: when growing the filesystem, we "free" an extent when
575 : * growing the last AG. This extent is new space and so it is not tracked as
576 : * used space in the btree. The growfs code will pass in an owner of
577 : * XFS_RMAP_OWN_NULL to indicate that it expected that there is no owner of this
578 : * extent. We verify that - the extent lookup result in a record that does not
579 : * overlap.
580 : *
581 : * Special Case #2: EFIs do not record the owner of the extent, so when
582 : * recovering EFIs from the log we pass in XFS_RMAP_OWN_UNKNOWN to tell the rmap
583 : * btree to ignore the owner (i.e. wildcard match) so we don't trigger
584 : * corruption checks during log recovery.
585 : */
586 : STATIC int
587 91703442 : xfs_rmap_unmap(
588 : struct xfs_btree_cur *cur,
589 : xfs_agblock_t bno,
590 : xfs_extlen_t len,
591 : bool unwritten,
592 : const struct xfs_owner_info *oinfo)
593 : {
594 91703442 : struct xfs_mount *mp = cur->bc_mp;
595 91703442 : struct xfs_rmap_irec ltrec;
596 91703442 : uint64_t ltoff;
597 91703442 : int error = 0;
598 91703442 : int i;
599 91703442 : uint64_t owner;
600 91703442 : uint64_t offset;
601 91703442 : unsigned int flags;
602 91703442 : bool ignore_off;
603 :
604 91703442 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
605 91703442 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
606 77342155 : (flags & XFS_RMAP_BMBT_BLOCK);
607 91703442 : if (unwritten)
608 3284751 : flags |= XFS_RMAP_UNWRITTEN;
609 91703442 : trace_xfs_rmap_unmap(mp, cur->bc_ag.pag->pag_agno, bno, len,
610 : unwritten, oinfo);
611 :
612 : /*
613 : * We should always have a left record because there's a static record
614 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
615 : * will not ever be removed from the tree.
616 : */
617 91694949 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec, &i);
618 91693104 : if (error)
619 108 : goto out_error;
620 91692996 : if (XFS_IS_CORRUPT(mp, i != 1)) {
621 0 : error = -EFSCORRUPTED;
622 0 : goto out_error;
623 : }
624 :
625 91692996 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
626 91692996 : cur->bc_ag.pag->pag_agno, ltrec.rm_startblock,
627 : ltrec.rm_blockcount, ltrec.rm_owner,
628 : ltrec.rm_offset, ltrec.rm_flags);
629 91688754 : ltoff = ltrec.rm_offset;
630 :
631 : /*
632 : * For growfs, the incoming extent must be beyond the left record we
633 : * just found as it is new space and won't be used by anyone. This is
634 : * just a corruption check as we don't actually do anything with this
635 : * extent. Note that we need to use >= instead of > because it might
636 : * be the case that the "left" extent goes all the way to EOFS.
637 : */
638 91688754 : if (owner == XFS_RMAP_OWN_NULL) {
639 79 : if (XFS_IS_CORRUPT(mp,
640 : bno <
641 : ltrec.rm_startblock + ltrec.rm_blockcount)) {
642 0 : error = -EFSCORRUPTED;
643 0 : goto out_error;
644 : }
645 79 : goto out_done;
646 : }
647 :
648 : /*
649 : * If we're doing an unknown-owner removal for EFI recovery, we expect
650 : * to find the full range in the rmapbt or nothing at all. If we
651 : * don't find any rmaps overlapping either end of the range, we're
652 : * done. Hopefully this means that the EFI creator already queued
653 : * (and finished) a RUI to remove the rmap.
654 : */
655 91688675 : if (owner == XFS_RMAP_OWN_UNKNOWN &&
656 640 : ltrec.rm_startblock + ltrec.rm_blockcount <= bno) {
657 234 : struct xfs_rmap_irec rtrec;
658 :
659 234 : error = xfs_btree_increment(cur, 0, &i);
660 234 : if (error)
661 0 : goto out_error;
662 234 : if (i == 0)
663 234 : goto out_done;
664 220 : error = xfs_rmap_get_rec(cur, &rtrec, &i);
665 220 : if (error)
666 0 : goto out_error;
667 220 : if (XFS_IS_CORRUPT(mp, i != 1)) {
668 0 : error = -EFSCORRUPTED;
669 0 : goto out_error;
670 : }
671 220 : if (rtrec.rm_startblock >= bno + len)
672 220 : goto out_done;
673 : }
674 :
675 : /* Make sure the extent we found covers the entire freeing range. */
676 91688441 : if (XFS_IS_CORRUPT(mp,
677 : ltrec.rm_startblock > bno ||
678 : ltrec.rm_startblock + ltrec.rm_blockcount <
679 : bno + len)) {
680 0 : error = -EFSCORRUPTED;
681 0 : goto out_error;
682 : }
683 :
684 : /* Check owner information. */
685 91688441 : error = xfs_rmap_free_check_owner(mp, ltoff, <rec, len, owner,
686 : offset, flags);
687 91687217 : if (error)
688 0 : goto out_error;
689 :
690 91687217 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
691 : /* exact match, simply remove the record from rmap tree */
692 72463520 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
693 : ltrec.rm_startblock, ltrec.rm_blockcount,
694 : ltrec.rm_owner, ltrec.rm_offset,
695 : ltrec.rm_flags);
696 72463862 : error = xfs_btree_delete(cur, &i);
697 72476094 : if (error)
698 0 : goto out_error;
699 72476094 : if (XFS_IS_CORRUPT(mp, i != 1)) {
700 0 : error = -EFSCORRUPTED;
701 0 : goto out_error;
702 : }
703 19223697 : } else if (ltrec.rm_startblock == bno) {
704 : /*
705 : * overlap left hand side of extent: move the start, trim the
706 : * length and update the current record.
707 : *
708 : * ltbno ltlen
709 : * Orig: |oooooooooooooooooooo|
710 : * Freeing: |fffffffff|
711 : * Result: |rrrrrrrrrr|
712 : * bno len
713 : */
714 4166311 : ltrec.rm_startblock += len;
715 4166311 : ltrec.rm_blockcount -= len;
716 4166311 : if (!ignore_off)
717 333960 : ltrec.rm_offset += len;
718 4166311 : error = xfs_rmap_update(cur, <rec);
719 4166453 : if (error)
720 0 : goto out_error;
721 15057386 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
722 : /*
723 : * overlap right hand side of extent: trim the length and update
724 : * the current record.
725 : *
726 : * ltbno ltlen
727 : * Orig: |oooooooooooooooooooo|
728 : * Freeing: |fffffffff|
729 : * Result: |rrrrrrrrrr|
730 : * bno len
731 : */
732 2031093 : ltrec.rm_blockcount -= len;
733 2031093 : error = xfs_rmap_update(cur, <rec);
734 2031264 : if (error)
735 0 : goto out_error;
736 : } else {
737 :
738 : /*
739 : * overlap middle of extent: trim the length of the existing
740 : * record to the length of the new left-extent size, increment
741 : * the insertion position so we can insert a new record
742 : * containing the remaining right-extent space.
743 : *
744 : * ltbno ltlen
745 : * Orig: |oooooooooooooooooooo|
746 : * Freeing: |fffffffff|
747 : * Result: |rrrrr| |rrrr|
748 : * bno len
749 : */
750 13026293 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
751 :
752 13026293 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
753 13026293 : error = xfs_rmap_update(cur, <rec);
754 13026294 : if (error)
755 0 : goto out_error;
756 :
757 13026294 : error = xfs_btree_increment(cur, 0, &i);
758 13026294 : if (error)
759 0 : goto out_error;
760 :
761 13026294 : cur->bc_rec.r.rm_startblock = bno + len;
762 13026294 : cur->bc_rec.r.rm_blockcount = orig_len - len -
763 13026294 : ltrec.rm_blockcount;
764 13026294 : cur->bc_rec.r.rm_owner = ltrec.rm_owner;
765 13026294 : if (ignore_off)
766 3343346 : cur->bc_rec.r.rm_offset = 0;
767 : else
768 9682948 : cur->bc_rec.r.rm_offset = offset + len;
769 13026294 : cur->bc_rec.r.rm_flags = flags;
770 13026294 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno,
771 : cur->bc_rec.r.rm_startblock,
772 : cur->bc_rec.r.rm_blockcount,
773 : cur->bc_rec.r.rm_owner,
774 : cur->bc_rec.r.rm_offset,
775 : cur->bc_rec.r.rm_flags);
776 13026294 : error = xfs_btree_insert(cur, &i);
777 13026294 : if (error)
778 0 : goto out_error;
779 : }
780 :
781 13026294 : out_done:
782 91700418 : trace_xfs_rmap_unmap_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
783 : unwritten, oinfo);
784 91694575 : out_error:
785 91694575 : if (error)
786 108 : trace_xfs_rmap_unmap_error(mp, cur->bc_ag.pag->pag_agno,
787 108 : error, _RET_IP_);
788 91694575 : return error;
789 : }
790 :
791 : /*
792 : * Remove a reference to an extent in the rmap btree.
793 : */
794 : int
795 1348693 : xfs_rmap_free(
796 : struct xfs_trans *tp,
797 : struct xfs_buf *agbp,
798 : struct xfs_perag *pag,
799 : xfs_agblock_t bno,
800 : xfs_extlen_t len,
801 : const struct xfs_owner_info *oinfo)
802 : {
803 1348693 : struct xfs_mount *mp = tp->t_mountp;
804 1348693 : struct xfs_btree_cur *cur;
805 1348693 : int error;
806 :
807 1348693 : if (!xfs_has_rmapbt(mp))
808 : return 0;
809 :
810 1035547 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
811 :
812 1035550 : error = xfs_rmap_unmap(cur, bno, len, false, oinfo);
813 :
814 1035546 : xfs_btree_del_cursor(cur, error);
815 1035546 : return error;
816 : }
817 :
818 : /*
819 : * A mergeable rmap must have the same owner and the same values for
820 : * the unwritten, attr_fork, and bmbt flags. The startblock and
821 : * offset are checked separately.
822 : */
823 : static bool
824 203925506 : xfs_rmap_is_mergeable(
825 : struct xfs_rmap_irec *irec,
826 : uint64_t owner,
827 : unsigned int flags)
828 : {
829 203925506 : if (irec->rm_owner == XFS_RMAP_OWN_NULL)
830 : return false;
831 203925506 : if (irec->rm_owner != owner)
832 : return false;
833 159441673 : if ((flags & XFS_RMAP_UNWRITTEN) ^
834 159441673 : (irec->rm_flags & XFS_RMAP_UNWRITTEN))
835 : return false;
836 155627693 : if ((flags & XFS_RMAP_ATTR_FORK) ^
837 : (irec->rm_flags & XFS_RMAP_ATTR_FORK))
838 : return false;
839 155622860 : if ((flags & XFS_RMAP_BMBT_BLOCK) ^
840 : (irec->rm_flags & XFS_RMAP_BMBT_BLOCK))
841 163351 : return false;
842 : return true;
843 : }
844 :
845 : /*
846 : * When we allocate a new block, the first thing we do is add a reference to
847 : * the extent in the rmap btree. This takes the form of a [agbno, length,
848 : * owner, offset] record. Flags are encoded in the high bits of the offset
849 : * field.
850 : */
851 : STATIC int
852 92318039 : xfs_rmap_map(
853 : struct xfs_btree_cur *cur,
854 : xfs_agblock_t bno,
855 : xfs_extlen_t len,
856 : bool unwritten,
857 : const struct xfs_owner_info *oinfo)
858 : {
859 92318039 : struct xfs_mount *mp = cur->bc_mp;
860 92318039 : struct xfs_rmap_irec ltrec;
861 92318039 : struct xfs_rmap_irec gtrec;
862 92318039 : int have_gt;
863 92318039 : int have_lt;
864 92318039 : int error = 0;
865 92318039 : int i;
866 92318039 : uint64_t owner;
867 92318039 : uint64_t offset;
868 92318039 : unsigned int flags = 0;
869 92318039 : bool ignore_off;
870 :
871 92318039 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
872 92318039 : ASSERT(owner != 0);
873 92318039 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
874 87533910 : (flags & XFS_RMAP_BMBT_BLOCK);
875 92318039 : if (unwritten)
876 18898421 : flags |= XFS_RMAP_UNWRITTEN;
877 92318039 : trace_xfs_rmap_map(mp, cur->bc_ag.pag->pag_agno, bno, len,
878 : unwritten, oinfo);
879 92314654 : ASSERT(!xfs_rmap_should_skip_owner_update(oinfo));
880 :
881 : /*
882 : * For the initial lookup, look for an exact match or the left-adjacent
883 : * record for our insertion point. This will also give us the record for
884 : * start block contiguity tests.
885 : */
886 92314654 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec,
887 : &have_lt);
888 92315863 : if (error)
889 901 : goto out_error;
890 92314962 : if (have_lt) {
891 92313647 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
892 92313647 : cur->bc_ag.pag->pag_agno, ltrec.rm_startblock,
893 : ltrec.rm_blockcount, ltrec.rm_owner,
894 : ltrec.rm_offset, ltrec.rm_flags);
895 :
896 92313602 : if (!xfs_rmap_is_mergeable(<rec, owner, flags))
897 24210312 : have_lt = 0;
898 : }
899 :
900 92314917 : if (XFS_IS_CORRUPT(mp,
901 : have_lt != 0 &&
902 : ltrec.rm_startblock + ltrec.rm_blockcount > bno)) {
903 0 : error = -EFSCORRUPTED;
904 0 : goto out_error;
905 : }
906 :
907 : /*
908 : * Increment the cursor to see if we have a right-adjacent record to our
909 : * insertion point. This will give us the record for end block
910 : * contiguity tests.
911 : */
912 92314917 : error = xfs_btree_increment(cur, 0, &have_gt);
913 92319101 : if (error)
914 6 : goto out_error;
915 92319095 : if (have_gt) {
916 81147081 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
917 81147249 : if (error)
918 0 : goto out_error;
919 81147249 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
920 0 : error = -EFSCORRUPTED;
921 0 : goto out_error;
922 : }
923 81147249 : if (XFS_IS_CORRUPT(mp, bno + len > gtrec.rm_startblock)) {
924 0 : error = -EFSCORRUPTED;
925 0 : goto out_error;
926 : }
927 81147249 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
928 81147249 : cur->bc_ag.pag->pag_agno, gtrec.rm_startblock,
929 : gtrec.rm_blockcount, gtrec.rm_owner,
930 : gtrec.rm_offset, gtrec.rm_flags);
931 81146704 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
932 23171173 : have_gt = 0;
933 : }
934 :
935 : /*
936 : * Note: cursor currently points one record to the right of ltrec, even
937 : * if there is no record in the tree to the right.
938 : */
939 92318718 : if (have_lt &&
940 68106531 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
941 12600461 : (ignore_off || ltrec.rm_offset + ltrec.rm_blockcount == offset)) {
942 : /*
943 : * left edge contiguous, merge into left record.
944 : *
945 : * ltbno ltlen
946 : * orig: |ooooooooo|
947 : * adding: |aaaaaaaaa|
948 : * result: |rrrrrrrrrrrrrrrrrrr|
949 : * bno len
950 : */
951 11066091 : ltrec.rm_blockcount += len;
952 11066091 : if (have_gt &&
953 3866429 : bno + len == gtrec.rm_startblock &&
954 2904 : (ignore_off || offset + len == gtrec.rm_offset) &&
955 147289 : (unsigned long)ltrec.rm_blockcount + len +
956 147289 : gtrec.rm_blockcount <= XFS_RMAP_LEN_MAX) {
957 : /*
958 : * right edge also contiguous, delete right record
959 : * and merge into left record.
960 : *
961 : * ltbno ltlen gtbno gtlen
962 : * orig: |ooooooooo| |ooooooooo|
963 : * adding: |aaaaaaaaa|
964 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
965 : */
966 147289 : ltrec.rm_blockcount += gtrec.rm_blockcount;
967 147289 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
968 : gtrec.rm_startblock,
969 : gtrec.rm_blockcount,
970 : gtrec.rm_owner,
971 : gtrec.rm_offset,
972 : gtrec.rm_flags);
973 147286 : error = xfs_btree_delete(cur, &i);
974 147293 : if (error)
975 0 : goto out_error;
976 147293 : if (XFS_IS_CORRUPT(mp, i != 1)) {
977 0 : error = -EFSCORRUPTED;
978 0 : goto out_error;
979 : }
980 : }
981 :
982 : /* point the cursor back to the left record and update */
983 11066095 : error = xfs_btree_decrement(cur, 0, &have_gt);
984 11066131 : if (error)
985 0 : goto out_error;
986 11066131 : error = xfs_rmap_update(cur, <rec);
987 11066162 : if (error)
988 0 : goto out_error;
989 81252627 : } else if (have_gt &&
990 54108957 : bno + len == gtrec.rm_startblock &&
991 1195945 : (ignore_off || offset + len == gtrec.rm_offset)) {
992 : /*
993 : * right edge contiguous, merge into right record.
994 : *
995 : * gtbno gtlen
996 : * Orig: |ooooooooo|
997 : * adding: |aaaaaaaaa|
998 : * Result: |rrrrrrrrrrrrrrrrrrr|
999 : * bno len
1000 : */
1001 102857 : gtrec.rm_startblock = bno;
1002 102857 : gtrec.rm_blockcount += len;
1003 102857 : if (!ignore_off)
1004 6034 : gtrec.rm_offset = offset;
1005 102857 : error = xfs_rmap_update(cur, >rec);
1006 102857 : if (error)
1007 0 : goto out_error;
1008 : } else {
1009 : /*
1010 : * no contiguous edge with identical owner, insert
1011 : * new record at current cursor position.
1012 : */
1013 81149770 : cur->bc_rec.r.rm_startblock = bno;
1014 81149770 : cur->bc_rec.r.rm_blockcount = len;
1015 81149770 : cur->bc_rec.r.rm_owner = owner;
1016 81149770 : cur->bc_rec.r.rm_offset = offset;
1017 81149770 : cur->bc_rec.r.rm_flags = flags;
1018 81149770 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1019 : owner, offset, flags);
1020 81149404 : error = xfs_btree_insert(cur, &i);
1021 81150369 : if (error)
1022 37 : goto out_error;
1023 81150332 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1024 0 : error = -EFSCORRUPTED;
1025 0 : goto out_error;
1026 : }
1027 : }
1028 :
1029 92319351 : trace_xfs_rmap_map_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1030 : unwritten, oinfo);
1031 92317125 : out_error:
1032 92317125 : if (error)
1033 944 : trace_xfs_rmap_map_error(mp, cur->bc_ag.pag->pag_agno,
1034 944 : error, _RET_IP_);
1035 92317125 : return error;
1036 : }
1037 :
1038 : /*
1039 : * Add a reference to an extent in the rmap btree.
1040 : */
1041 : int
1042 3131416 : xfs_rmap_alloc(
1043 : struct xfs_trans *tp,
1044 : struct xfs_buf *agbp,
1045 : struct xfs_perag *pag,
1046 : xfs_agblock_t bno,
1047 : xfs_extlen_t len,
1048 : const struct xfs_owner_info *oinfo)
1049 : {
1050 3131416 : struct xfs_mount *mp = tp->t_mountp;
1051 3131416 : struct xfs_btree_cur *cur;
1052 3131416 : int error;
1053 :
1054 3131416 : if (!xfs_has_rmapbt(mp))
1055 : return 0;
1056 :
1057 2285225 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
1058 2285400 : error = xfs_rmap_map(cur, bno, len, false, oinfo);
1059 :
1060 2285137 : xfs_btree_del_cursor(cur, error);
1061 2285137 : return error;
1062 : }
1063 :
1064 : #define RMAP_LEFT_CONTIG (1 << 0)
1065 : #define RMAP_RIGHT_CONTIG (1 << 1)
1066 : #define RMAP_LEFT_FILLING (1 << 2)
1067 : #define RMAP_RIGHT_FILLING (1 << 3)
1068 : #define RMAP_LEFT_VALID (1 << 6)
1069 : #define RMAP_RIGHT_VALID (1 << 7)
1070 :
1071 : #define LEFT r[0]
1072 : #define RIGHT r[1]
1073 : #define PREV r[2]
1074 : #define NEW r[3]
1075 :
1076 : /*
1077 : * Convert an unwritten extent to a real extent or vice versa.
1078 : * Does not handle overlapping extents.
1079 : */
1080 : STATIC int
1081 9903394 : xfs_rmap_convert(
1082 : struct xfs_btree_cur *cur,
1083 : xfs_agblock_t bno,
1084 : xfs_extlen_t len,
1085 : bool unwritten,
1086 : const struct xfs_owner_info *oinfo)
1087 : {
1088 9903394 : struct xfs_mount *mp = cur->bc_mp;
1089 9903394 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1090 : /* left is 0, right is 1, */
1091 : /* prev is 2, new is 3 */
1092 9903394 : uint64_t owner;
1093 9903394 : uint64_t offset;
1094 9903394 : uint64_t new_endoff;
1095 9903394 : unsigned int oldext;
1096 9903394 : unsigned int newext;
1097 9903394 : unsigned int flags = 0;
1098 9903394 : int i;
1099 9903394 : int state = 0;
1100 9903394 : int error;
1101 :
1102 9903394 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1103 9903394 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1104 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1105 9903394 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1106 9903394 : new_endoff = offset + len;
1107 9903394 : trace_xfs_rmap_convert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1108 : unwritten, oinfo);
1109 :
1110 : /*
1111 : * For the initial lookup, look for an exact match or the left-adjacent
1112 : * record for our insertion point. This will also give us the record for
1113 : * start block contiguity tests.
1114 : */
1115 9903307 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, &PREV, &i);
1116 9903325 : if (error)
1117 0 : goto done;
1118 9903325 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1119 0 : error = -EFSCORRUPTED;
1120 0 : goto done;
1121 : }
1122 :
1123 9903325 : trace_xfs_rmap_lookup_le_range_result(cur->bc_mp,
1124 9903325 : cur->bc_ag.pag->pag_agno, PREV.rm_startblock,
1125 : PREV.rm_blockcount, PREV.rm_owner,
1126 : PREV.rm_offset, PREV.rm_flags);
1127 :
1128 9903189 : ASSERT(PREV.rm_offset <= offset);
1129 9903189 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1130 9903189 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1131 9903189 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1132 :
1133 : /*
1134 : * Set flags determining what part of the previous oldext allocation
1135 : * extent is being replaced by a newext allocation.
1136 : */
1137 9903189 : if (PREV.rm_offset == offset)
1138 9416432 : state |= RMAP_LEFT_FILLING;
1139 9903189 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1140 9058086 : state |= RMAP_RIGHT_FILLING;
1141 :
1142 : /*
1143 : * Decrement the cursor to see if we have a left-adjacent record to our
1144 : * insertion point. This will give us the record for end block
1145 : * contiguity tests.
1146 : */
1147 9903189 : error = xfs_btree_decrement(cur, 0, &i);
1148 9903373 : if (error)
1149 0 : goto done;
1150 9903373 : if (i) {
1151 9903377 : state |= RMAP_LEFT_VALID;
1152 9903377 : error = xfs_rmap_get_rec(cur, &LEFT, &i);
1153 9903453 : if (error)
1154 0 : goto done;
1155 9903453 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1156 0 : error = -EFSCORRUPTED;
1157 0 : goto done;
1158 : }
1159 9903453 : if (XFS_IS_CORRUPT(mp,
1160 : LEFT.rm_startblock + LEFT.rm_blockcount >
1161 : bno)) {
1162 0 : error = -EFSCORRUPTED;
1163 0 : goto done;
1164 : }
1165 9903453 : trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp,
1166 9903453 : cur->bc_ag.pag->pag_agno, LEFT.rm_startblock,
1167 : LEFT.rm_blockcount, LEFT.rm_owner,
1168 : LEFT.rm_offset, LEFT.rm_flags);
1169 9903301 : if (LEFT.rm_startblock + LEFT.rm_blockcount == bno &&
1170 8649073 : LEFT.rm_offset + LEFT.rm_blockcount == offset &&
1171 935389 : xfs_rmap_is_mergeable(&LEFT, owner, newext))
1172 928836 : state |= RMAP_LEFT_CONTIG;
1173 : }
1174 :
1175 : /*
1176 : * Increment the cursor to see if we have a right-adjacent record to our
1177 : * insertion point. This will give us the record for end block
1178 : * contiguity tests.
1179 : */
1180 9903297 : error = xfs_btree_increment(cur, 0, &i);
1181 9903435 : if (error)
1182 0 : goto done;
1183 9903435 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1184 0 : error = -EFSCORRUPTED;
1185 0 : goto done;
1186 : }
1187 9903435 : error = xfs_btree_increment(cur, 0, &i);
1188 9903454 : if (error)
1189 0 : goto done;
1190 9903454 : if (i) {
1191 9016461 : state |= RMAP_RIGHT_VALID;
1192 9016461 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1193 9016556 : if (error)
1194 0 : goto done;
1195 9016556 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1196 0 : error = -EFSCORRUPTED;
1197 0 : goto done;
1198 : }
1199 9016556 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1200 0 : error = -EFSCORRUPTED;
1201 0 : goto done;
1202 : }
1203 9016556 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1204 9016556 : cur->bc_ag.pag->pag_agno, RIGHT.rm_startblock,
1205 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1206 : RIGHT.rm_offset, RIGHT.rm_flags);
1207 9016527 : if (bno + len == RIGHT.rm_startblock &&
1208 6410925 : offset + len == RIGHT.rm_offset &&
1209 157263 : xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1210 153102 : state |= RMAP_RIGHT_CONTIG;
1211 : }
1212 :
1213 : /* check that left + prev + right is not too long */
1214 9903520 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1215 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1216 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1217 57312 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1218 57312 : (unsigned long)LEFT.rm_blockcount + len +
1219 57312 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1220 0 : state &= ~RMAP_RIGHT_CONTIG;
1221 :
1222 9903413 : trace_xfs_rmap_convert_state(mp, cur->bc_ag.pag->pag_agno, state,
1223 9903520 : _RET_IP_);
1224 :
1225 : /* reset the cursor back to PREV */
1226 9903305 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, NULL, &i);
1227 9903411 : if (error)
1228 0 : goto done;
1229 9903411 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1230 0 : error = -EFSCORRUPTED;
1231 0 : goto done;
1232 : }
1233 :
1234 : /*
1235 : * Switch out based on the FILLING and CONTIG state bits.
1236 : */
1237 9903411 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1238 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1239 57312 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1240 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1241 : /*
1242 : * Setting all of a previous oldext extent to newext.
1243 : * The left and right neighbors are both contiguous with new.
1244 : */
1245 57312 : error = xfs_btree_increment(cur, 0, &i);
1246 57312 : if (error)
1247 0 : goto done;
1248 57312 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1249 0 : error = -EFSCORRUPTED;
1250 0 : goto done;
1251 : }
1252 57312 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1253 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1254 : RIGHT.rm_owner, RIGHT.rm_offset,
1255 : RIGHT.rm_flags);
1256 57312 : error = xfs_btree_delete(cur, &i);
1257 57312 : if (error)
1258 0 : goto done;
1259 57312 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1260 0 : error = -EFSCORRUPTED;
1261 0 : goto done;
1262 : }
1263 57312 : error = xfs_btree_decrement(cur, 0, &i);
1264 57312 : if (error)
1265 0 : goto done;
1266 57312 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1267 0 : error = -EFSCORRUPTED;
1268 0 : goto done;
1269 : }
1270 57312 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1271 : PREV.rm_startblock, PREV.rm_blockcount,
1272 : PREV.rm_owner, PREV.rm_offset,
1273 : PREV.rm_flags);
1274 57312 : error = xfs_btree_delete(cur, &i);
1275 57312 : if (error)
1276 0 : goto done;
1277 57312 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1278 0 : error = -EFSCORRUPTED;
1279 0 : goto done;
1280 : }
1281 57312 : error = xfs_btree_decrement(cur, 0, &i);
1282 57312 : if (error)
1283 0 : goto done;
1284 57312 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1285 0 : error = -EFSCORRUPTED;
1286 0 : goto done;
1287 : }
1288 57312 : NEW = LEFT;
1289 57312 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1290 57312 : error = xfs_rmap_update(cur, &NEW);
1291 57312 : if (error)
1292 0 : goto done;
1293 : break;
1294 :
1295 589738 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1296 : /*
1297 : * Setting all of a previous oldext extent to newext.
1298 : * The left neighbor is contiguous, the right is not.
1299 : */
1300 589738 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1301 : PREV.rm_startblock, PREV.rm_blockcount,
1302 : PREV.rm_owner, PREV.rm_offset,
1303 : PREV.rm_flags);
1304 589738 : error = xfs_btree_delete(cur, &i);
1305 589739 : if (error)
1306 0 : goto done;
1307 589739 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1308 0 : error = -EFSCORRUPTED;
1309 0 : goto done;
1310 : }
1311 589739 : error = xfs_btree_decrement(cur, 0, &i);
1312 589738 : if (error)
1313 0 : goto done;
1314 589738 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1315 0 : error = -EFSCORRUPTED;
1316 0 : goto done;
1317 : }
1318 589738 : NEW = LEFT;
1319 589738 : NEW.rm_blockcount += PREV.rm_blockcount;
1320 589738 : error = xfs_rmap_update(cur, &NEW);
1321 589739 : if (error)
1322 0 : goto done;
1323 : break;
1324 :
1325 18980 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1326 : /*
1327 : * Setting all of a previous oldext extent to newext.
1328 : * The right neighbor is contiguous, the left is not.
1329 : */
1330 18980 : error = xfs_btree_increment(cur, 0, &i);
1331 18980 : if (error)
1332 0 : goto done;
1333 18980 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1334 0 : error = -EFSCORRUPTED;
1335 0 : goto done;
1336 : }
1337 18980 : trace_xfs_rmap_delete(mp, cur->bc_ag.pag->pag_agno,
1338 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1339 : RIGHT.rm_owner, RIGHT.rm_offset,
1340 : RIGHT.rm_flags);
1341 18980 : error = xfs_btree_delete(cur, &i);
1342 18980 : if (error)
1343 0 : goto done;
1344 18980 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1345 0 : error = -EFSCORRUPTED;
1346 0 : goto done;
1347 : }
1348 18980 : error = xfs_btree_decrement(cur, 0, &i);
1349 18980 : if (error)
1350 0 : goto done;
1351 18980 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1352 0 : error = -EFSCORRUPTED;
1353 0 : goto done;
1354 : }
1355 18980 : NEW = PREV;
1356 18980 : NEW.rm_blockcount = len + RIGHT.rm_blockcount;
1357 18980 : NEW.rm_flags = newext;
1358 18980 : error = xfs_rmap_update(cur, &NEW);
1359 18980 : if (error)
1360 0 : goto done;
1361 : break;
1362 :
1363 8243610 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1364 : /*
1365 : * Setting all of a previous oldext extent to newext.
1366 : * Neither the left nor right neighbors are contiguous with
1367 : * the new one.
1368 : */
1369 8243610 : NEW = PREV;
1370 8243610 : NEW.rm_flags = newext;
1371 8243610 : error = xfs_rmap_update(cur, &NEW);
1372 8243616 : if (error)
1373 0 : goto done;
1374 : break;
1375 :
1376 281786 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1377 : /*
1378 : * Setting the first part of a previous oldext extent to newext.
1379 : * The left neighbor is contiguous.
1380 : */
1381 281786 : NEW = PREV;
1382 281786 : NEW.rm_offset += len;
1383 281786 : NEW.rm_startblock += len;
1384 281786 : NEW.rm_blockcount -= len;
1385 281786 : error = xfs_rmap_update(cur, &NEW);
1386 281786 : if (error)
1387 0 : goto done;
1388 281786 : error = xfs_btree_decrement(cur, 0, &i);
1389 281786 : if (error)
1390 0 : goto done;
1391 281786 : NEW = LEFT;
1392 281786 : NEW.rm_blockcount += len;
1393 281786 : error = xfs_rmap_update(cur, &NEW);
1394 281786 : if (error)
1395 0 : goto done;
1396 : break;
1397 :
1398 225186 : case RMAP_LEFT_FILLING:
1399 : /*
1400 : * Setting the first part of a previous oldext extent to newext.
1401 : * The left neighbor is not contiguous.
1402 : */
1403 225186 : NEW = PREV;
1404 225186 : NEW.rm_startblock += len;
1405 225186 : NEW.rm_offset += len;
1406 225186 : NEW.rm_blockcount -= len;
1407 225186 : error = xfs_rmap_update(cur, &NEW);
1408 225182 : if (error)
1409 0 : goto done;
1410 225182 : NEW.rm_startblock = bno;
1411 225182 : NEW.rm_owner = owner;
1412 225182 : NEW.rm_offset = offset;
1413 225182 : NEW.rm_blockcount = len;
1414 225182 : NEW.rm_flags = newext;
1415 225182 : cur->bc_rec.r = NEW;
1416 225182 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno,
1417 : len, owner, offset, newext);
1418 225180 : error = xfs_btree_insert(cur, &i);
1419 225187 : if (error)
1420 1 : goto done;
1421 225186 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1422 0 : error = -EFSCORRUPTED;
1423 0 : goto done;
1424 : }
1425 : break;
1426 :
1427 76810 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1428 : /*
1429 : * Setting the last part of a previous oldext extent to newext.
1430 : * The right neighbor is contiguous with the new allocation.
1431 : */
1432 76810 : NEW = PREV;
1433 76810 : NEW.rm_blockcount -= len;
1434 76810 : error = xfs_rmap_update(cur, &NEW);
1435 76810 : if (error)
1436 0 : goto done;
1437 76810 : error = xfs_btree_increment(cur, 0, &i);
1438 76810 : if (error)
1439 0 : goto done;
1440 76810 : NEW = RIGHT;
1441 76810 : NEW.rm_offset = offset;
1442 76810 : NEW.rm_startblock = bno;
1443 76810 : NEW.rm_blockcount += len;
1444 76810 : error = xfs_rmap_update(cur, &NEW);
1445 76810 : if (error)
1446 0 : goto done;
1447 : break;
1448 :
1449 71783 : case RMAP_RIGHT_FILLING:
1450 : /*
1451 : * Setting the last part of a previous oldext extent to newext.
1452 : * The right neighbor is not contiguous.
1453 : */
1454 71783 : NEW = PREV;
1455 71783 : NEW.rm_blockcount -= len;
1456 71783 : error = xfs_rmap_update(cur, &NEW);
1457 71783 : if (error)
1458 0 : goto done;
1459 71783 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1460 : oldext, &i);
1461 71783 : if (error)
1462 0 : goto done;
1463 71783 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1464 0 : error = -EFSCORRUPTED;
1465 0 : goto done;
1466 : }
1467 71783 : NEW.rm_startblock = bno;
1468 71783 : NEW.rm_owner = owner;
1469 71783 : NEW.rm_offset = offset;
1470 71783 : NEW.rm_blockcount = len;
1471 71783 : NEW.rm_flags = newext;
1472 71783 : cur->bc_rec.r = NEW;
1473 71783 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno,
1474 : len, owner, offset, newext);
1475 71783 : error = xfs_btree_insert(cur, &i);
1476 71783 : if (error)
1477 0 : goto done;
1478 71783 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1479 0 : error = -EFSCORRUPTED;
1480 0 : goto done;
1481 : }
1482 : break;
1483 :
1484 338206 : case 0:
1485 : /*
1486 : * Setting the middle part of a previous oldext extent to
1487 : * newext. Contiguity is impossible here.
1488 : * One extent becomes three extents.
1489 : */
1490 : /* new right extent - oldext */
1491 338206 : NEW.rm_startblock = bno + len;
1492 338206 : NEW.rm_owner = owner;
1493 338206 : NEW.rm_offset = new_endoff;
1494 338206 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1495 : new_endoff;
1496 338206 : NEW.rm_flags = PREV.rm_flags;
1497 338206 : error = xfs_rmap_update(cur, &NEW);
1498 338207 : if (error)
1499 0 : goto done;
1500 : /* new left extent - oldext */
1501 338207 : NEW = PREV;
1502 338207 : NEW.rm_blockcount = offset - PREV.rm_offset;
1503 338207 : cur->bc_rec.r = NEW;
1504 338207 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno,
1505 : NEW.rm_startblock, NEW.rm_blockcount,
1506 : NEW.rm_owner, NEW.rm_offset,
1507 : NEW.rm_flags);
1508 338206 : error = xfs_btree_insert(cur, &i);
1509 338206 : if (error)
1510 0 : goto done;
1511 338206 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1512 0 : error = -EFSCORRUPTED;
1513 0 : goto done;
1514 : }
1515 : /*
1516 : * Reset the cursor to the position of the new extent
1517 : * we are about to insert as we can't trust it after
1518 : * the previous insert.
1519 : */
1520 338206 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1521 : oldext, &i);
1522 338206 : if (error)
1523 0 : goto done;
1524 338206 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1525 0 : error = -EFSCORRUPTED;
1526 0 : goto done;
1527 : }
1528 : /* new middle extent - newext */
1529 338206 : cur->bc_rec.r.rm_flags &= ~XFS_RMAP_UNWRITTEN;
1530 338206 : cur->bc_rec.r.rm_flags |= newext;
1531 338206 : trace_xfs_rmap_insert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1532 : owner, offset, newext);
1533 338206 : error = xfs_btree_insert(cur, &i);
1534 338207 : if (error)
1535 0 : goto done;
1536 338207 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1537 0 : error = -EFSCORRUPTED;
1538 0 : goto done;
1539 : }
1540 : break;
1541 :
1542 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1543 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1544 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1545 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1546 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1547 : case RMAP_LEFT_CONTIG:
1548 : case RMAP_RIGHT_CONTIG:
1549 : /*
1550 : * These cases are all impossible.
1551 : */
1552 0 : ASSERT(0);
1553 : }
1554 :
1555 9903419 : trace_xfs_rmap_convert_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1556 : unwritten, oinfo);
1557 9903318 : done:
1558 9903318 : if (error)
1559 2 : trace_xfs_rmap_convert_error(cur->bc_mp,
1560 1 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
1561 9903318 : return error;
1562 : }
1563 :
1564 : /*
1565 : * Convert an unwritten extent to a real extent or vice versa. If there is no
1566 : * possibility of overlapping extents, delegate to the simpler convert
1567 : * function.
1568 : */
1569 : STATIC int
1570 14792607 : xfs_rmap_convert_shared(
1571 : struct xfs_btree_cur *cur,
1572 : xfs_agblock_t bno,
1573 : xfs_extlen_t len,
1574 : bool unwritten,
1575 : const struct xfs_owner_info *oinfo)
1576 : {
1577 14792607 : struct xfs_mount *mp = cur->bc_mp;
1578 14792607 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1579 : /* left is 0, right is 1, */
1580 : /* prev is 2, new is 3 */
1581 14792607 : uint64_t owner;
1582 14792607 : uint64_t offset;
1583 14792607 : uint64_t new_endoff;
1584 14792607 : unsigned int oldext;
1585 14792607 : unsigned int newext;
1586 14792607 : unsigned int flags = 0;
1587 14792607 : int i;
1588 14792607 : int state = 0;
1589 14792607 : int error;
1590 :
1591 14792607 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1592 14792607 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1593 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1594 14792607 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1595 14792607 : new_endoff = offset + len;
1596 14792607 : trace_xfs_rmap_convert(mp, cur->bc_ag.pag->pag_agno, bno, len,
1597 : unwritten, oinfo);
1598 :
1599 : /*
1600 : * For the initial lookup, look for and exact match or the left-adjacent
1601 : * record for our insertion point. This will also give us the record for
1602 : * start block contiguity tests.
1603 : */
1604 14792603 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, oldext,
1605 : &PREV, &i);
1606 14792602 : if (error)
1607 1 : goto done;
1608 14792601 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1609 0 : error = -EFSCORRUPTED;
1610 0 : goto done;
1611 : }
1612 :
1613 14792601 : ASSERT(PREV.rm_offset <= offset);
1614 14792601 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1615 14792601 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1616 14792601 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1617 :
1618 : /*
1619 : * Set flags determining what part of the previous oldext allocation
1620 : * extent is being replaced by a newext allocation.
1621 : */
1622 14792601 : if (PREV.rm_offset == offset)
1623 13569343 : state |= RMAP_LEFT_FILLING;
1624 14792601 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1625 10681980 : state |= RMAP_RIGHT_FILLING;
1626 :
1627 : /* Is there a left record that abuts our range? */
1628 14792601 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, newext,
1629 : &LEFT, &i);
1630 14792608 : if (error)
1631 0 : goto done;
1632 14792608 : if (i) {
1633 672121 : state |= RMAP_LEFT_VALID;
1634 672121 : if (XFS_IS_CORRUPT(mp,
1635 : LEFT.rm_startblock + LEFT.rm_blockcount >
1636 : bno)) {
1637 0 : error = -EFSCORRUPTED;
1638 0 : goto done;
1639 : }
1640 672121 : if (xfs_rmap_is_mergeable(&LEFT, owner, newext))
1641 672120 : state |= RMAP_LEFT_CONTIG;
1642 : }
1643 :
1644 : /* Is there a right record that abuts our range? */
1645 14792608 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
1646 : newext, &i);
1647 14792611 : if (error)
1648 0 : goto done;
1649 14792611 : if (i) {
1650 429274 : state |= RMAP_RIGHT_VALID;
1651 429274 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1652 429274 : if (error)
1653 0 : goto done;
1654 429274 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1655 0 : error = -EFSCORRUPTED;
1656 0 : goto done;
1657 : }
1658 429274 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1659 0 : error = -EFSCORRUPTED;
1660 0 : goto done;
1661 : }
1662 429274 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
1663 429274 : cur->bc_ag.pag->pag_agno, RIGHT.rm_startblock,
1664 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1665 : RIGHT.rm_offset, RIGHT.rm_flags);
1666 429274 : if (xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1667 429274 : state |= RMAP_RIGHT_CONTIG;
1668 : }
1669 :
1670 : /* check that left + prev + right is not too long */
1671 14792611 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1672 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1673 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1674 103876 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1675 103876 : (unsigned long)LEFT.rm_blockcount + len +
1676 103876 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1677 0 : state &= ~RMAP_RIGHT_CONTIG;
1678 :
1679 14792611 : trace_xfs_rmap_convert_state(mp, cur->bc_ag.pag->pag_agno, state,
1680 14792611 : _RET_IP_);
1681 : /*
1682 : * Switch out based on the FILLING and CONTIG state bits.
1683 : */
1684 14792610 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1685 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1686 103876 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1687 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1688 : /*
1689 : * Setting all of a previous oldext extent to newext.
1690 : * The left and right neighbors are both contiguous with new.
1691 : */
1692 103876 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1693 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1694 : RIGHT.rm_offset, RIGHT.rm_flags);
1695 103876 : if (error)
1696 0 : goto done;
1697 103876 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1698 : PREV.rm_blockcount, PREV.rm_owner,
1699 : PREV.rm_offset, PREV.rm_flags);
1700 103876 : if (error)
1701 0 : goto done;
1702 103876 : NEW = LEFT;
1703 103876 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1704 : NEW.rm_blockcount, NEW.rm_owner,
1705 : NEW.rm_offset, NEW.rm_flags, &i);
1706 103876 : if (error)
1707 0 : goto done;
1708 103876 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1709 0 : error = -EFSCORRUPTED;
1710 0 : goto done;
1711 : }
1712 103876 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1713 103876 : error = xfs_rmap_update(cur, &NEW);
1714 103876 : if (error)
1715 0 : goto done;
1716 : break;
1717 :
1718 388985 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1719 : /*
1720 : * Setting all of a previous oldext extent to newext.
1721 : * The left neighbor is contiguous, the right is not.
1722 : */
1723 388985 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1724 : PREV.rm_blockcount, PREV.rm_owner,
1725 : PREV.rm_offset, PREV.rm_flags);
1726 388985 : if (error)
1727 0 : goto done;
1728 388985 : NEW = LEFT;
1729 388985 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1730 : NEW.rm_blockcount, NEW.rm_owner,
1731 : NEW.rm_offset, NEW.rm_flags, &i);
1732 388985 : if (error)
1733 0 : goto done;
1734 388985 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1735 0 : error = -EFSCORRUPTED;
1736 0 : goto done;
1737 : }
1738 388985 : NEW.rm_blockcount += PREV.rm_blockcount;
1739 388985 : error = xfs_rmap_update(cur, &NEW);
1740 388985 : if (error)
1741 0 : goto done;
1742 : break;
1743 :
1744 242280 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1745 : /*
1746 : * Setting all of a previous oldext extent to newext.
1747 : * The right neighbor is contiguous, the left is not.
1748 : */
1749 242280 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1750 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1751 : RIGHT.rm_offset, RIGHT.rm_flags);
1752 242280 : if (error)
1753 0 : goto done;
1754 242280 : NEW = PREV;
1755 242280 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1756 : NEW.rm_blockcount, NEW.rm_owner,
1757 : NEW.rm_offset, NEW.rm_flags, &i);
1758 242280 : if (error)
1759 0 : goto done;
1760 242280 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1761 0 : error = -EFSCORRUPTED;
1762 0 : goto done;
1763 : }
1764 242280 : NEW.rm_blockcount += RIGHT.rm_blockcount;
1765 242280 : NEW.rm_flags = RIGHT.rm_flags;
1766 242280 : error = xfs_rmap_update(cur, &NEW);
1767 242280 : if (error)
1768 0 : goto done;
1769 : break;
1770 :
1771 9345956 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1772 : /*
1773 : * Setting all of a previous oldext extent to newext.
1774 : * Neither the left nor right neighbors are contiguous with
1775 : * the new one.
1776 : */
1777 9345956 : NEW = PREV;
1778 9345956 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1779 : NEW.rm_blockcount, NEW.rm_owner,
1780 : NEW.rm_offset, NEW.rm_flags, &i);
1781 9345955 : if (error)
1782 0 : goto done;
1783 9345955 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1784 0 : error = -EFSCORRUPTED;
1785 0 : goto done;
1786 : }
1787 9345955 : NEW.rm_flags = newext;
1788 9345955 : error = xfs_rmap_update(cur, &NEW);
1789 9345958 : if (error)
1790 0 : goto done;
1791 : break;
1792 :
1793 179259 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1794 : /*
1795 : * Setting the first part of a previous oldext extent to newext.
1796 : * The left neighbor is contiguous.
1797 : */
1798 179259 : NEW = PREV;
1799 179259 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1800 : NEW.rm_blockcount, NEW.rm_owner,
1801 : NEW.rm_offset, NEW.rm_flags);
1802 179259 : if (error)
1803 0 : goto done;
1804 179259 : NEW.rm_offset += len;
1805 179259 : NEW.rm_startblock += len;
1806 179259 : NEW.rm_blockcount -= len;
1807 179259 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1808 : NEW.rm_blockcount, NEW.rm_owner,
1809 : NEW.rm_offset, NEW.rm_flags);
1810 179259 : if (error)
1811 0 : goto done;
1812 179259 : NEW = LEFT;
1813 179259 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1814 : NEW.rm_blockcount, NEW.rm_owner,
1815 : NEW.rm_offset, NEW.rm_flags, &i);
1816 179259 : if (error)
1817 0 : goto done;
1818 179259 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1819 0 : error = -EFSCORRUPTED;
1820 0 : goto done;
1821 : }
1822 179259 : NEW.rm_blockcount += len;
1823 179259 : error = xfs_rmap_update(cur, &NEW);
1824 179259 : if (error)
1825 0 : goto done;
1826 : break;
1827 :
1828 3309003 : case RMAP_LEFT_FILLING:
1829 : /*
1830 : * Setting the first part of a previous oldext extent to newext.
1831 : * The left neighbor is not contiguous.
1832 : */
1833 3309003 : NEW = PREV;
1834 3309003 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1835 : NEW.rm_blockcount, NEW.rm_owner,
1836 : NEW.rm_offset, NEW.rm_flags);
1837 3309003 : if (error)
1838 0 : goto done;
1839 3309003 : NEW.rm_offset += len;
1840 3309003 : NEW.rm_startblock += len;
1841 3309003 : NEW.rm_blockcount -= len;
1842 3309003 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1843 : NEW.rm_blockcount, NEW.rm_owner,
1844 : NEW.rm_offset, NEW.rm_flags);
1845 3309003 : if (error)
1846 0 : goto done;
1847 3309003 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1848 3309003 : if (error)
1849 0 : goto done;
1850 : break;
1851 :
1852 83118 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1853 : /*
1854 : * Setting the last part of a previous oldext extent to newext.
1855 : * The right neighbor is contiguous with the new allocation.
1856 : */
1857 83118 : NEW = PREV;
1858 83118 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1859 : NEW.rm_blockcount, NEW.rm_owner,
1860 : NEW.rm_offset, NEW.rm_flags, &i);
1861 83118 : if (error)
1862 0 : goto done;
1863 83118 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1864 0 : error = -EFSCORRUPTED;
1865 0 : goto done;
1866 : }
1867 83118 : NEW.rm_blockcount = offset - NEW.rm_offset;
1868 83118 : error = xfs_rmap_update(cur, &NEW);
1869 83118 : if (error)
1870 0 : goto done;
1871 83118 : NEW = RIGHT;
1872 83118 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
1873 : NEW.rm_blockcount, NEW.rm_owner,
1874 : NEW.rm_offset, NEW.rm_flags);
1875 83118 : if (error)
1876 0 : goto done;
1877 83118 : NEW.rm_offset = offset;
1878 83118 : NEW.rm_startblock = bno;
1879 83118 : NEW.rm_blockcount += len;
1880 83118 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1881 : NEW.rm_blockcount, NEW.rm_owner,
1882 : NEW.rm_offset, NEW.rm_flags);
1883 83118 : if (error)
1884 0 : goto done;
1885 : break;
1886 :
1887 517782 : case RMAP_RIGHT_FILLING:
1888 : /*
1889 : * Setting the last part of a previous oldext extent to newext.
1890 : * The right neighbor is not contiguous.
1891 : */
1892 517782 : NEW = PREV;
1893 517782 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1894 : NEW.rm_blockcount, NEW.rm_owner,
1895 : NEW.rm_offset, NEW.rm_flags, &i);
1896 517782 : if (error)
1897 0 : goto done;
1898 517782 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1899 0 : error = -EFSCORRUPTED;
1900 0 : goto done;
1901 : }
1902 517782 : NEW.rm_blockcount -= len;
1903 517782 : error = xfs_rmap_update(cur, &NEW);
1904 517782 : if (error)
1905 0 : goto done;
1906 517782 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1907 517782 : if (error)
1908 0 : goto done;
1909 : break;
1910 :
1911 622351 : case 0:
1912 : /*
1913 : * Setting the middle part of a previous oldext extent to
1914 : * newext. Contiguity is impossible here.
1915 : * One extent becomes three extents.
1916 : */
1917 : /* new right extent - oldext */
1918 622351 : NEW.rm_startblock = bno + len;
1919 622351 : NEW.rm_owner = owner;
1920 622351 : NEW.rm_offset = new_endoff;
1921 622351 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1922 : new_endoff;
1923 622351 : NEW.rm_flags = PREV.rm_flags;
1924 622351 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1925 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
1926 : NEW.rm_flags);
1927 622351 : if (error)
1928 0 : goto done;
1929 : /* new left extent - oldext */
1930 622351 : NEW = PREV;
1931 622351 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1932 : NEW.rm_blockcount, NEW.rm_owner,
1933 : NEW.rm_offset, NEW.rm_flags, &i);
1934 622351 : if (error)
1935 0 : goto done;
1936 622351 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1937 0 : error = -EFSCORRUPTED;
1938 0 : goto done;
1939 : }
1940 622351 : NEW.rm_blockcount = offset - NEW.rm_offset;
1941 622351 : error = xfs_rmap_update(cur, &NEW);
1942 622351 : if (error)
1943 0 : goto done;
1944 : /* new middle extent - newext */
1945 622351 : NEW.rm_startblock = bno;
1946 622351 : NEW.rm_blockcount = len;
1947 622351 : NEW.rm_owner = owner;
1948 622351 : NEW.rm_offset = offset;
1949 622351 : NEW.rm_flags = newext;
1950 622351 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
1951 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
1952 : NEW.rm_flags);
1953 622351 : if (error)
1954 0 : goto done;
1955 : break;
1956 :
1957 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1958 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1959 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1960 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1961 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1962 : case RMAP_LEFT_CONTIG:
1963 : case RMAP_RIGHT_CONTIG:
1964 : /*
1965 : * These cases are all impossible.
1966 : */
1967 0 : ASSERT(0);
1968 : }
1969 :
1970 14792612 : trace_xfs_rmap_convert_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
1971 : unwritten, oinfo);
1972 14792609 : done:
1973 14792609 : if (error)
1974 2 : trace_xfs_rmap_convert_error(cur->bc_mp,
1975 1 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
1976 14792609 : return error;
1977 : }
1978 :
1979 : #undef NEW
1980 : #undef LEFT
1981 : #undef RIGHT
1982 : #undef PREV
1983 :
1984 : /*
1985 : * Find an extent in the rmap btree and unmap it. For rmap extent types that
1986 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
1987 : * that the prev/next records in the btree might belong to another owner.
1988 : * Therefore we must use delete+insert to alter any of the key fields.
1989 : *
1990 : * For every other situation there can only be one owner for a given extent,
1991 : * so we can call the regular _free function.
1992 : */
1993 : STATIC int
1994 92822021 : xfs_rmap_unmap_shared(
1995 : struct xfs_btree_cur *cur,
1996 : xfs_agblock_t bno,
1997 : xfs_extlen_t len,
1998 : bool unwritten,
1999 : const struct xfs_owner_info *oinfo)
2000 : {
2001 92822021 : struct xfs_mount *mp = cur->bc_mp;
2002 92822021 : struct xfs_rmap_irec ltrec;
2003 92822021 : uint64_t ltoff;
2004 92822021 : int error = 0;
2005 92822021 : int i;
2006 92822021 : uint64_t owner;
2007 92822021 : uint64_t offset;
2008 92822021 : unsigned int flags;
2009 :
2010 92822021 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2011 92822021 : if (unwritten)
2012 16591172 : flags |= XFS_RMAP_UNWRITTEN;
2013 92822021 : trace_xfs_rmap_unmap(mp, cur->bc_ag.pag->pag_agno, bno, len,
2014 : unwritten, oinfo);
2015 :
2016 : /*
2017 : * We should always have a left record because there's a static record
2018 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
2019 : * will not ever be removed from the tree.
2020 : */
2021 92821047 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, flags,
2022 : <rec, &i);
2023 92820088 : if (error)
2024 521 : goto out_error;
2025 92819567 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2026 0 : error = -EFSCORRUPTED;
2027 0 : goto out_error;
2028 : }
2029 92819567 : ltoff = ltrec.rm_offset;
2030 :
2031 : /* Make sure the extent we found covers the entire freeing range. */
2032 92819567 : if (XFS_IS_CORRUPT(mp,
2033 : ltrec.rm_startblock > bno ||
2034 : ltrec.rm_startblock + ltrec.rm_blockcount <
2035 : bno + len)) {
2036 0 : error = -EFSCORRUPTED;
2037 0 : goto out_error;
2038 : }
2039 :
2040 : /* Make sure the owner matches what we expect to find in the tree. */
2041 92819567 : if (XFS_IS_CORRUPT(mp, owner != ltrec.rm_owner)) {
2042 0 : error = -EFSCORRUPTED;
2043 0 : goto out_error;
2044 : }
2045 :
2046 : /* Make sure the unwritten flag matches. */
2047 92819567 : if (XFS_IS_CORRUPT(mp,
2048 : (flags & XFS_RMAP_UNWRITTEN) !=
2049 : (ltrec.rm_flags & XFS_RMAP_UNWRITTEN))) {
2050 0 : error = -EFSCORRUPTED;
2051 0 : goto out_error;
2052 : }
2053 :
2054 : /* Check the offset. */
2055 92819567 : if (XFS_IS_CORRUPT(mp, ltrec.rm_offset > offset)) {
2056 0 : error = -EFSCORRUPTED;
2057 0 : goto out_error;
2058 : }
2059 92819567 : if (XFS_IS_CORRUPT(mp, offset > ltoff + ltrec.rm_blockcount)) {
2060 0 : error = -EFSCORRUPTED;
2061 0 : goto out_error;
2062 : }
2063 :
2064 92819567 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
2065 : /* Exact match, simply remove the record from rmap tree. */
2066 69577581 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2067 : ltrec.rm_blockcount, ltrec.rm_owner,
2068 : ltrec.rm_offset, ltrec.rm_flags);
2069 69579583 : if (error)
2070 4 : goto out_error;
2071 23241986 : } else if (ltrec.rm_startblock == bno) {
2072 : /*
2073 : * Overlap left hand side of extent: move the start, trim the
2074 : * length and update the current record.
2075 : *
2076 : * ltbno ltlen
2077 : * Orig: |oooooooooooooooooooo|
2078 : * Freeing: |fffffffff|
2079 : * Result: |rrrrrrrrrr|
2080 : * bno len
2081 : */
2082 :
2083 : /* Delete prev rmap. */
2084 6369398 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2085 : ltrec.rm_blockcount, ltrec.rm_owner,
2086 : ltrec.rm_offset, ltrec.rm_flags);
2087 6369426 : if (error)
2088 0 : goto out_error;
2089 :
2090 : /* Add an rmap at the new offset. */
2091 6369426 : ltrec.rm_startblock += len;
2092 6369426 : ltrec.rm_blockcount -= len;
2093 6369426 : ltrec.rm_offset += len;
2094 6369426 : error = xfs_rmap_insert(cur, ltrec.rm_startblock,
2095 : ltrec.rm_blockcount, ltrec.rm_owner,
2096 : ltrec.rm_offset, ltrec.rm_flags);
2097 6369446 : if (error)
2098 0 : goto out_error;
2099 16872588 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
2100 : /*
2101 : * Overlap right hand side of extent: trim the length and
2102 : * update the current record.
2103 : *
2104 : * ltbno ltlen
2105 : * Orig: |oooooooooooooooooooo|
2106 : * Freeing: |fffffffff|
2107 : * Result: |rrrrrrrrrr|
2108 : * bno len
2109 : */
2110 5573617 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2111 : ltrec.rm_blockcount, ltrec.rm_owner,
2112 : ltrec.rm_offset, ltrec.rm_flags, &i);
2113 5573630 : if (error)
2114 0 : goto out_error;
2115 5573630 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2116 0 : error = -EFSCORRUPTED;
2117 0 : goto out_error;
2118 : }
2119 5573630 : ltrec.rm_blockcount -= len;
2120 5573630 : error = xfs_rmap_update(cur, <rec);
2121 5573649 : if (error)
2122 0 : goto out_error;
2123 : } else {
2124 : /*
2125 : * Overlap middle of extent: trim the length of the existing
2126 : * record to the length of the new left-extent size, increment
2127 : * the insertion position so we can insert a new record
2128 : * containing the remaining right-extent space.
2129 : *
2130 : * ltbno ltlen
2131 : * Orig: |oooooooooooooooooooo|
2132 : * Freeing: |fffffffff|
2133 : * Result: |rrrrr| |rrrr|
2134 : * bno len
2135 : */
2136 11298971 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
2137 :
2138 : /* Shrink the left side of the rmap */
2139 11298971 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2140 : ltrec.rm_blockcount, ltrec.rm_owner,
2141 : ltrec.rm_offset, ltrec.rm_flags, &i);
2142 11298993 : if (error)
2143 0 : goto out_error;
2144 11298993 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2145 0 : error = -EFSCORRUPTED;
2146 0 : goto out_error;
2147 : }
2148 11298993 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
2149 11298993 : error = xfs_rmap_update(cur, <rec);
2150 11299003 : if (error)
2151 0 : goto out_error;
2152 :
2153 : /* Add an rmap at the new offset */
2154 11299003 : error = xfs_rmap_insert(cur, bno + len,
2155 11299003 : orig_len - len - ltrec.rm_blockcount,
2156 : ltrec.rm_owner, offset + len,
2157 : ltrec.rm_flags);
2158 11299026 : if (error)
2159 1 : goto out_error;
2160 : }
2161 :
2162 92821699 : trace_xfs_rmap_unmap_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2163 : unwritten, oinfo);
2164 92820930 : out_error:
2165 92820930 : if (error)
2166 1052 : trace_xfs_rmap_unmap_error(cur->bc_mp,
2167 526 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2168 92820930 : return error;
2169 : }
2170 :
2171 : /*
2172 : * Find an extent in the rmap btree and map it. For rmap extent types that
2173 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2174 : * that the prev/next records in the btree might belong to another owner.
2175 : * Therefore we must use delete+insert to alter any of the key fields.
2176 : *
2177 : * For every other situation there can only be one owner for a given extent,
2178 : * so we can call the regular _alloc function.
2179 : */
2180 : STATIC int
2181 109599125 : xfs_rmap_map_shared(
2182 : struct xfs_btree_cur *cur,
2183 : xfs_agblock_t bno,
2184 : xfs_extlen_t len,
2185 : bool unwritten,
2186 : const struct xfs_owner_info *oinfo)
2187 : {
2188 109599125 : struct xfs_mount *mp = cur->bc_mp;
2189 109599125 : struct xfs_rmap_irec ltrec;
2190 109599125 : struct xfs_rmap_irec gtrec;
2191 109599125 : int have_gt;
2192 109599125 : int have_lt;
2193 109599125 : int error = 0;
2194 109599125 : int i;
2195 109599125 : uint64_t owner;
2196 109599125 : uint64_t offset;
2197 109599125 : unsigned int flags = 0;
2198 :
2199 109599125 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2200 109599125 : if (unwritten)
2201 19620062 : flags |= XFS_RMAP_UNWRITTEN;
2202 109599125 : trace_xfs_rmap_map(mp, cur->bc_ag.pag->pag_agno, bno, len,
2203 : unwritten, oinfo);
2204 :
2205 : /* Is there a left record that abuts our range? */
2206 109598568 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, flags,
2207 : <rec, &have_lt);
2208 109599633 : if (error)
2209 440 : goto out_error;
2210 109599193 : if (have_lt &&
2211 26010084 : !xfs_rmap_is_mergeable(<rec, owner, flags))
2212 702769 : have_lt = 0;
2213 :
2214 : /* Is there a right record that abuts our range? */
2215 109599193 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
2216 : flags, &have_gt);
2217 109599168 : if (error)
2218 5 : goto out_error;
2219 109599163 : if (have_gt) {
2220 2261183 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
2221 2261183 : if (error)
2222 0 : goto out_error;
2223 2261183 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
2224 0 : error = -EFSCORRUPTED;
2225 0 : goto out_error;
2226 : }
2227 2261183 : trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp,
2228 2261183 : cur->bc_ag.pag->pag_agno, gtrec.rm_startblock,
2229 : gtrec.rm_blockcount, gtrec.rm_owner,
2230 : gtrec.rm_offset, gtrec.rm_flags);
2231 :
2232 2261183 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
2233 373803 : have_gt = 0;
2234 : }
2235 :
2236 109599163 : if (have_lt &&
2237 25307316 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
2238 5240410 : ltrec.rm_offset + ltrec.rm_blockcount == offset) {
2239 : /*
2240 : * Left edge contiguous, merge into left record.
2241 : *
2242 : * ltbno ltlen
2243 : * orig: |ooooooooo|
2244 : * adding: |aaaaaaaaa|
2245 : * result: |rrrrrrrrrrrrrrrrrrr|
2246 : * bno len
2247 : */
2248 5240411 : ltrec.rm_blockcount += len;
2249 5240411 : if (have_gt &&
2250 1185031 : bno + len == gtrec.rm_startblock &&
2251 1185031 : offset + len == gtrec.rm_offset) {
2252 : /*
2253 : * Right edge also contiguous, delete right record
2254 : * and merge into left record.
2255 : *
2256 : * ltbno ltlen gtbno gtlen
2257 : * orig: |ooooooooo| |ooooooooo|
2258 : * adding: |aaaaaaaaa|
2259 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
2260 : */
2261 1185031 : ltrec.rm_blockcount += gtrec.rm_blockcount;
2262 1185031 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2263 : gtrec.rm_blockcount, gtrec.rm_owner,
2264 : gtrec.rm_offset, gtrec.rm_flags);
2265 1185031 : if (error)
2266 0 : goto out_error;
2267 : }
2268 :
2269 : /* Point the cursor back to the left record and update. */
2270 5240411 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2271 : ltrec.rm_blockcount, ltrec.rm_owner,
2272 : ltrec.rm_offset, ltrec.rm_flags, &i);
2273 5240411 : if (error)
2274 0 : goto out_error;
2275 5240411 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2276 0 : error = -EFSCORRUPTED;
2277 0 : goto out_error;
2278 : }
2279 :
2280 5240411 : error = xfs_rmap_update(cur, <rec);
2281 5240411 : if (error)
2282 0 : goto out_error;
2283 104358752 : } else if (have_gt &&
2284 702349 : bno + len == gtrec.rm_startblock &&
2285 702349 : offset + len == gtrec.rm_offset) {
2286 : /*
2287 : * Right edge contiguous, merge into right record.
2288 : *
2289 : * gtbno gtlen
2290 : * Orig: |ooooooooo|
2291 : * adding: |aaaaaaaaa|
2292 : * Result: |rrrrrrrrrrrrrrrrrrr|
2293 : * bno len
2294 : */
2295 : /* Delete the old record. */
2296 702349 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2297 : gtrec.rm_blockcount, gtrec.rm_owner,
2298 : gtrec.rm_offset, gtrec.rm_flags);
2299 702349 : if (error)
2300 0 : goto out_error;
2301 :
2302 : /* Move the start and re-add it. */
2303 702349 : gtrec.rm_startblock = bno;
2304 702349 : gtrec.rm_blockcount += len;
2305 702349 : gtrec.rm_offset = offset;
2306 702349 : error = xfs_rmap_insert(cur, gtrec.rm_startblock,
2307 : gtrec.rm_blockcount, gtrec.rm_owner,
2308 : gtrec.rm_offset, gtrec.rm_flags);
2309 702349 : if (error)
2310 0 : goto out_error;
2311 : } else {
2312 : /*
2313 : * No contiguous edge with identical owner, insert
2314 : * new record at current cursor position.
2315 : */
2316 103656403 : error = xfs_rmap_insert(cur, bno, len, owner, offset, flags);
2317 103656149 : if (error)
2318 168 : goto out_error;
2319 : }
2320 :
2321 109598741 : trace_xfs_rmap_map_done(mp, cur->bc_ag.pag->pag_agno, bno, len,
2322 : unwritten, oinfo);
2323 109599100 : out_error:
2324 109599100 : if (error)
2325 1226 : trace_xfs_rmap_map_error(cur->bc_mp,
2326 613 : cur->bc_ag.pag->pag_agno, error, _RET_IP_);
2327 109599100 : return error;
2328 : }
2329 :
2330 : /* Insert a raw rmap into the rmapbt. */
2331 : int
2332 0 : xfs_rmap_map_raw(
2333 : struct xfs_btree_cur *cur,
2334 : struct xfs_rmap_irec *rmap)
2335 : {
2336 0 : struct xfs_owner_info oinfo;
2337 :
2338 0 : oinfo.oi_owner = rmap->rm_owner;
2339 0 : oinfo.oi_offset = rmap->rm_offset;
2340 0 : oinfo.oi_flags = 0;
2341 0 : if (rmap->rm_flags & XFS_RMAP_ATTR_FORK)
2342 0 : oinfo.oi_flags |= XFS_OWNER_INFO_ATTR_FORK;
2343 0 : if (rmap->rm_flags & XFS_RMAP_BMBT_BLOCK)
2344 0 : oinfo.oi_flags |= XFS_OWNER_INFO_BMBT_BLOCK;
2345 :
2346 0 : if (rmap->rm_flags || XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2347 0 : return xfs_rmap_map(cur, rmap->rm_startblock,
2348 : rmap->rm_blockcount,
2349 0 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2350 : &oinfo);
2351 :
2352 0 : return xfs_rmap_map_shared(cur, rmap->rm_startblock,
2353 : rmap->rm_blockcount,
2354 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2355 : &oinfo);
2356 : }
2357 :
2358 : struct xfs_rmap_query_range_info {
2359 : xfs_rmap_query_range_fn fn;
2360 : void *priv;
2361 : };
2362 :
2363 : /* Format btree record and pass to our callback. */
2364 : STATIC int
2365 49079637387 : xfs_rmap_query_range_helper(
2366 : struct xfs_btree_cur *cur,
2367 : const union xfs_btree_rec *rec,
2368 : void *priv)
2369 : {
2370 49079637387 : struct xfs_rmap_query_range_info *query = priv;
2371 49079637387 : struct xfs_rmap_irec irec;
2372 49079637387 : xfs_failaddr_t fa;
2373 :
2374 49079637387 : fa = xfs_rmap_btrec_to_irec(rec, &irec);
2375 48967379506 : if (!fa)
2376 48963012796 : fa = xfs_rmap_check_irec(cur, &irec);
2377 49018366780 : if (fa)
2378 0 : return xfs_rmap_complain_bad_rec(cur, fa, &irec);
2379 :
2380 49018366780 : return query->fn(cur, &irec, query->priv);
2381 : }
2382 :
2383 : /* Find all rmaps between two keys. */
2384 : int
2385 612421009 : xfs_rmap_query_range(
2386 : struct xfs_btree_cur *cur,
2387 : const struct xfs_rmap_irec *low_rec,
2388 : const struct xfs_rmap_irec *high_rec,
2389 : xfs_rmap_query_range_fn fn,
2390 : void *priv)
2391 : {
2392 612421009 : union xfs_btree_irec low_brec = { .r = *low_rec };
2393 612421009 : union xfs_btree_irec high_brec = { .r = *high_rec };
2394 612421009 : struct xfs_rmap_query_range_info query = { .priv = priv, .fn = fn };
2395 :
2396 612421009 : return xfs_btree_query_range(cur, &low_brec, &high_brec,
2397 : xfs_rmap_query_range_helper, &query);
2398 : }
2399 :
2400 : /* Find all rmaps. */
2401 : int
2402 5092265 : xfs_rmap_query_all(
2403 : struct xfs_btree_cur *cur,
2404 : xfs_rmap_query_range_fn fn,
2405 : void *priv)
2406 : {
2407 5092265 : struct xfs_rmap_query_range_info query;
2408 :
2409 5092265 : query.priv = priv;
2410 5092265 : query.fn = fn;
2411 5092265 : return xfs_btree_query_all(cur, xfs_rmap_query_range_helper, &query);
2412 : }
2413 :
2414 : /* Clean up after calling xfs_rmap_finish_one. */
2415 : void
2416 345841273 : xfs_rmap_finish_one_cleanup(
2417 : struct xfs_trans *tp,
2418 : struct xfs_btree_cur *rcur,
2419 : int error)
2420 : {
2421 345841693 : struct xfs_buf *agbp;
2422 :
2423 345841273 : if (rcur == NULL)
2424 : return;
2425 345840256 : agbp = rcur->bc_ag.agbp;
2426 345839836 : xfs_btree_del_cursor(rcur, error);
2427 345849773 : if (error)
2428 2117 : xfs_trans_brelse(tp, agbp);
2429 : }
2430 :
2431 : /*
2432 : * Process one of the deferred rmap operations. We pass back the
2433 : * btree cursor to maintain our lock on the rmapbt between calls.
2434 : * This saves time and eliminates a buffer deadlock between the
2435 : * superblock and the AGF because we'll always grab them in the same
2436 : * order.
2437 : */
2438 : int
2439 407814506 : xfs_rmap_finish_one(
2440 : struct xfs_trans *tp,
2441 : struct xfs_rmap_intent *ri,
2442 : struct xfs_btree_cur **pcur)
2443 : {
2444 407814506 : struct xfs_mount *mp = tp->t_mountp;
2445 407814506 : struct xfs_btree_cur *rcur;
2446 407814506 : struct xfs_buf *agbp = NULL;
2447 407814506 : int error = 0;
2448 407814506 : struct xfs_owner_info oinfo;
2449 407814506 : xfs_agblock_t bno;
2450 407814506 : bool unwritten;
2451 :
2452 407814506 : bno = XFS_FSB_TO_AGBNO(mp, ri->ri_bmap.br_startblock);
2453 :
2454 407804714 : trace_xfs_rmap_deferred(mp, ri->ri_pag->pag_agno, ri->ri_type, bno,
2455 : ri->ri_owner, ri->ri_whichfork,
2456 : ri->ri_bmap.br_startoff, ri->ri_bmap.br_blockcount,
2457 : ri->ri_bmap.br_state);
2458 :
2459 407807943 : if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_RMAP_FINISH_ONE))
2460 : return -EIO;
2461 :
2462 : /*
2463 : * If we haven't gotten a cursor or the cursor AG doesn't match
2464 : * the startblock, get one now.
2465 : */
2466 407813029 : rcur = *pcur;
2467 407813029 : if (rcur != NULL && rcur->bc_ag.pag != ri->ri_pag) {
2468 420 : xfs_rmap_finish_one_cleanup(tp, rcur, 0);
2469 420 : rcur = NULL;
2470 420 : *pcur = NULL;
2471 : }
2472 407813029 : if (rcur == NULL) {
2473 : /*
2474 : * Refresh the freelist before we start changing the
2475 : * rmapbt, because a shape change could cause us to
2476 : * allocate blocks.
2477 : */
2478 345847489 : error = xfs_free_extent_fix_freelist(tp, ri->ri_pag, &agbp);
2479 345859432 : if (error)
2480 : return error;
2481 345858013 : if (XFS_IS_CORRUPT(tp->t_mountp, !agbp))
2482 0 : return -EFSCORRUPTED;
2483 :
2484 345858013 : rcur = xfs_rmapbt_init_cursor(mp, tp, agbp, ri->ri_pag);
2485 : }
2486 407825701 : *pcur = rcur;
2487 :
2488 407825701 : xfs_rmap_ino_owner(&oinfo, ri->ri_owner, ri->ri_whichfork,
2489 : ri->ri_bmap.br_startoff);
2490 407825701 : unwritten = ri->ri_bmap.br_state == XFS_EXT_UNWRITTEN;
2491 407825701 : bno = XFS_FSB_TO_AGBNO(rcur->bc_mp, ri->ri_bmap.br_startblock);
2492 :
2493 407821990 : switch (ri->ri_type) {
2494 90034061 : case XFS_RMAP_ALLOC:
2495 : case XFS_RMAP_MAP:
2496 90034061 : error = xfs_rmap_map(rcur, bno, ri->ri_bmap.br_blockcount,
2497 : unwritten, &oinfo);
2498 90034061 : break;
2499 109599282 : case XFS_RMAP_MAP_SHARED:
2500 109599282 : error = xfs_rmap_map_shared(rcur, bno,
2501 109599282 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2502 109599282 : break;
2503 90670105 : case XFS_RMAP_FREE:
2504 : case XFS_RMAP_UNMAP:
2505 90670105 : error = xfs_rmap_unmap(rcur, bno, ri->ri_bmap.br_blockcount,
2506 : unwritten, &oinfo);
2507 90670105 : break;
2508 92822483 : case XFS_RMAP_UNMAP_SHARED:
2509 92822483 : error = xfs_rmap_unmap_shared(rcur, bno,
2510 92822483 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2511 92822483 : break;
2512 9903451 : case XFS_RMAP_CONVERT:
2513 9903451 : error = xfs_rmap_convert(rcur, bno, ri->ri_bmap.br_blockcount,
2514 9903451 : !unwritten, &oinfo);
2515 9903451 : break;
2516 14792608 : case XFS_RMAP_CONVERT_SHARED:
2517 14792608 : error = xfs_rmap_convert_shared(rcur, bno,
2518 14792608 : ri->ri_bmap.br_blockcount, !unwritten, &oinfo);
2519 14792608 : break;
2520 0 : default:
2521 0 : ASSERT(0);
2522 0 : error = -EFSCORRUPTED;
2523 : }
2524 :
2525 : return error;
2526 : }
2527 :
2528 : /*
2529 : * Don't defer an rmap if we aren't an rmap filesystem.
2530 : */
2531 : static bool
2532 : xfs_rmap_update_is_needed(
2533 : struct xfs_mount *mp,
2534 : int whichfork)
2535 : {
2536 402449502 : return xfs_has_rmapbt(mp) && whichfork != XFS_COW_FORK;
2537 : }
2538 :
2539 : /*
2540 : * Record a rmap intent; the list is kept sorted first by AG and then by
2541 : * increasing age.
2542 : */
2543 : static void
2544 407756165 : __xfs_rmap_add(
2545 : struct xfs_trans *tp,
2546 : enum xfs_rmap_intent_type type,
2547 : uint64_t owner,
2548 : int whichfork,
2549 : struct xfs_bmbt_irec *bmap)
2550 : {
2551 407756165 : struct xfs_rmap_intent *ri;
2552 :
2553 1223208267 : trace_xfs_rmap_defer(tp->t_mountp,
2554 407726051 : XFS_FSB_TO_AGNO(tp->t_mountp, bmap->br_startblock),
2555 : type,
2556 407756165 : XFS_FSB_TO_AGBNO(tp->t_mountp, bmap->br_startblock),
2557 : owner, whichfork,
2558 : bmap->br_startoff,
2559 : bmap->br_blockcount,
2560 : bmap->br_state);
2561 :
2562 407733625 : ri = kmem_cache_alloc(xfs_rmap_intent_cache, GFP_NOFS | __GFP_NOFAIL);
2563 407779078 : INIT_LIST_HEAD(&ri->ri_list);
2564 407779078 : ri->ri_type = type;
2565 407779078 : ri->ri_owner = owner;
2566 407779078 : ri->ri_whichfork = whichfork;
2567 407779078 : ri->ri_bmap = *bmap;
2568 :
2569 407779078 : xfs_rmap_update_get_group(tp->t_mountp, ri);
2570 407811442 : xfs_defer_add(tp, XFS_DEFER_OPS_TYPE_RMAP, &ri->ri_list);
2571 407772459 : }
2572 :
2573 : /* Map an extent into a file. */
2574 : void
2575 314571197 : xfs_rmap_map_extent(
2576 : struct xfs_trans *tp,
2577 : struct xfs_inode *ip,
2578 : int whichfork,
2579 : struct xfs_bmbt_irec *PREV)
2580 : {
2581 314571197 : enum xfs_rmap_intent_type type = XFS_RMAP_MAP;
2582 :
2583 629142394 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2584 : return;
2585 :
2586 196215095 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2587 109598873 : type = XFS_RMAP_MAP_SHARED;
2588 :
2589 196215095 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2590 : }
2591 :
2592 : /* Unmap an extent out of a file. */
2593 : void
2594 266237067 : xfs_rmap_unmap_extent(
2595 : struct xfs_trans *tp,
2596 : struct xfs_inode *ip,
2597 : int whichfork,
2598 : struct xfs_bmbt_irec *PREV)
2599 : {
2600 266237067 : enum xfs_rmap_intent_type type = XFS_RMAP_UNMAP;
2601 :
2602 532474134 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2603 : return;
2604 :
2605 169614160 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2606 92821163 : type = XFS_RMAP_UNMAP_SHARED;
2607 :
2608 169614160 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2609 : }
2610 :
2611 : /*
2612 : * Convert a data fork extent from unwritten to real or vice versa.
2613 : *
2614 : * Note that tp can be NULL here as no transaction is used for COW fork
2615 : * unwritten conversion.
2616 : */
2617 : void
2618 57597543 : xfs_rmap_convert_extent(
2619 : struct xfs_mount *mp,
2620 : struct xfs_trans *tp,
2621 : struct xfs_inode *ip,
2622 : int whichfork,
2623 : struct xfs_bmbt_irec *PREV)
2624 : {
2625 57597543 : enum xfs_rmap_intent_type type = XFS_RMAP_CONVERT;
2626 :
2627 115195086 : if (!xfs_rmap_update_is_needed(mp, whichfork))
2628 : return;
2629 :
2630 24695769 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2631 14792926 : type = XFS_RMAP_CONVERT_SHARED;
2632 :
2633 24695769 : __xfs_rmap_add(tp, type, ip->i_ino, whichfork, PREV);
2634 : }
2635 :
2636 : /* Schedule the creation of an rmap for non-file data. */
2637 : void
2638 3400477 : xfs_rmap_alloc_extent(
2639 : struct xfs_trans *tp,
2640 : xfs_agnumber_t agno,
2641 : xfs_agblock_t bno,
2642 : xfs_extlen_t len,
2643 : uint64_t owner)
2644 : {
2645 3400477 : struct xfs_bmbt_irec bmap;
2646 :
2647 3400477 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2648 0 : return;
2649 :
2650 3400477 : bmap.br_startblock = XFS_AGB_TO_FSB(tp->t_mountp, agno, bno);
2651 3400477 : bmap.br_blockcount = len;
2652 3400477 : bmap.br_startoff = 0;
2653 3400477 : bmap.br_state = XFS_EXT_NORM;
2654 :
2655 3400477 : __xfs_rmap_add(tp, XFS_RMAP_ALLOC, owner, XFS_DATA_FORK, &bmap);
2656 : }
2657 :
2658 : /* Schedule the deletion of an rmap for non-file data. */
2659 : void
2660 13834666 : xfs_rmap_free_extent(
2661 : struct xfs_trans *tp,
2662 : xfs_agnumber_t agno,
2663 : xfs_agblock_t bno,
2664 : xfs_extlen_t len,
2665 : uint64_t owner)
2666 : {
2667 13834666 : struct xfs_bmbt_irec bmap;
2668 :
2669 13834666 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2670 0 : return;
2671 :
2672 13834666 : bmap.br_startblock = XFS_AGB_TO_FSB(tp->t_mountp, agno, bno);
2673 13834666 : bmap.br_blockcount = len;
2674 13834666 : bmap.br_startoff = 0;
2675 13834666 : bmap.br_state = XFS_EXT_NORM;
2676 :
2677 13834666 : __xfs_rmap_add(tp, XFS_RMAP_FREE, owner, XFS_DATA_FORK, &bmap);
2678 : }
2679 :
2680 : /* Compare rmap records. Returns -1 if a < b, 1 if a > b, and 0 if equal. */
2681 : int
2682 122649889 : xfs_rmap_compare(
2683 : const struct xfs_rmap_irec *a,
2684 : const struct xfs_rmap_irec *b)
2685 : {
2686 122649889 : __u64 oa;
2687 122649889 : __u64 ob;
2688 :
2689 122649889 : oa = xfs_rmap_irec_offset_pack(a);
2690 122649889 : ob = xfs_rmap_irec_offset_pack(b);
2691 :
2692 122649889 : if (a->rm_startblock < b->rm_startblock)
2693 : return -1;
2694 122618210 : else if (a->rm_startblock > b->rm_startblock)
2695 : return 1;
2696 27448327 : else if (a->rm_owner < b->rm_owner)
2697 : return -1;
2698 26890885 : else if (a->rm_owner > b->rm_owner)
2699 : return 1;
2700 26253933 : else if (oa < ob)
2701 : return -1;
2702 26064659 : else if (oa > ob)
2703 : return 1;
2704 : else
2705 12739 : return 0;
2706 : }
2707 :
2708 : /*
2709 : * Scan the physical storage part of the keyspace of the reverse mapping index
2710 : * and tell us if the area has no records, is fully mapped by records, or is
2711 : * partially filled.
2712 : */
2713 : int
2714 322500568 : xfs_rmap_has_records(
2715 : struct xfs_btree_cur *cur,
2716 : xfs_agblock_t bno,
2717 : xfs_extlen_t len,
2718 : enum xbtree_recpacking *outcome)
2719 : {
2720 322500568 : union xfs_btree_key mask = {
2721 : .rmap.rm_startblock = cpu_to_be32(-1U),
2722 : };
2723 322500568 : union xfs_btree_irec low;
2724 322500568 : union xfs_btree_irec high;
2725 :
2726 322500568 : memset(&low, 0, sizeof(low));
2727 322500568 : low.r.rm_startblock = bno;
2728 322500568 : memset(&high, 0xFF, sizeof(high));
2729 322500568 : high.r.rm_startblock = bno + len - 1;
2730 :
2731 322500568 : return xfs_btree_has_records(cur, &low, &high, &mask, outcome);
2732 : }
2733 :
2734 : struct xfs_rmap_ownercount {
2735 : /* Owner that we're looking for. */
2736 : struct xfs_rmap_irec good;
2737 :
2738 : /* rmap search keys */
2739 : struct xfs_rmap_irec low;
2740 : struct xfs_rmap_irec high;
2741 :
2742 : struct xfs_rmap_matches *results;
2743 :
2744 : /* Stop early if we find a nonmatch? */
2745 : bool stop_on_nonmatch;
2746 : };
2747 :
2748 : /* Does this rmap represent space that can have multiple owners? */
2749 : static inline bool
2750 : xfs_rmap_shareable(
2751 : struct xfs_mount *mp,
2752 : const struct xfs_rmap_irec *rmap)
2753 : {
2754 23751097 : if (!xfs_has_reflink(mp))
2755 : return false;
2756 47502148 : if (XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2757 : return false;
2758 23521007 : if (rmap->rm_flags & (XFS_RMAP_ATTR_FORK |
2759 : XFS_RMAP_BMBT_BLOCK))
2760 70884 : return false;
2761 : return true;
2762 : }
2763 :
2764 : static inline void
2765 439449966 : xfs_rmap_ownercount_init(
2766 : struct xfs_rmap_ownercount *roc,
2767 : xfs_agblock_t bno,
2768 : xfs_extlen_t len,
2769 : const struct xfs_owner_info *oinfo,
2770 : struct xfs_rmap_matches *results)
2771 : {
2772 439449966 : memset(roc, 0, sizeof(*roc));
2773 439449966 : roc->results = results;
2774 :
2775 439449966 : roc->low.rm_startblock = bno;
2776 439449966 : memset(&roc->high, 0xFF, sizeof(roc->high));
2777 439449966 : roc->high.rm_startblock = bno + len - 1;
2778 :
2779 439449966 : memset(results, 0, sizeof(*results));
2780 439449966 : roc->good.rm_startblock = bno;
2781 439449966 : roc->good.rm_blockcount = len;
2782 439449966 : roc->good.rm_owner = oinfo->oi_owner;
2783 439449966 : roc->good.rm_offset = oinfo->oi_offset;
2784 439449966 : if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK)
2785 557647 : roc->good.rm_flags |= XFS_RMAP_ATTR_FORK;
2786 439449966 : if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK)
2787 1489822 : roc->good.rm_flags |= XFS_RMAP_BMBT_BLOCK;
2788 439449966 : }
2789 :
2790 : /* Figure out if this is a match for the owner. */
2791 : STATIC int
2792 455098995 : xfs_rmap_count_owners_helper(
2793 : struct xfs_btree_cur *cur,
2794 : const struct xfs_rmap_irec *rec,
2795 : void *priv)
2796 : {
2797 455098995 : struct xfs_rmap_ownercount *roc = priv;
2798 455098995 : struct xfs_rmap_irec check = *rec;
2799 455098995 : unsigned int keyflags;
2800 455098995 : bool filedata;
2801 455098995 : int64_t delta;
2802 :
2803 455098995 : filedata = !XFS_RMAP_NON_INODE_OWNER(check.rm_owner) &&
2804 163079217 : !(check.rm_flags & XFS_RMAP_BMBT_BLOCK);
2805 :
2806 : /* Trim the part of check that comes before the comparison range. */
2807 455098995 : delta = (int64_t)roc->good.rm_startblock - check.rm_startblock;
2808 455098995 : if (delta > 0) {
2809 234067083 : check.rm_startblock += delta;
2810 234067083 : check.rm_blockcount -= delta;
2811 234067083 : if (filedata)
2812 139785 : check.rm_offset += delta;
2813 : }
2814 :
2815 : /* Trim the part of check that comes after the comparison range. */
2816 455098995 : delta = (check.rm_startblock + check.rm_blockcount) -
2817 455098995 : (roc->good.rm_startblock + roc->good.rm_blockcount);
2818 455098995 : if (delta > 0)
2819 254252845 : check.rm_blockcount -= delta;
2820 :
2821 : /* Don't care about unwritten status for establishing ownership. */
2822 455098995 : keyflags = check.rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK);
2823 :
2824 455098995 : if (check.rm_startblock == roc->good.rm_startblock &&
2825 438571379 : check.rm_blockcount == roc->good.rm_blockcount &&
2826 438571379 : check.rm_owner == roc->good.rm_owner &&
2827 431338492 : check.rm_offset == roc->good.rm_offset &&
2828 431342135 : keyflags == roc->good.rm_flags) {
2829 431347898 : roc->results->matches++;
2830 : } else {
2831 23751097 : roc->results->non_owner_matches++;
2832 47502194 : if (xfs_rmap_shareable(cur->bc_mp, &roc->good) ^
2833 : xfs_rmap_shareable(cur->bc_mp, &check))
2834 23450144 : roc->results->bad_non_owner_matches++;
2835 : }
2836 :
2837 455098995 : if (roc->results->non_owner_matches && roc->stop_on_nonmatch)
2838 0 : return -ECANCELED;
2839 :
2840 : return 0;
2841 : }
2842 :
2843 : /* Count the number of owners and non-owners of this range of blocks. */
2844 : int
2845 438072038 : xfs_rmap_count_owners(
2846 : struct xfs_btree_cur *cur,
2847 : xfs_agblock_t bno,
2848 : xfs_extlen_t len,
2849 : const struct xfs_owner_info *oinfo,
2850 : struct xfs_rmap_matches *results)
2851 : {
2852 438072038 : struct xfs_rmap_ownercount roc;
2853 438072038 : int error;
2854 :
2855 438072038 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, results);
2856 437962176 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
2857 : xfs_rmap_count_owners_helper, &roc);
2858 438297270 : if (error)
2859 : return error;
2860 :
2861 : /*
2862 : * There can't be any non-owner rmaps that conflict with the given
2863 : * owner if we didn't find any rmaps matching the owner.
2864 : */
2865 438297270 : if (!results->matches)
2866 8451719 : results->bad_non_owner_matches = 0;
2867 :
2868 : return 0;
2869 : }
2870 :
2871 : /*
2872 : * Given an extent and some owner info, can we find records overlapping
2873 : * the extent whose owner info does not match the given owner?
2874 : */
2875 : int
2876 1472624 : xfs_rmap_has_other_keys(
2877 : struct xfs_btree_cur *cur,
2878 : xfs_agblock_t bno,
2879 : xfs_extlen_t len,
2880 : const struct xfs_owner_info *oinfo,
2881 : bool *has_other)
2882 : {
2883 1472624 : struct xfs_rmap_matches res;
2884 1472624 : struct xfs_rmap_ownercount roc;
2885 1472624 : int error;
2886 :
2887 1472624 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, &res);
2888 1472662 : roc.stop_on_nonmatch = true;
2889 :
2890 1472662 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
2891 : xfs_rmap_count_owners_helper, &roc);
2892 1472778 : if (error == -ECANCELED) {
2893 0 : *has_other = true;
2894 0 : return 0;
2895 : }
2896 1472778 : if (error)
2897 : return error;
2898 :
2899 1472778 : *has_other = false;
2900 1472778 : return 0;
2901 : }
2902 :
2903 : const struct xfs_owner_info XFS_RMAP_OINFO_SKIP_UPDATE = {
2904 : .oi_owner = XFS_RMAP_OWN_NULL,
2905 : };
2906 : const struct xfs_owner_info XFS_RMAP_OINFO_ANY_OWNER = {
2907 : .oi_owner = XFS_RMAP_OWN_UNKNOWN,
2908 : };
2909 : const struct xfs_owner_info XFS_RMAP_OINFO_FS = {
2910 : .oi_owner = XFS_RMAP_OWN_FS,
2911 : };
2912 : const struct xfs_owner_info XFS_RMAP_OINFO_LOG = {
2913 : .oi_owner = XFS_RMAP_OWN_LOG,
2914 : };
2915 : const struct xfs_owner_info XFS_RMAP_OINFO_AG = {
2916 : .oi_owner = XFS_RMAP_OWN_AG,
2917 : };
2918 : const struct xfs_owner_info XFS_RMAP_OINFO_INOBT = {
2919 : .oi_owner = XFS_RMAP_OWN_INOBT,
2920 : };
2921 : const struct xfs_owner_info XFS_RMAP_OINFO_INODES = {
2922 : .oi_owner = XFS_RMAP_OWN_INODES,
2923 : };
2924 : const struct xfs_owner_info XFS_RMAP_OINFO_REFC = {
2925 : .oi_owner = XFS_RMAP_OWN_REFC,
2926 : };
2927 : const struct xfs_owner_info XFS_RMAP_OINFO_COW = {
2928 : .oi_owner = XFS_RMAP_OWN_COW,
2929 : };
2930 :
2931 : int __init
2932 50 : xfs_rmap_intent_init_cache(void)
2933 : {
2934 50 : xfs_rmap_intent_cache = kmem_cache_create("xfs_rmap_intent",
2935 : sizeof(struct xfs_rmap_intent),
2936 : 0, 0, NULL);
2937 :
2938 50 : return xfs_rmap_intent_cache != NULL ? 0 : -ENOMEM;
2939 : }
2940 :
2941 : void
2942 49 : xfs_rmap_intent_destroy_cache(void)
2943 : {
2944 49 : kmem_cache_destroy(xfs_rmap_intent_cache);
2945 49 : xfs_rmap_intent_cache = NULL;
2946 49 : }
|
