Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2014 Red Hat, Inc.
4 : * All Rights Reserved.
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_bit.h"
13 : #include "xfs_mount.h"
14 : #include "xfs_sb.h"
15 : #include "xfs_defer.h"
16 : #include "xfs_btree.h"
17 : #include "xfs_trans.h"
18 : #include "xfs_alloc.h"
19 : #include "xfs_rmap.h"
20 : #include "xfs_rmap_btree.h"
21 : #include "xfs_trace.h"
22 : #include "xfs_errortag.h"
23 : #include "xfs_error.h"
24 : #include "xfs_inode.h"
25 : #include "xfs_ag.h"
26 : #include "xfs_health.h"
27 : #include "xfs_rtgroup.h"
28 : #include "xfs_rtrmap_btree.h"
29 :
30 : struct kmem_cache *xfs_rmap_intent_cache;
31 :
32 : /*
33 : * Lookup the first record less than or equal to [bno, len, owner, offset]
34 : * in the btree given by cur.
35 : */
36 : int
37 1803995898 : xfs_rmap_lookup_le(
38 : struct xfs_btree_cur *cur,
39 : xfs_agblock_t bno,
40 : uint64_t owner,
41 : uint64_t offset,
42 : unsigned int flags,
43 : struct xfs_rmap_irec *irec,
44 : int *stat)
45 : {
46 1803995898 : int get_stat = 0;
47 1803995898 : int error;
48 :
49 1803995898 : cur->bc_rec.r.rm_startblock = bno;
50 1803995898 : cur->bc_rec.r.rm_blockcount = 0;
51 1803995898 : cur->bc_rec.r.rm_owner = owner;
52 1803995898 : cur->bc_rec.r.rm_offset = offset;
53 1803995898 : cur->bc_rec.r.rm_flags = flags;
54 :
55 1803995898 : error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat);
56 1804511912 : if (error || !(*stat) || !irec)
57 : return error;
58 :
59 1775644374 : error = xfs_rmap_get_rec(cur, irec, &get_stat);
60 1775149945 : if (error)
61 : return error;
62 1775149945 : if (!get_stat) {
63 0 : xfs_btree_mark_sick(cur);
64 0 : return -EFSCORRUPTED;
65 : }
66 :
67 : return 0;
68 : }
69 :
70 : /*
71 : * Lookup the record exactly matching [bno, len, owner, offset]
72 : * in the btree given by cur.
73 : */
74 : int
75 0 : xfs_rmap_lookup_eq(
76 : struct xfs_btree_cur *cur,
77 : xfs_agblock_t bno,
78 : xfs_extlen_t len,
79 : uint64_t owner,
80 : uint64_t offset,
81 : unsigned int flags,
82 : int *stat)
83 : {
84 983336463 : cur->bc_rec.r.rm_startblock = bno;
85 983336463 : cur->bc_rec.r.rm_blockcount = len;
86 983336463 : cur->bc_rec.r.rm_owner = owner;
87 983336463 : cur->bc_rec.r.rm_offset = offset;
88 983336463 : cur->bc_rec.r.rm_flags = flags;
89 0 : return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat);
90 : }
91 :
92 : /*
93 : * Update the record referred to by cur to the value given
94 : * by [bno, len, owner, offset].
95 : * This either works (return 0) or gets an EFSCORRUPTED error.
96 : */
97 : STATIC int
98 175355628 : xfs_rmap_update(
99 : struct xfs_btree_cur *cur,
100 : struct xfs_rmap_irec *irec)
101 : {
102 175355628 : union xfs_btree_rec rec;
103 175355628 : int error;
104 :
105 175355628 : trace_xfs_rmap_update(cur, irec->rm_startblock, irec->rm_blockcount,
106 : irec->rm_owner, irec->rm_offset, irec->rm_flags);
107 :
108 175353650 : rec.rmap.rm_startblock = cpu_to_be32(irec->rm_startblock);
109 175353650 : rec.rmap.rm_blockcount = cpu_to_be32(irec->rm_blockcount);
110 175353650 : rec.rmap.rm_owner = cpu_to_be64(irec->rm_owner);
111 175353650 : rec.rmap.rm_offset = cpu_to_be64(
112 : xfs_rmap_irec_offset_pack(irec));
113 175353650 : error = xfs_btree_update(cur, &rec);
114 175356769 : if (error)
115 0 : trace_xfs_rmap_update_error(cur, error, _RET_IP_);
116 175356769 : return error;
117 : }
118 :
119 : int
120 369952119 : xfs_rmap_insert(
121 : struct xfs_btree_cur *rcur,
122 : xfs_agblock_t agbno,
123 : xfs_extlen_t len,
124 : uint64_t owner,
125 : uint64_t offset,
126 : unsigned int flags)
127 : {
128 369952119 : int i;
129 369952119 : int error;
130 :
131 369952119 : trace_xfs_rmap_insert(rcur, agbno, len, owner, offset, flags);
132 :
133 369949065 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
134 369952696 : if (error)
135 0 : goto done;
136 369952696 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 0)) {
137 0 : xfs_btree_mark_sick(rcur);
138 0 : error = -EFSCORRUPTED;
139 0 : goto done;
140 : }
141 :
142 369952696 : rcur->bc_rec.r.rm_startblock = agbno;
143 369952696 : rcur->bc_rec.r.rm_blockcount = len;
144 369952696 : rcur->bc_rec.r.rm_owner = owner;
145 369952696 : rcur->bc_rec.r.rm_offset = offset;
146 369952696 : rcur->bc_rec.r.rm_flags = flags;
147 369952696 : error = xfs_btree_insert(rcur, &i);
148 369953386 : if (error)
149 165 : goto done;
150 369953221 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
151 0 : xfs_btree_mark_sick(rcur);
152 0 : error = -EFSCORRUPTED;
153 0 : goto done;
154 : }
155 369953386 : done:
156 369953386 : if (error)
157 165 : trace_xfs_rmap_insert_error(rcur, error, _RET_IP_);
158 369953386 : return error;
159 : }
160 :
161 : STATIC int
162 141286702 : xfs_rmap_delete(
163 : struct xfs_btree_cur *rcur,
164 : xfs_agblock_t agbno,
165 : xfs_extlen_t len,
166 : uint64_t owner,
167 : uint64_t offset,
168 : unsigned int flags)
169 : {
170 141286702 : int i;
171 141286702 : int error;
172 :
173 141286702 : trace_xfs_rmap_delete(rcur, agbno, len, owner, offset, flags);
174 :
175 141286149 : error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i);
176 141290009 : if (error)
177 0 : goto done;
178 141290009 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
179 0 : xfs_btree_mark_sick(rcur);
180 0 : error = -EFSCORRUPTED;
181 0 : goto done;
182 : }
183 :
184 141290009 : error = xfs_btree_delete(rcur, &i);
185 141288659 : if (error)
186 2 : goto done;
187 141288657 : if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) {
188 0 : xfs_btree_mark_sick(rcur);
189 0 : error = -EFSCORRUPTED;
190 0 : goto done;
191 : }
192 141288659 : done:
193 141288659 : if (error)
194 2 : trace_xfs_rmap_delete_error(rcur, error, _RET_IP_);
195 141288659 : return error;
196 : }
197 :
198 : /* Convert an internal btree record to an rmap record. */
199 : xfs_failaddr_t
200 >22257*10^7 : xfs_rmap_btrec_to_irec(
201 : const union xfs_btree_rec *rec,
202 : struct xfs_rmap_irec *irec)
203 : {
204 >22257*10^7 : irec->rm_startblock = be32_to_cpu(rec->rmap.rm_startblock);
205 >22257*10^7 : irec->rm_blockcount = be32_to_cpu(rec->rmap.rm_blockcount);
206 >22257*10^7 : irec->rm_owner = be64_to_cpu(rec->rmap.rm_owner);
207 >22257*10^7 : return xfs_rmap_irec_offset_unpack(be64_to_cpu(rec->rmap.rm_offset),
208 : irec);
209 : }
210 :
211 : inline xfs_failaddr_t
212 96426223803 : xfs_rmap_check_perag_irec(
213 : struct xfs_perag *pag,
214 : const struct xfs_rmap_irec *irec)
215 : {
216 96426223803 : struct xfs_mount *mp = pag->pag_mount;
217 96426223803 : bool is_inode;
218 96426223803 : bool is_unwritten;
219 96426223803 : bool is_bmbt;
220 96426223803 : bool is_attr;
221 :
222 96426223803 : if (irec->rm_blockcount == 0)
223 0 : return __this_address;
224 96426223803 : if (irec->rm_startblock <= XFS_AGFL_BLOCK(mp)) {
225 30832944 : if (irec->rm_owner != XFS_RMAP_OWN_FS)
226 0 : return __this_address;
227 30832944 : if (irec->rm_blockcount != XFS_AGFL_BLOCK(mp) + 1)
228 0 : return __this_address;
229 : } else {
230 : /* check for valid extent range, including overflow */
231 96395390859 : if (!xfs_verify_agbext(pag, irec->rm_startblock,
232 96395390859 : irec->rm_blockcount))
233 0 : return __this_address;
234 : }
235 :
236 96426223803 : if (!(xfs_verify_ino(mp, irec->rm_owner) ||
237 4673769123 : (irec->rm_owner <= XFS_RMAP_OWN_FS &&
238 : irec->rm_owner >= XFS_RMAP_OWN_MIN)))
239 0 : return __this_address;
240 :
241 : /* Check flags. */
242 96571879676 : is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner);
243 96571879676 : is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK;
244 96571879676 : is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK;
245 96571879676 : is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN;
246 :
247 96571879676 : if (is_bmbt && irec->rm_offset != 0)
248 0 : return __this_address;
249 :
250 96571879676 : if (!is_inode && irec->rm_offset != 0)
251 0 : return __this_address;
252 :
253 96571879676 : if (is_unwritten && (is_bmbt || !is_inode || is_attr))
254 0 : return __this_address;
255 :
256 96571879676 : if (!is_inode && (is_bmbt || is_unwritten || is_attr))
257 0 : return __this_address;
258 :
259 : /* Check for a valid fork offset, if applicable. */
260 >18388*10^7 : if (is_inode && !is_bmbt &&
261 87373017358 : !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
262 0 : return __this_address;
263 :
264 : return NULL;
265 : }
266 :
267 : inline xfs_failaddr_t
268 >12597*10^7 : xfs_rmap_check_rtgroup_irec(
269 : struct xfs_rtgroup *rtg,
270 : const struct xfs_rmap_irec *irec)
271 : {
272 >12597*10^7 : struct xfs_mount *mp = rtg->rtg_mount;
273 >12597*10^7 : bool is_inode;
274 >12597*10^7 : bool is_unwritten;
275 >12597*10^7 : bool is_bmbt;
276 >12597*10^7 : bool is_attr;
277 >12597*10^7 : bool is_cow;
278 :
279 >12597*10^7 : if (irec->rm_blockcount == 0)
280 0 : return __this_address;
281 :
282 >12597*10^7 : if (irec->rm_owner == XFS_RMAP_OWN_FS) {
283 7165734 : if (irec->rm_startblock != 0)
284 0 : return __this_address;
285 7165734 : if (irec->rm_blockcount != mp->m_sb.sb_rextsize)
286 0 : return __this_address;
287 7165734 : if (irec->rm_offset != 0)
288 0 : return __this_address;
289 >12597*10^7 : } else if (irec->rm_owner == XFS_RMAP_OWN_COW) {
290 144096804 : if (!xfs_has_rtreflink(mp))
291 0 : return __this_address;
292 144096804 : if (!xfs_verify_rgbext(rtg, irec->rm_startblock,
293 : irec->rm_blockcount))
294 0 : return __this_address;
295 : } else {
296 >12582*10^7 : if (!xfs_verify_rgbext(rtg, irec->rm_startblock,
297 : irec->rm_blockcount))
298 0 : return __this_address;
299 : }
300 :
301 >12597*10^7 : if (!(xfs_verify_ino(mp, irec->rm_owner) ||
302 151260990 : (irec->rm_owner <= XFS_RMAP_OWN_FS &&
303 : irec->rm_owner >= XFS_RMAP_OWN_MIN)))
304 0 : return __this_address;
305 :
306 : /* Check flags. */
307 >12510*10^7 : is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner);
308 >12510*10^7 : is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK;
309 >12510*10^7 : is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK;
310 >12510*10^7 : is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN;
311 >12510*10^7 : is_cow = xfs_has_rtreflink(mp) &&
312 : irec->rm_owner == XFS_RMAP_OWN_COW;
313 :
314 >12510*10^7 : if (!is_inode && !is_cow && irec->rm_owner != XFS_RMAP_OWN_FS)
315 0 : return __this_address;
316 :
317 >12510*10^7 : if (!is_inode && irec->rm_offset != 0)
318 0 : return __this_address;
319 :
320 >12510*10^7 : if (is_bmbt || is_attr)
321 0 : return __this_address;
322 :
323 >12510*10^7 : if (is_unwritten && !is_inode)
324 0 : return __this_address;
325 :
326 >12510*10^7 : if (is_unwritten && is_cow)
327 0 : return __this_address;
328 :
329 : /* Check for a valid fork offset, if applicable. */
330 >24992*10^7 : if (is_inode &&
331 >12494*10^7 : !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
332 0 : return __this_address;
333 :
334 : return NULL;
335 : }
336 :
337 : /* Simple checks for rmap records. */
338 : xfs_failaddr_t
339 >22209*10^7 : xfs_rmap_check_irec(
340 : struct xfs_btree_cur *cur,
341 : const struct xfs_rmap_irec *irec)
342 : {
343 >22209*10^7 : if (cur->bc_btnum == XFS_BTNUM_RTRMAP) {
344 >12575*10^7 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
345 325376943 : return xfs_rmap_check_rtgroup_irec(cur->bc_mem.rtg,
346 : irec);
347 >12543*10^7 : return xfs_rmap_check_rtgroup_irec(cur->bc_ino.rtg, irec);
348 : }
349 :
350 96337940450 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
351 373165698 : return xfs_rmap_check_perag_irec(cur->bc_mem.pag, irec);
352 95964774752 : return xfs_rmap_check_perag_irec(cur->bc_ag.pag, irec);
353 : }
354 :
355 : static inline int
356 0 : xfs_rmap_complain_bad_rec(
357 : struct xfs_btree_cur *cur,
358 : xfs_failaddr_t fa,
359 : const struct xfs_rmap_irec *irec)
360 : {
361 0 : struct xfs_mount *mp = cur->bc_mp;
362 :
363 0 : if (cur->bc_flags & XFS_BTREE_IN_XFILE)
364 0 : xfs_warn(mp,
365 : "In-Memory Reverse Mapping BTree record corruption detected at %pS!", fa);
366 0 : else if (cur->bc_btnum == XFS_BTNUM_RTRMAP)
367 0 : xfs_warn(mp,
368 : "RT Reverse Mapping BTree record corruption in rtgroup %u detected at %pS!",
369 : cur->bc_ino.rtg->rtg_rgno, fa);
370 : else
371 0 : xfs_warn(mp,
372 : "Reverse Mapping BTree record corruption in AG %d detected at %pS!",
373 : cur->bc_ag.pag->pag_agno, fa);
374 0 : xfs_warn(mp,
375 : "Owner 0x%llx, flags 0x%x, start block 0x%x block count 0x%x",
376 : irec->rm_owner, irec->rm_flags, irec->rm_startblock,
377 : irec->rm_blockcount);
378 0 : xfs_btree_mark_sick(cur);
379 0 : return -EFSCORRUPTED;
380 : }
381 :
382 : /*
383 : * Get the data from the pointed-to record.
384 : */
385 : int
386 3379143843 : xfs_rmap_get_rec(
387 : struct xfs_btree_cur *cur,
388 : struct xfs_rmap_irec *irec,
389 : int *stat)
390 : {
391 3379143843 : union xfs_btree_rec *rec;
392 3379143843 : xfs_failaddr_t fa;
393 3379143843 : int error;
394 :
395 3379143843 : error = xfs_btree_get_rec(cur, &rec, stat);
396 3378974425 : if (error || !*stat)
397 : return error;
398 :
399 3378986947 : fa = xfs_rmap_btrec_to_irec(rec, irec);
400 3378916192 : if (!fa)
401 3378918326 : fa = xfs_rmap_check_irec(cur, irec);
402 3378642562 : if (fa)
403 0 : return xfs_rmap_complain_bad_rec(cur, fa, irec);
404 :
405 : return 0;
406 : }
407 :
408 : struct xfs_find_left_neighbor_info {
409 : struct xfs_rmap_irec high;
410 : struct xfs_rmap_irec *irec;
411 : };
412 :
413 : /* For each rmap given, figure out if it matches the key we want. */
414 : STATIC int
415 882348081 : xfs_rmap_find_left_neighbor_helper(
416 : struct xfs_btree_cur *cur,
417 : const struct xfs_rmap_irec *rec,
418 : void *priv)
419 : {
420 882348081 : struct xfs_find_left_neighbor_info *info = priv;
421 :
422 882348081 : trace_xfs_rmap_find_left_neighbor_candidate(cur, rec->rm_startblock,
423 882348081 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
424 882348081 : rec->rm_flags);
425 :
426 882318167 : if (rec->rm_owner != info->high.rm_owner)
427 : return 0;
428 162178823 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
429 162178823 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
430 161997830 : rec->rm_offset + rec->rm_blockcount - 1 != info->high.rm_offset)
431 : return 0;
432 :
433 61672314 : *info->irec = *rec;
434 61672314 : return -ECANCELED;
435 : }
436 :
437 : /*
438 : * Find the record to the left of the given extent, being careful only to
439 : * return a match with the same owner and adjacent physical and logical
440 : * block ranges.
441 : */
442 : STATIC int
443 380500799 : xfs_rmap_find_left_neighbor(
444 : struct xfs_btree_cur *cur,
445 : xfs_agblock_t bno,
446 : uint64_t owner,
447 : uint64_t offset,
448 : unsigned int flags,
449 : struct xfs_rmap_irec *irec,
450 : int *stat)
451 : {
452 380500799 : struct xfs_find_left_neighbor_info info;
453 380500799 : int found = 0;
454 380500799 : int error;
455 :
456 380500799 : *stat = 0;
457 380500799 : if (bno == 0)
458 : return 0;
459 380500799 : info.high.rm_startblock = bno - 1;
460 380500799 : info.high.rm_owner = owner;
461 380500799 : if (!XFS_RMAP_NON_INODE_OWNER(owner) &&
462 380500428 : !(flags & XFS_RMAP_BMBT_BLOCK)) {
463 380500799 : if (offset == 0)
464 : return 0;
465 376311653 : info.high.rm_offset = offset - 1;
466 : } else
467 0 : info.high.rm_offset = 0;
468 376311653 : info.high.rm_flags = flags;
469 376311653 : info.high.rm_blockcount = 0;
470 376311653 : info.irec = irec;
471 :
472 376311653 : trace_xfs_rmap_find_left_neighbor_query(cur, bno, 0, owner, offset,
473 : flags);
474 :
475 : /*
476 : * Historically, we always used the range query to walk every reverse
477 : * mapping that could possibly overlap the key that the caller asked
478 : * for, and filter out the ones that don't. That is very slow when
479 : * there are a lot of records.
480 : *
481 : * However, there are two scenarios where the classic btree search can
482 : * produce correct results -- if the index contains a record that is an
483 : * exact match for the lookup key; and if there are no other records
484 : * between the record we want and the key we supplied.
485 : *
486 : * As an optimization, try a non-overlapped lookup first. This makes
487 : * extent conversion and remap operations run a bit faster if the
488 : * physical extents aren't being shared. If we don't find what we
489 : * want, we fall back to the overlapped query.
490 : */
491 376297851 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
492 : &found);
493 376312024 : if (error)
494 : return error;
495 376311737 : if (found)
496 376307978 : error = xfs_rmap_find_left_neighbor_helper(cur, irec, &info);
497 376303537 : if (!error)
498 318321759 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
499 : xfs_rmap_find_left_neighbor_helper, &info);
500 376327829 : if (error != -ECANCELED)
501 : return error;
502 :
503 61672310 : *stat = 1;
504 61672310 : trace_xfs_rmap_find_left_neighbor_result(cur, irec->rm_startblock,
505 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
506 : irec->rm_flags);
507 61672310 : return 0;
508 : }
509 :
510 : /* For each rmap given, figure out if it matches the key we want. */
511 : STATIC int
512 520414631 : xfs_rmap_lookup_le_range_helper(
513 : struct xfs_btree_cur *cur,
514 : const struct xfs_rmap_irec *rec,
515 : void *priv)
516 : {
517 520414631 : struct xfs_find_left_neighbor_info *info = priv;
518 :
519 520414631 : trace_xfs_rmap_lookup_le_range_candidate(cur, rec->rm_startblock,
520 520414631 : rec->rm_blockcount, rec->rm_owner, rec->rm_offset,
521 520414631 : rec->rm_flags);
522 :
523 520327164 : if (rec->rm_owner != info->high.rm_owner)
524 : return 0;
525 461557742 : if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) &&
526 461557742 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
527 461557742 : (rec->rm_offset > info->high.rm_offset ||
528 461273171 : rec->rm_offset + rec->rm_blockcount <= info->high.rm_offset))
529 : return 0;
530 :
531 461081130 : *info->irec = *rec;
532 461081130 : return -ECANCELED;
533 : }
534 :
535 : /*
536 : * Find the record to the left of the given extent, being careful only to
537 : * return a match with the same owner and overlapping physical and logical
538 : * block ranges. This is the overlapping-interval version of
539 : * xfs_rmap_lookup_le.
540 : */
541 : int
542 461201842 : xfs_rmap_lookup_le_range(
543 : struct xfs_btree_cur *cur,
544 : xfs_agblock_t bno,
545 : uint64_t owner,
546 : uint64_t offset,
547 : unsigned int flags,
548 : struct xfs_rmap_irec *irec,
549 : int *stat)
550 : {
551 461201842 : struct xfs_find_left_neighbor_info info;
552 461201842 : int found = 0;
553 461201842 : int error;
554 :
555 461201842 : info.high.rm_startblock = bno;
556 461201842 : info.high.rm_owner = owner;
557 461201842 : if (!XFS_RMAP_NON_INODE_OWNER(owner) && !(flags & XFS_RMAP_BMBT_BLOCK))
558 461201842 : info.high.rm_offset = offset;
559 : else
560 0 : info.high.rm_offset = 0;
561 461201842 : info.high.rm_flags = flags;
562 461201842 : info.high.rm_blockcount = 0;
563 461201842 : *stat = 0;
564 461201842 : info.irec = irec;
565 :
566 461201842 : trace_xfs_rmap_lookup_le_range(cur, bno, 0, owner, offset, flags);
567 :
568 : /*
569 : * Historically, we always used the range query to walk every reverse
570 : * mapping that could possibly overlap the key that the caller asked
571 : * for, and filter out the ones that don't. That is very slow when
572 : * there are a lot of records.
573 : *
574 : * However, there are two scenarios where the classic btree search can
575 : * produce correct results -- if the index contains a record that is an
576 : * exact match for the lookup key; and if there are no other records
577 : * between the record we want and the key we supplied.
578 : *
579 : * As an optimization, try a non-overlapped lookup first. This makes
580 : * scrub run much faster on most filesystems because bmbt records are
581 : * usually an exact match for rmap records. If we don't find what we
582 : * want, we fall back to the overlapped query.
583 : */
584 460769500 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec,
585 : &found);
586 461184790 : if (error)
587 : return error;
588 461184516 : if (found)
589 461177080 : error = xfs_rmap_lookup_le_range_helper(cur, irec, &info);
590 461124436 : if (!error)
591 2072577 : error = xfs_rmap_query_range(cur, &info.high, &info.high,
592 : xfs_rmap_lookup_le_range_helper, &info);
593 461124450 : if (error != -ECANCELED)
594 : return error;
595 :
596 461113660 : *stat = 1;
597 461113660 : trace_xfs_rmap_lookup_le_range_result(cur, irec->rm_startblock,
598 : irec->rm_blockcount, irec->rm_owner, irec->rm_offset,
599 : irec->rm_flags);
600 461113660 : return 0;
601 : }
602 :
603 : /*
604 : * Perform all the relevant owner checks for a removal op. If we're doing an
605 : * unknown-owner removal then we have no owner information to check.
606 : */
607 : static int
608 177322726 : xfs_rmap_free_check_owner(
609 : struct xfs_btree_cur *cur,
610 : uint64_t ltoff,
611 : struct xfs_rmap_irec *rec,
612 : xfs_extlen_t len,
613 : uint64_t owner,
614 : uint64_t offset,
615 : unsigned int flags)
616 : {
617 177322726 : struct xfs_mount *mp = cur->bc_mp;
618 177322726 : int error = 0;
619 :
620 177322726 : if (owner == XFS_RMAP_OWN_UNKNOWN)
621 : return 0;
622 :
623 : /* Make sure the unwritten flag matches. */
624 177322224 : if (XFS_IS_CORRUPT(mp,
625 : (flags & XFS_RMAP_UNWRITTEN) !=
626 : (rec->rm_flags & XFS_RMAP_UNWRITTEN))) {
627 0 : xfs_btree_mark_sick(cur);
628 0 : error = -EFSCORRUPTED;
629 0 : goto out;
630 : }
631 :
632 : /* Make sure the owner matches what we expect to find in the tree. */
633 177322224 : if (XFS_IS_CORRUPT(mp, owner != rec->rm_owner)) {
634 0 : xfs_btree_mark_sick(cur);
635 0 : error = -EFSCORRUPTED;
636 0 : goto out;
637 : }
638 :
639 : /* Check the offset, if necessary. */
640 177322224 : if (XFS_RMAP_NON_INODE_OWNER(owner))
641 22581279 : goto out;
642 :
643 154740945 : if (flags & XFS_RMAP_BMBT_BLOCK) {
644 2806333 : if (XFS_IS_CORRUPT(mp,
645 : !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK))) {
646 0 : xfs_btree_mark_sick(cur);
647 0 : error = -EFSCORRUPTED;
648 0 : goto out;
649 : }
650 : } else {
651 151934612 : if (XFS_IS_CORRUPT(mp, rec->rm_offset > offset)) {
652 0 : xfs_btree_mark_sick(cur);
653 0 : error = -EFSCORRUPTED;
654 0 : goto out;
655 : }
656 151934612 : if (XFS_IS_CORRUPT(mp,
657 : offset + len > ltoff + rec->rm_blockcount)) {
658 0 : xfs_btree_mark_sick(cur);
659 0 : error = -EFSCORRUPTED;
660 0 : goto out;
661 : }
662 : }
663 :
664 : out:
665 : return error;
666 : }
667 :
668 : /*
669 : * Find the extent in the rmap btree and remove it.
670 : *
671 : * The record we find should always be an exact match for the extent that we're
672 : * looking for, since we insert them into the btree without modification.
673 : *
674 : * Special Case #1: when growing the filesystem, we "free" an extent when
675 : * growing the last AG. This extent is new space and so it is not tracked as
676 : * used space in the btree. The growfs code will pass in an owner of
677 : * XFS_RMAP_OWN_NULL to indicate that it expected that there is no owner of this
678 : * extent. We verify that - the extent lookup result in a record that does not
679 : * overlap.
680 : *
681 : * Special Case #2: EFIs do not record the owner of the extent, so when
682 : * recovering EFIs from the log we pass in XFS_RMAP_OWN_UNKNOWN to tell the rmap
683 : * btree to ignore the owner (i.e. wildcard match) so we don't trigger
684 : * corruption checks during log recovery.
685 : */
686 : STATIC int
687 177454576 : xfs_rmap_unmap(
688 : struct xfs_btree_cur *cur,
689 : xfs_agblock_t bno,
690 : xfs_extlen_t len,
691 : bool unwritten,
692 : const struct xfs_owner_info *oinfo)
693 : {
694 177454576 : struct xfs_mount *mp = cur->bc_mp;
695 177454576 : struct xfs_rmap_irec ltrec;
696 177454576 : uint64_t ltoff;
697 177454576 : int error = 0;
698 177454576 : int i;
699 177454576 : uint64_t owner;
700 177454576 : uint64_t offset;
701 177454576 : unsigned int flags;
702 177454576 : bool ignore_off;
703 :
704 177454576 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
705 177454576 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
706 154769771 : (flags & XFS_RMAP_BMBT_BLOCK);
707 177454576 : if (unwritten)
708 13233193 : flags |= XFS_RMAP_UNWRITTEN;
709 177454576 : trace_xfs_rmap_unmap(cur, bno, len, unwritten, oinfo);
710 :
711 : /*
712 : * We should always have a left record because there's a static record
713 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
714 : * will not ever be removed from the tree.
715 : */
716 177439904 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec, &i);
717 177431729 : if (error)
718 96 : goto out_error;
719 177431633 : if (XFS_IS_CORRUPT(mp, i != 1)) {
720 0 : xfs_btree_mark_sick(cur);
721 0 : error = -EFSCORRUPTED;
722 0 : goto out_error;
723 : }
724 :
725 177431633 : trace_xfs_rmap_lookup_le_range_result(cur, ltrec.rm_startblock,
726 : ltrec.rm_blockcount, ltrec.rm_owner, ltrec.rm_offset,
727 : ltrec.rm_flags);
728 177413065 : ltoff = ltrec.rm_offset;
729 :
730 : /*
731 : * For growfs, the incoming extent must be beyond the left record we
732 : * just found as it is new space and won't be used by anyone. This is
733 : * just a corruption check as we don't actually do anything with this
734 : * extent. Note that we need to use >= instead of > because it might
735 : * be the case that the "left" extent goes all the way to EOFS.
736 : */
737 177413065 : if (owner == XFS_RMAP_OWN_NULL) {
738 79 : if (XFS_IS_CORRUPT(mp,
739 : bno <
740 : ltrec.rm_startblock + ltrec.rm_blockcount)) {
741 0 : xfs_btree_mark_sick(cur);
742 0 : error = -EFSCORRUPTED;
743 0 : goto out_error;
744 : }
745 79 : goto out_done;
746 : }
747 :
748 : /*
749 : * If we're doing an unknown-owner removal for EFI recovery, we expect
750 : * to find the full range in the rmapbt or nothing at all. If we
751 : * don't find any rmaps overlapping either end of the range, we're
752 : * done. Hopefully this means that the EFI creator already queued
753 : * (and finished) a RUI to remove the rmap.
754 : */
755 177412986 : if (owner == XFS_RMAP_OWN_UNKNOWN &&
756 96689 : ltrec.rm_startblock + ltrec.rm_blockcount <= bno) {
757 96127 : struct xfs_rmap_irec rtrec;
758 :
759 96127 : error = xfs_btree_increment(cur, 0, &i);
760 96312 : if (error)
761 0 : goto out_error;
762 96312 : if (i == 0)
763 96400 : goto out_done;
764 95730 : error = xfs_rmap_get_rec(cur, &rtrec, &i);
765 95818 : if (error)
766 0 : goto out_error;
767 95818 : if (XFS_IS_CORRUPT(mp, i != 1)) {
768 0 : xfs_btree_mark_sick(cur);
769 0 : error = -EFSCORRUPTED;
770 0 : goto out_error;
771 : }
772 95818 : if (rtrec.rm_startblock >= bno + len)
773 95818 : goto out_done;
774 : }
775 :
776 : /* Make sure the extent we found covers the entire freeing range. */
777 177316859 : if (XFS_IS_CORRUPT(mp,
778 : ltrec.rm_startblock > bno ||
779 : ltrec.rm_startblock + ltrec.rm_blockcount <
780 : bno + len)) {
781 0 : xfs_btree_mark_sick(cur);
782 0 : error = -EFSCORRUPTED;
783 0 : goto out_error;
784 : }
785 :
786 : /* Check owner information. */
787 177316859 : error = xfs_rmap_free_check_owner(cur, ltoff, <rec, len, owner,
788 : offset, flags);
789 177315623 : if (error)
790 0 : goto out_error;
791 :
792 177315623 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
793 : /* exact match, simply remove the record from rmap tree */
794 148705580 : trace_xfs_rmap_delete(cur, ltrec.rm_startblock,
795 : ltrec.rm_blockcount, ltrec.rm_owner,
796 : ltrec.rm_offset, ltrec.rm_flags);
797 148712197 : error = xfs_btree_delete(cur, &i);
798 148730406 : if (error)
799 0 : goto out_error;
800 148730406 : if (XFS_IS_CORRUPT(mp, i != 1)) {
801 0 : xfs_btree_mark_sick(cur);
802 0 : error = -EFSCORRUPTED;
803 0 : goto out_error;
804 : }
805 28610043 : } else if (ltrec.rm_startblock == bno) {
806 : /*
807 : * overlap left hand side of extent: move the start, trim the
808 : * length and update the current record.
809 : *
810 : * ltbno ltlen
811 : * Orig: |oooooooooooooooooooo|
812 : * Freeing: |fffffffff|
813 : * Result: |rrrrrrrrrr|
814 : * bno len
815 : */
816 7349350 : ltrec.rm_startblock += len;
817 7349350 : ltrec.rm_blockcount -= len;
818 7349350 : if (!ignore_off)
819 616894 : ltrec.rm_offset += len;
820 7349350 : error = xfs_rmap_update(cur, <rec);
821 7350155 : if (error)
822 0 : goto out_error;
823 21260693 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
824 : /*
825 : * overlap right hand side of extent: trim the length and update
826 : * the current record.
827 : *
828 : * ltbno ltlen
829 : * Orig: |oooooooooooooooooooo|
830 : * Freeing: |fffffffff|
831 : * Result: |rrrrrrrrrr|
832 : * bno len
833 : */
834 3259315 : ltrec.rm_blockcount -= len;
835 3259315 : error = xfs_rmap_update(cur, <rec);
836 3259566 : if (error)
837 0 : goto out_error;
838 : } else {
839 :
840 : /*
841 : * overlap middle of extent: trim the length of the existing
842 : * record to the length of the new left-extent size, increment
843 : * the insertion position so we can insert a new record
844 : * containing the remaining right-extent space.
845 : *
846 : * ltbno ltlen
847 : * Orig: |oooooooooooooooooooo|
848 : * Freeing: |fffffffff|
849 : * Result: |rrrrr| |rrrr|
850 : * bno len
851 : */
852 18001378 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
853 :
854 18001378 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
855 18001378 : error = xfs_rmap_update(cur, <rec);
856 18001378 : if (error)
857 0 : goto out_error;
858 :
859 18001378 : error = xfs_btree_increment(cur, 0, &i);
860 18001384 : if (error)
861 0 : goto out_error;
862 :
863 18001384 : cur->bc_rec.r.rm_startblock = bno + len;
864 18001384 : cur->bc_rec.r.rm_blockcount = orig_len - len -
865 18001384 : ltrec.rm_blockcount;
866 18001384 : cur->bc_rec.r.rm_owner = ltrec.rm_owner;
867 18001384 : if (ignore_off)
868 5012679 : cur->bc_rec.r.rm_offset = 0;
869 : else
870 12988705 : cur->bc_rec.r.rm_offset = offset + len;
871 18001384 : cur->bc_rec.r.rm_flags = flags;
872 18001384 : trace_xfs_rmap_insert(cur, cur->bc_rec.r.rm_startblock,
873 : cur->bc_rec.r.rm_blockcount,
874 : cur->bc_rec.r.rm_owner,
875 : cur->bc_rec.r.rm_offset,
876 : cur->bc_rec.r.rm_flags);
877 18001383 : error = xfs_btree_insert(cur, &i);
878 18001379 : if (error)
879 0 : goto out_error;
880 : }
881 :
882 18001379 : out_done:
883 177437985 : trace_xfs_rmap_unmap_done(cur, bno, len, unwritten, oinfo);
884 177420090 : out_error:
885 177420090 : if (error)
886 96 : trace_xfs_rmap_unmap_error(cur, error, _RET_IP_);
887 177420090 : return error;
888 : }
889 :
890 : #ifdef CONFIG_XFS_LIVE_HOOKS
891 : /*
892 : * Use a static key here to reduce the overhead of rmapbt live updates. If
893 : * the compiler supports jump labels, the static branch will be replaced by a
894 : * nop sled when there are no hook users. Online fsck is currently the only
895 : * caller, so this is a reasonable tradeoff.
896 : *
897 : * Note: Patching the kernel code requires taking the cpu hotplug lock. Other
898 : * parts of the kernel allocate memory with that lock held, which means that
899 : * XFS callers cannot hold any locks that might be used by memory reclaim or
900 : * writeback when calling the static_branch_{inc,dec} functions.
901 : */
902 : DEFINE_STATIC_XFS_HOOK_SWITCH(xfs_rmap_hooks_switch);
903 :
904 : void
905 43783 : xfs_rmap_hook_disable(void)
906 : {
907 43783 : xfs_hooks_switch_off(&xfs_rmap_hooks_switch);
908 43910 : }
909 :
910 : void
911 44092 : xfs_rmap_hook_enable(void)
912 : {
913 44092 : xfs_hooks_switch_on(&xfs_rmap_hooks_switch);
914 44178 : }
915 :
916 : /* Call downstream hooks for a reverse mapping update. */
917 : static inline void
918 849265176 : xfs_rmap_update_hook(
919 : struct xfs_trans *tp,
920 : struct xfs_perag *pag,
921 : struct xfs_rtgroup *rtg,
922 : enum xfs_rmap_intent_type op,
923 : xfs_agblock_t startblock,
924 : xfs_extlen_t blockcount,
925 : bool unwritten,
926 : const struct xfs_owner_info *oinfo)
927 : {
928 856768789 : if (xfs_hooks_switched_on(&xfs_rmap_hooks_switch)) {
929 7503549 : struct xfs_rmap_update_params p = {
930 : .startblock = startblock,
931 : .blockcount = blockcount,
932 : .unwritten = unwritten,
933 : .oinfo = *oinfo, /* struct copy */
934 : };
935 :
936 7503549 : if (pag)
937 2398504 : xfs_hooks_call(&pag->pag_rmap_update_hooks, op, &p);
938 5105045 : else if (rtg)
939 5105045 : xfs_hooks_call(&rtg->rtg_rmap_update_hooks, op, &p);
940 : }
941 849278620 : }
942 :
943 : /* Call the specified function during a reverse mapping update. */
944 : int
945 32107 : xfs_rmap_hook_add(
946 : struct xfs_perag *pag,
947 : struct xfs_rmap_hook *hook)
948 : {
949 32107 : return xfs_hooks_add(&pag->pag_rmap_update_hooks, &hook->update_hook);
950 : }
951 :
952 : /* Stop calling the specified function during a reverse mapping update. */
953 : void
954 32260 : xfs_rmap_hook_del(
955 : struct xfs_perag *pag,
956 : struct xfs_rmap_hook *hook)
957 : {
958 32260 : xfs_hooks_del(&pag->pag_rmap_update_hooks, &hook->update_hook);
959 31913 : }
960 :
961 : # ifdef CONFIG_XFS_RT
962 : /* Call the specified function during a rt reverse mapping update. */
963 : int
964 11143 : xfs_rtrmap_hook_add(
965 : struct xfs_rtgroup *rtg,
966 : struct xfs_rmap_hook *hook)
967 : {
968 11143 : return xfs_hooks_add(&rtg->rtg_rmap_update_hooks, &hook->update_hook);
969 : }
970 :
971 : /* Stop calling the specified function during a rt reverse mapping update. */
972 : void
973 11139 : xfs_rtrmap_hook_del(
974 : struct xfs_rtgroup *rtg,
975 : struct xfs_rmap_hook *hook)
976 : {
977 11139 : xfs_hooks_del(&rtg->rtg_rmap_update_hooks, &hook->update_hook);
978 11135 : }
979 : # endif /* CONFIG_XFS_RT */
980 : #else
981 : # define xfs_rmap_update_hook(t, p, r, o, s, b, u, oi) do { } while(0)
982 : #endif /* CONFIG_XFS_LIVE_HOOKS */
983 :
984 : /*
985 : * Remove a reference to an extent in the rmap btree.
986 : */
987 : int
988 4182773 : xfs_rmap_free(
989 : struct xfs_trans *tp,
990 : struct xfs_buf *agbp,
991 : struct xfs_perag *pag,
992 : xfs_agblock_t bno,
993 : xfs_extlen_t len,
994 : const struct xfs_owner_info *oinfo)
995 : {
996 4182773 : struct xfs_mount *mp = tp->t_mountp;
997 4182773 : struct xfs_btree_cur *cur;
998 4182773 : int error;
999 :
1000 4182773 : if (!xfs_has_rmapbt(mp))
1001 : return 0;
1002 :
1003 4117479 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
1004 4118578 : xfs_rmap_update_hook(tp, pag, NULL, XFS_RMAP_UNMAP, bno, len, false,
1005 : oinfo);
1006 4117752 : error = xfs_rmap_unmap(cur, bno, len, false, oinfo);
1007 :
1008 4117017 : xfs_btree_del_cursor(cur, error);
1009 4117017 : return error;
1010 : }
1011 :
1012 : /*
1013 : * A mergeable rmap must have the same owner and the same values for
1014 : * the unwritten, attr_fork, and bmbt flags. The startblock and
1015 : * offset are checked separately.
1016 : */
1017 : static bool
1018 513968465 : xfs_rmap_is_mergeable(
1019 : struct xfs_rmap_irec *irec,
1020 : uint64_t owner,
1021 : unsigned int flags)
1022 : {
1023 513968465 : if (irec->rm_owner == XFS_RMAP_OWN_NULL)
1024 : return false;
1025 513968465 : if (irec->rm_owner != owner)
1026 : return false;
1027 314780514 : if ((flags & XFS_RMAP_UNWRITTEN) ^
1028 314780514 : (irec->rm_flags & XFS_RMAP_UNWRITTEN))
1029 : return false;
1030 299157756 : if ((flags & XFS_RMAP_ATTR_FORK) ^
1031 : (irec->rm_flags & XFS_RMAP_ATTR_FORK))
1032 : return false;
1033 299148669 : if ((flags & XFS_RMAP_BMBT_BLOCK) ^
1034 : (irec->rm_flags & XFS_RMAP_BMBT_BLOCK))
1035 528372 : return false;
1036 : return true;
1037 : }
1038 :
1039 : /*
1040 : * When we allocate a new block, the first thing we do is add a reference to
1041 : * the extent in the rmap btree. This takes the form of a [agbno, length,
1042 : * owner, offset] record. Flags are encoded in the high bits of the offset
1043 : * field.
1044 : */
1045 : STATIC int
1046 236339133 : xfs_rmap_map(
1047 : struct xfs_btree_cur *cur,
1048 : xfs_agblock_t bno,
1049 : xfs_extlen_t len,
1050 : bool unwritten,
1051 : const struct xfs_owner_info *oinfo)
1052 : {
1053 236339133 : struct xfs_mount *mp = cur->bc_mp;
1054 236339133 : struct xfs_rmap_irec ltrec;
1055 236339133 : struct xfs_rmap_irec gtrec;
1056 236339133 : int have_gt;
1057 236339133 : int have_lt;
1058 236339133 : int error = 0;
1059 236339133 : int i;
1060 236339133 : uint64_t owner;
1061 236339133 : uint64_t offset;
1062 236339133 : unsigned int flags = 0;
1063 236339133 : bool ignore_off;
1064 :
1065 236339133 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1066 236339133 : ASSERT(owner != 0);
1067 236339133 : ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) ||
1068 226179854 : (flags & XFS_RMAP_BMBT_BLOCK);
1069 236339133 : if (unwritten)
1070 80693945 : flags |= XFS_RMAP_UNWRITTEN;
1071 236339133 : trace_xfs_rmap_map(cur, bno, len, unwritten, oinfo);
1072 236332310 : ASSERT(!xfs_rmap_should_skip_owner_update(oinfo));
1073 :
1074 : /*
1075 : * For the initial lookup, look for an exact match or the left-adjacent
1076 : * record for our insertion point. This will also give us the record for
1077 : * start block contiguity tests.
1078 : */
1079 236332310 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec,
1080 : &have_lt);
1081 236337106 : if (error)
1082 899 : goto out_error;
1083 236336207 : if (have_lt) {
1084 236254999 : trace_xfs_rmap_lookup_le_range_result(cur, ltrec.rm_startblock,
1085 : ltrec.rm_blockcount, ltrec.rm_owner,
1086 : ltrec.rm_offset, ltrec.rm_flags);
1087 :
1088 236249004 : if (!xfs_rmap_is_mergeable(<rec, owner, flags))
1089 102054721 : have_lt = 0;
1090 : }
1091 :
1092 236330212 : if (XFS_IS_CORRUPT(mp,
1093 : have_lt != 0 &&
1094 : ltrec.rm_startblock + ltrec.rm_blockcount > bno)) {
1095 0 : xfs_btree_mark_sick(cur);
1096 0 : error = -EFSCORRUPTED;
1097 0 : goto out_error;
1098 : }
1099 :
1100 : /*
1101 : * Increment the cursor to see if we have a right-adjacent record to our
1102 : * insertion point. This will give us the record for end block
1103 : * contiguity tests.
1104 : */
1105 236330212 : error = xfs_btree_increment(cur, 0, &have_gt);
1106 236339662 : if (error)
1107 5 : goto out_error;
1108 236339657 : if (have_gt) {
1109 205989264 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
1110 205990120 : if (error)
1111 0 : goto out_error;
1112 205990120 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
1113 0 : xfs_btree_mark_sick(cur);
1114 0 : error = -EFSCORRUPTED;
1115 0 : goto out_error;
1116 : }
1117 205990120 : if (XFS_IS_CORRUPT(mp, bno + len > gtrec.rm_startblock)) {
1118 0 : xfs_btree_mark_sick(cur);
1119 0 : error = -EFSCORRUPTED;
1120 0 : goto out_error;
1121 : }
1122 205990120 : trace_xfs_rmap_find_right_neighbor_result(cur,
1123 : gtrec.rm_startblock, gtrec.rm_blockcount,
1124 : gtrec.rm_owner, gtrec.rm_offset,
1125 : gtrec.rm_flags);
1126 205987483 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
1127 108299543 : have_gt = 0;
1128 : }
1129 :
1130 : /*
1131 : * Note: cursor currently points one record to the right of ltrec, even
1132 : * if there is no record in the tree to the right.
1133 : */
1134 236337876 : if (have_lt &&
1135 134198934 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
1136 34907522 : (ignore_off || ltrec.rm_offset + ltrec.rm_blockcount == offset)) {
1137 : /*
1138 : * left edge contiguous, merge into left record.
1139 : *
1140 : * ltbno ltlen
1141 : * orig: |ooooooooo|
1142 : * adding: |aaaaaaaaa|
1143 : * result: |rrrrrrrrrrrrrrrrrrr|
1144 : * bno len
1145 : */
1146 29061660 : ltrec.rm_blockcount += len;
1147 29061660 : if (have_gt &&
1148 4470374 : bno + len == gtrec.rm_startblock &&
1149 35738 : (ignore_off || offset + len == gtrec.rm_offset) &&
1150 262658 : (unsigned long)ltrec.rm_blockcount + len +
1151 262658 : gtrec.rm_blockcount <= XFS_RMAP_LEN_MAX) {
1152 : /*
1153 : * right edge also contiguous, delete right record
1154 : * and merge into left record.
1155 : *
1156 : * ltbno ltlen gtbno gtlen
1157 : * orig: |ooooooooo| |ooooooooo|
1158 : * adding: |aaaaaaaaa|
1159 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
1160 : */
1161 262658 : ltrec.rm_blockcount += gtrec.rm_blockcount;
1162 262658 : trace_xfs_rmap_delete(cur, gtrec.rm_startblock,
1163 : gtrec.rm_blockcount, gtrec.rm_owner,
1164 : gtrec.rm_offset, gtrec.rm_flags);
1165 262648 : error = xfs_btree_delete(cur, &i);
1166 262710 : if (error)
1167 0 : goto out_error;
1168 262710 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1169 0 : xfs_btree_mark_sick(cur);
1170 0 : error = -EFSCORRUPTED;
1171 0 : goto out_error;
1172 : }
1173 : }
1174 :
1175 : /* point the cursor back to the left record and update */
1176 29061712 : error = xfs_btree_decrement(cur, 0, &have_gt);
1177 29061786 : if (error)
1178 0 : goto out_error;
1179 29061786 : error = xfs_rmap_update(cur, <rec);
1180 29061737 : if (error)
1181 0 : goto out_error;
1182 207276216 : } else if (have_gt &&
1183 93216772 : bno + len == gtrec.rm_startblock &&
1184 2691644 : (ignore_off || offset + len == gtrec.rm_offset)) {
1185 : /*
1186 : * right edge contiguous, merge into right record.
1187 : *
1188 : * gtbno gtlen
1189 : * Orig: |ooooooooo|
1190 : * adding: |aaaaaaaaa|
1191 : * Result: |rrrrrrrrrrrrrrrrrrr|
1192 : * bno len
1193 : */
1194 297569 : gtrec.rm_startblock = bno;
1195 297569 : gtrec.rm_blockcount += len;
1196 297569 : if (!ignore_off)
1197 32644 : gtrec.rm_offset = offset;
1198 297569 : error = xfs_rmap_update(cur, >rec);
1199 297655 : if (error)
1200 0 : goto out_error;
1201 : } else {
1202 : /*
1203 : * no contiguous edge with identical owner, insert
1204 : * new record at current cursor position.
1205 : */
1206 206978647 : cur->bc_rec.r.rm_startblock = bno;
1207 206978647 : cur->bc_rec.r.rm_blockcount = len;
1208 206978647 : cur->bc_rec.r.rm_owner = owner;
1209 206978647 : cur->bc_rec.r.rm_offset = offset;
1210 206978647 : cur->bc_rec.r.rm_flags = flags;
1211 206978647 : trace_xfs_rmap_insert(cur, bno, len, owner, offset, flags);
1212 206974749 : error = xfs_btree_insert(cur, &i);
1213 206980312 : if (error)
1214 58 : goto out_error;
1215 206980254 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1216 0 : xfs_btree_mark_sick(cur);
1217 0 : error = -EFSCORRUPTED;
1218 0 : goto out_error;
1219 : }
1220 : }
1221 :
1222 236339646 : trace_xfs_rmap_map_done(cur, bno, len, unwritten, oinfo);
1223 236335952 : out_error:
1224 236335952 : if (error)
1225 962 : trace_xfs_rmap_map_error(cur, error, _RET_IP_);
1226 236335952 : return error;
1227 : }
1228 :
1229 : /*
1230 : * Add a reference to an extent in the rmap btree.
1231 : */
1232 : int
1233 8169259 : xfs_rmap_alloc(
1234 : struct xfs_trans *tp,
1235 : struct xfs_buf *agbp,
1236 : struct xfs_perag *pag,
1237 : xfs_agblock_t bno,
1238 : xfs_extlen_t len,
1239 : const struct xfs_owner_info *oinfo)
1240 : {
1241 8169259 : struct xfs_mount *mp = tp->t_mountp;
1242 8169259 : struct xfs_btree_cur *cur;
1243 8169259 : int error;
1244 :
1245 8169259 : if (!xfs_has_rmapbt(mp))
1246 : return 0;
1247 :
1248 7999455 : cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
1249 8000403 : xfs_rmap_update_hook(tp, pag, NULL, XFS_RMAP_MAP, bno, len, false,
1250 : oinfo);
1251 8000007 : error = xfs_rmap_map(cur, bno, len, false, oinfo);
1252 :
1253 7999371 : xfs_btree_del_cursor(cur, error);
1254 7999371 : return error;
1255 : }
1256 :
1257 : #define RMAP_LEFT_CONTIG (1 << 0)
1258 : #define RMAP_RIGHT_CONTIG (1 << 1)
1259 : #define RMAP_LEFT_FILLING (1 << 2)
1260 : #define RMAP_RIGHT_FILLING (1 << 3)
1261 : #define RMAP_LEFT_VALID (1 << 6)
1262 : #define RMAP_RIGHT_VALID (1 << 7)
1263 :
1264 : #define LEFT r[0]
1265 : #define RIGHT r[1]
1266 : #define PREV r[2]
1267 : #define NEW r[3]
1268 :
1269 : /*
1270 : * Convert an unwritten extent to a real extent or vice versa.
1271 : * Does not handle overlapping extents.
1272 : */
1273 : STATIC int
1274 28785597 : xfs_rmap_convert(
1275 : struct xfs_btree_cur *cur,
1276 : xfs_agblock_t bno,
1277 : xfs_extlen_t len,
1278 : bool unwritten,
1279 : const struct xfs_owner_info *oinfo)
1280 : {
1281 28785597 : struct xfs_mount *mp = cur->bc_mp;
1282 28785597 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1283 : /* left is 0, right is 1, */
1284 : /* prev is 2, new is 3 */
1285 28785597 : uint64_t owner;
1286 28785597 : uint64_t offset;
1287 28785597 : uint64_t new_endoff;
1288 28785597 : unsigned int oldext;
1289 28785597 : unsigned int newext;
1290 28785597 : unsigned int flags = 0;
1291 28785597 : int i;
1292 28785597 : int state = 0;
1293 28785597 : int error;
1294 :
1295 28785597 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1296 28785597 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1297 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1298 28785597 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1299 28785597 : new_endoff = offset + len;
1300 28785597 : trace_xfs_rmap_convert(cur, bno, len, unwritten, oinfo);
1301 :
1302 : /*
1303 : * For the initial lookup, look for an exact match or the left-adjacent
1304 : * record for our insertion point. This will also give us the record for
1305 : * start block contiguity tests.
1306 : */
1307 28785481 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, &PREV, &i);
1308 28785065 : if (error)
1309 1 : goto done;
1310 28785064 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1311 0 : xfs_btree_mark_sick(cur);
1312 0 : error = -EFSCORRUPTED;
1313 0 : goto done;
1314 : }
1315 :
1316 28785064 : trace_xfs_rmap_lookup_le_range_result(cur, PREV.rm_startblock,
1317 : PREV.rm_blockcount, PREV.rm_owner, PREV.rm_offset,
1318 : PREV.rm_flags);
1319 :
1320 28784904 : ASSERT(PREV.rm_offset <= offset);
1321 28784904 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1322 28784904 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1323 28784904 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1324 :
1325 : /*
1326 : * Set flags determining what part of the previous oldext allocation
1327 : * extent is being replaced by a newext allocation.
1328 : */
1329 28784904 : if (PREV.rm_offset == offset)
1330 24524459 : state |= RMAP_LEFT_FILLING;
1331 28784904 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1332 23327130 : state |= RMAP_RIGHT_FILLING;
1333 :
1334 : /*
1335 : * Decrement the cursor to see if we have a left-adjacent record to our
1336 : * insertion point. This will give us the record for end block
1337 : * contiguity tests.
1338 : */
1339 28784904 : error = xfs_btree_decrement(cur, 0, &i);
1340 28785420 : if (error)
1341 0 : goto done;
1342 28785420 : if (i) {
1343 28785375 : state |= RMAP_LEFT_VALID;
1344 28785375 : error = xfs_rmap_get_rec(cur, &LEFT, &i);
1345 28785577 : if (error)
1346 0 : goto done;
1347 28785577 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1348 0 : xfs_btree_mark_sick(cur);
1349 0 : error = -EFSCORRUPTED;
1350 0 : goto done;
1351 : }
1352 28785577 : if (XFS_IS_CORRUPT(mp,
1353 : LEFT.rm_startblock + LEFT.rm_blockcount >
1354 : bno)) {
1355 0 : xfs_btree_mark_sick(cur);
1356 0 : error = -EFSCORRUPTED;
1357 0 : goto done;
1358 : }
1359 28785577 : trace_xfs_rmap_find_left_neighbor_result(cur,
1360 : LEFT.rm_startblock, LEFT.rm_blockcount,
1361 : LEFT.rm_owner, LEFT.rm_offset, LEFT.rm_flags);
1362 28785061 : if (LEFT.rm_startblock + LEFT.rm_blockcount == bno &&
1363 20703915 : LEFT.rm_offset + LEFT.rm_blockcount == offset &&
1364 2035531 : xfs_rmap_is_mergeable(&LEFT, owner, newext))
1365 2014318 : state |= RMAP_LEFT_CONTIG;
1366 : }
1367 :
1368 : /*
1369 : * Increment the cursor to see if we have a right-adjacent record to our
1370 : * insertion point. This will give us the record for end block
1371 : * contiguity tests.
1372 : */
1373 28785106 : error = xfs_btree_increment(cur, 0, &i);
1374 28785622 : if (error)
1375 0 : goto done;
1376 28785622 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1377 0 : xfs_btree_mark_sick(cur);
1378 0 : error = -EFSCORRUPTED;
1379 0 : goto done;
1380 : }
1381 28785622 : error = xfs_btree_increment(cur, 0, &i);
1382 28785609 : if (error)
1383 0 : goto done;
1384 28785609 : if (i) {
1385 26201613 : state |= RMAP_RIGHT_VALID;
1386 26201613 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1387 26201819 : if (error)
1388 0 : goto done;
1389 26201819 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1390 0 : xfs_btree_mark_sick(cur);
1391 0 : error = -EFSCORRUPTED;
1392 0 : goto done;
1393 : }
1394 26201819 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1395 0 : xfs_btree_mark_sick(cur);
1396 0 : error = -EFSCORRUPTED;
1397 0 : goto done;
1398 : }
1399 26201819 : trace_xfs_rmap_find_right_neighbor_result(cur,
1400 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1401 : RIGHT.rm_owner, RIGHT.rm_offset,
1402 : RIGHT.rm_flags);
1403 26201496 : if (bno + len == RIGHT.rm_startblock &&
1404 15655056 : offset + len == RIGHT.rm_offset &&
1405 1216628 : xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1406 1201150 : state |= RMAP_RIGHT_CONTIG;
1407 : }
1408 :
1409 : /* check that left + prev + right is not too long */
1410 28785492 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1411 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1412 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1413 304873 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1414 304873 : (unsigned long)LEFT.rm_blockcount + len +
1415 304873 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1416 0 : state &= ~RMAP_RIGHT_CONTIG;
1417 :
1418 28785492 : trace_xfs_rmap_convert_state(cur, state, _RET_IP_);
1419 :
1420 : /* reset the cursor back to PREV */
1421 28785068 : error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, NULL, &i);
1422 28785410 : if (error)
1423 0 : goto done;
1424 28785410 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1425 0 : xfs_btree_mark_sick(cur);
1426 0 : error = -EFSCORRUPTED;
1427 0 : goto done;
1428 : }
1429 :
1430 : /*
1431 : * Switch out based on the FILLING and CONTIG state bits.
1432 : */
1433 28785410 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1434 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1435 304873 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1436 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1437 : /*
1438 : * Setting all of a previous oldext extent to newext.
1439 : * The left and right neighbors are both contiguous with new.
1440 : */
1441 304873 : error = xfs_btree_increment(cur, 0, &i);
1442 304873 : if (error)
1443 0 : goto done;
1444 304873 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1445 0 : xfs_btree_mark_sick(cur);
1446 0 : error = -EFSCORRUPTED;
1447 0 : goto done;
1448 : }
1449 304873 : trace_xfs_rmap_delete(cur, RIGHT.rm_startblock,
1450 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1451 : RIGHT.rm_offset, RIGHT.rm_flags);
1452 304873 : error = xfs_btree_delete(cur, &i);
1453 304873 : if (error)
1454 0 : goto done;
1455 304873 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1456 0 : xfs_btree_mark_sick(cur);
1457 0 : error = -EFSCORRUPTED;
1458 0 : goto done;
1459 : }
1460 304873 : error = xfs_btree_decrement(cur, 0, &i);
1461 304873 : if (error)
1462 0 : goto done;
1463 304873 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1464 0 : xfs_btree_mark_sick(cur);
1465 0 : error = -EFSCORRUPTED;
1466 0 : goto done;
1467 : }
1468 304873 : trace_xfs_rmap_delete(cur, PREV.rm_startblock,
1469 : PREV.rm_blockcount, PREV.rm_owner,
1470 : PREV.rm_offset, PREV.rm_flags);
1471 304873 : error = xfs_btree_delete(cur, &i);
1472 304873 : if (error)
1473 0 : goto done;
1474 304873 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1475 0 : xfs_btree_mark_sick(cur);
1476 0 : error = -EFSCORRUPTED;
1477 0 : goto done;
1478 : }
1479 304873 : error = xfs_btree_decrement(cur, 0, &i);
1480 304873 : if (error)
1481 0 : goto done;
1482 304873 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1483 0 : xfs_btree_mark_sick(cur);
1484 0 : error = -EFSCORRUPTED;
1485 0 : goto done;
1486 : }
1487 304873 : NEW = LEFT;
1488 304873 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1489 304873 : error = xfs_rmap_update(cur, &NEW);
1490 304873 : if (error)
1491 0 : goto done;
1492 : break;
1493 :
1494 1137366 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1495 : /*
1496 : * Setting all of a previous oldext extent to newext.
1497 : * The left neighbor is contiguous, the right is not.
1498 : */
1499 1137366 : trace_xfs_rmap_delete(cur, PREV.rm_startblock,
1500 : PREV.rm_blockcount, PREV.rm_owner,
1501 : PREV.rm_offset, PREV.rm_flags);
1502 1137368 : error = xfs_btree_delete(cur, &i);
1503 1137368 : if (error)
1504 0 : goto done;
1505 1137368 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1506 0 : xfs_btree_mark_sick(cur);
1507 0 : error = -EFSCORRUPTED;
1508 0 : goto done;
1509 : }
1510 1137368 : error = xfs_btree_decrement(cur, 0, &i);
1511 1137369 : if (error)
1512 0 : goto done;
1513 1137369 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1514 0 : xfs_btree_mark_sick(cur);
1515 0 : error = -EFSCORRUPTED;
1516 0 : goto done;
1517 : }
1518 1137369 : NEW = LEFT;
1519 1137369 : NEW.rm_blockcount += PREV.rm_blockcount;
1520 1137369 : error = xfs_rmap_update(cur, &NEW);
1521 1137368 : if (error)
1522 0 : goto done;
1523 : break;
1524 :
1525 543722 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1526 : /*
1527 : * Setting all of a previous oldext extent to newext.
1528 : * The right neighbor is contiguous, the left is not.
1529 : */
1530 543722 : error = xfs_btree_increment(cur, 0, &i);
1531 543722 : if (error)
1532 0 : goto done;
1533 543722 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1534 0 : xfs_btree_mark_sick(cur);
1535 0 : error = -EFSCORRUPTED;
1536 0 : goto done;
1537 : }
1538 543722 : trace_xfs_rmap_delete(cur, RIGHT.rm_startblock,
1539 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1540 : RIGHT.rm_offset, RIGHT.rm_flags);
1541 543720 : error = xfs_btree_delete(cur, &i);
1542 543714 : if (error)
1543 0 : goto done;
1544 543714 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1545 0 : xfs_btree_mark_sick(cur);
1546 0 : error = -EFSCORRUPTED;
1547 0 : goto done;
1548 : }
1549 543714 : error = xfs_btree_decrement(cur, 0, &i);
1550 543709 : if (error)
1551 0 : goto done;
1552 543709 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1553 0 : xfs_btree_mark_sick(cur);
1554 0 : error = -EFSCORRUPTED;
1555 0 : goto done;
1556 : }
1557 543709 : NEW = PREV;
1558 543709 : NEW.rm_blockcount = len + RIGHT.rm_blockcount;
1559 543709 : NEW.rm_flags = newext;
1560 543709 : error = xfs_rmap_update(cur, &NEW);
1561 543726 : if (error)
1562 0 : goto done;
1563 : break;
1564 :
1565 20631127 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1566 : /*
1567 : * Setting all of a previous oldext extent to newext.
1568 : * Neither the left nor right neighbors are contiguous with
1569 : * the new one.
1570 : */
1571 20631127 : NEW = PREV;
1572 20631127 : NEW.rm_flags = newext;
1573 20631127 : error = xfs_rmap_update(cur, &NEW);
1574 20631161 : if (error)
1575 0 : goto done;
1576 : break;
1577 :
1578 572077 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
1579 : /*
1580 : * Setting the first part of a previous oldext extent to newext.
1581 : * The left neighbor is contiguous.
1582 : */
1583 572077 : NEW = PREV;
1584 572077 : NEW.rm_offset += len;
1585 572077 : NEW.rm_startblock += len;
1586 572077 : NEW.rm_blockcount -= len;
1587 572077 : error = xfs_rmap_update(cur, &NEW);
1588 572077 : if (error)
1589 0 : goto done;
1590 572077 : error = xfs_btree_decrement(cur, 0, &i);
1591 572077 : if (error)
1592 0 : goto done;
1593 572077 : NEW = LEFT;
1594 572077 : NEW.rm_blockcount += len;
1595 572077 : error = xfs_rmap_update(cur, &NEW);
1596 572077 : if (error)
1597 0 : goto done;
1598 : break;
1599 :
1600 1335722 : case RMAP_LEFT_FILLING:
1601 : /*
1602 : * Setting the first part of a previous oldext extent to newext.
1603 : * The left neighbor is not contiguous.
1604 : */
1605 1335722 : NEW = PREV;
1606 1335722 : NEW.rm_startblock += len;
1607 1335722 : NEW.rm_offset += len;
1608 1335722 : NEW.rm_blockcount -= len;
1609 1335722 : error = xfs_rmap_update(cur, &NEW);
1610 1335722 : if (error)
1611 0 : goto done;
1612 1335722 : NEW.rm_startblock = bno;
1613 1335722 : NEW.rm_owner = owner;
1614 1335722 : NEW.rm_offset = offset;
1615 1335722 : NEW.rm_blockcount = len;
1616 1335722 : NEW.rm_flags = newext;
1617 1335722 : cur->bc_rec.r = NEW;
1618 1335722 : trace_xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1619 1335720 : error = xfs_btree_insert(cur, &i);
1620 1335719 : if (error)
1621 0 : goto done;
1622 1335719 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1623 0 : xfs_btree_mark_sick(cur);
1624 0 : error = -EFSCORRUPTED;
1625 0 : goto done;
1626 : }
1627 : break;
1628 :
1629 352562 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1630 : /*
1631 : * Setting the last part of a previous oldext extent to newext.
1632 : * The right neighbor is contiguous with the new allocation.
1633 : */
1634 352562 : NEW = PREV;
1635 352562 : NEW.rm_blockcount -= len;
1636 352562 : error = xfs_rmap_update(cur, &NEW);
1637 352561 : if (error)
1638 0 : goto done;
1639 352561 : error = xfs_btree_increment(cur, 0, &i);
1640 352561 : if (error)
1641 0 : goto done;
1642 352561 : NEW = RIGHT;
1643 352561 : NEW.rm_offset = offset;
1644 352561 : NEW.rm_startblock = bno;
1645 352561 : NEW.rm_blockcount += len;
1646 352561 : error = xfs_rmap_update(cur, &NEW);
1647 352562 : if (error)
1648 0 : goto done;
1649 : break;
1650 :
1651 357899 : case RMAP_RIGHT_FILLING:
1652 : /*
1653 : * Setting the last part of a previous oldext extent to newext.
1654 : * The right neighbor is not contiguous.
1655 : */
1656 357899 : NEW = PREV;
1657 357899 : NEW.rm_blockcount -= len;
1658 357899 : error = xfs_rmap_update(cur, &NEW);
1659 357900 : if (error)
1660 0 : goto done;
1661 357900 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1662 : oldext, &i);
1663 357899 : if (error)
1664 0 : goto done;
1665 357899 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1666 0 : xfs_btree_mark_sick(cur);
1667 0 : error = -EFSCORRUPTED;
1668 0 : goto done;
1669 : }
1670 357899 : NEW.rm_startblock = bno;
1671 357899 : NEW.rm_owner = owner;
1672 357899 : NEW.rm_offset = offset;
1673 357899 : NEW.rm_blockcount = len;
1674 357899 : NEW.rm_flags = newext;
1675 357899 : cur->bc_rec.r = NEW;
1676 357899 : trace_xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1677 357899 : error = xfs_btree_insert(cur, &i);
1678 357901 : if (error)
1679 0 : goto done;
1680 357901 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1681 0 : xfs_btree_mark_sick(cur);
1682 0 : error = -EFSCORRUPTED;
1683 0 : goto done;
1684 : }
1685 : break;
1686 :
1687 3550062 : case 0:
1688 : /*
1689 : * Setting the middle part of a previous oldext extent to
1690 : * newext. Contiguity is impossible here.
1691 : * One extent becomes three extents.
1692 : */
1693 : /* new right extent - oldext */
1694 3550062 : NEW.rm_startblock = bno + len;
1695 3550062 : NEW.rm_owner = owner;
1696 3550062 : NEW.rm_offset = new_endoff;
1697 3550062 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
1698 : new_endoff;
1699 3550062 : NEW.rm_flags = PREV.rm_flags;
1700 3550062 : error = xfs_rmap_update(cur, &NEW);
1701 3550099 : if (error)
1702 0 : goto done;
1703 : /* new left extent - oldext */
1704 3550099 : NEW = PREV;
1705 3550099 : NEW.rm_blockcount = offset - PREV.rm_offset;
1706 3550099 : cur->bc_rec.r = NEW;
1707 3550099 : trace_xfs_rmap_insert(cur, NEW.rm_startblock,
1708 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
1709 : NEW.rm_flags);
1710 3550092 : error = xfs_btree_insert(cur, &i);
1711 3550111 : if (error)
1712 0 : goto done;
1713 3550111 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1714 0 : xfs_btree_mark_sick(cur);
1715 0 : error = -EFSCORRUPTED;
1716 0 : goto done;
1717 : }
1718 : /*
1719 : * Reset the cursor to the position of the new extent
1720 : * we are about to insert as we can't trust it after
1721 : * the previous insert.
1722 : */
1723 3550111 : error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset,
1724 : oldext, &i);
1725 3550111 : if (error)
1726 0 : goto done;
1727 3550111 : if (XFS_IS_CORRUPT(mp, i != 0)) {
1728 0 : xfs_btree_mark_sick(cur);
1729 0 : error = -EFSCORRUPTED;
1730 0 : goto done;
1731 : }
1732 : /* new middle extent - newext */
1733 3550111 : cur->bc_rec.r.rm_flags &= ~XFS_RMAP_UNWRITTEN;
1734 3550111 : cur->bc_rec.r.rm_flags |= newext;
1735 3550111 : trace_xfs_rmap_insert(cur, bno, len, owner, offset, newext);
1736 3550106 : error = xfs_btree_insert(cur, &i);
1737 3550113 : if (error)
1738 0 : goto done;
1739 3550113 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1740 0 : xfs_btree_mark_sick(cur);
1741 0 : error = -EFSCORRUPTED;
1742 0 : goto done;
1743 : }
1744 : break;
1745 :
1746 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1747 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1748 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
1749 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1750 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
1751 : case RMAP_LEFT_CONTIG:
1752 : case RMAP_RIGHT_CONTIG:
1753 : /*
1754 : * These cases are all impossible.
1755 : */
1756 0 : ASSERT(0);
1757 : }
1758 :
1759 28785500 : trace_xfs_rmap_convert_done(cur, bno, len, unwritten, oinfo);
1760 28785174 : done:
1761 28785174 : if (error)
1762 1 : trace_xfs_rmap_convert_error(cur, error, _RET_IP_);
1763 28785174 : return error;
1764 : }
1765 :
1766 : /*
1767 : * Convert an unwritten extent to a real extent or vice versa. If there is no
1768 : * possibility of overlapping extents, delegate to the simpler convert
1769 : * function.
1770 : */
1771 : STATIC int
1772 45718707 : xfs_rmap_convert_shared(
1773 : struct xfs_btree_cur *cur,
1774 : xfs_agblock_t bno,
1775 : xfs_extlen_t len,
1776 : bool unwritten,
1777 : const struct xfs_owner_info *oinfo)
1778 : {
1779 45718707 : struct xfs_mount *mp = cur->bc_mp;
1780 45718707 : struct xfs_rmap_irec r[4]; /* neighbor extent entries */
1781 : /* left is 0, right is 1, */
1782 : /* prev is 2, new is 3 */
1783 45718707 : uint64_t owner;
1784 45718707 : uint64_t offset;
1785 45718707 : uint64_t new_endoff;
1786 45718707 : unsigned int oldext;
1787 45718707 : unsigned int newext;
1788 45718707 : unsigned int flags = 0;
1789 45718707 : int i;
1790 45718707 : int state = 0;
1791 45718707 : int error;
1792 :
1793 45718707 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
1794 45718707 : ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) ||
1795 : (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))));
1796 45718707 : oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0;
1797 45718707 : new_endoff = offset + len;
1798 45718707 : trace_xfs_rmap_convert(cur, bno, len, unwritten, oinfo);
1799 :
1800 : /*
1801 : * For the initial lookup, look for and exact match or the left-adjacent
1802 : * record for our insertion point. This will also give us the record for
1803 : * start block contiguity tests.
1804 : */
1805 45718677 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, oldext,
1806 : &PREV, &i);
1807 45718658 : if (error)
1808 2 : goto done;
1809 45718656 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1810 0 : xfs_btree_mark_sick(cur);
1811 0 : error = -EFSCORRUPTED;
1812 0 : goto done;
1813 : }
1814 :
1815 45718656 : ASSERT(PREV.rm_offset <= offset);
1816 45718656 : ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff);
1817 45718656 : ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext);
1818 45718656 : newext = ~oldext & XFS_RMAP_UNWRITTEN;
1819 :
1820 : /*
1821 : * Set flags determining what part of the previous oldext allocation
1822 : * extent is being replaced by a newext allocation.
1823 : */
1824 45718656 : if (PREV.rm_offset == offset)
1825 41065787 : state |= RMAP_LEFT_FILLING;
1826 45718656 : if (PREV.rm_offset + PREV.rm_blockcount == new_endoff)
1827 37367376 : state |= RMAP_RIGHT_FILLING;
1828 :
1829 : /* Is there a left record that abuts our range? */
1830 45718656 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, newext,
1831 : &LEFT, &i);
1832 45718733 : if (error)
1833 0 : goto done;
1834 45718733 : if (i) {
1835 2246836 : state |= RMAP_LEFT_VALID;
1836 2246836 : if (XFS_IS_CORRUPT(mp,
1837 : LEFT.rm_startblock + LEFT.rm_blockcount >
1838 : bno)) {
1839 0 : xfs_btree_mark_sick(cur);
1840 0 : error = -EFSCORRUPTED;
1841 0 : goto done;
1842 : }
1843 2246836 : if (xfs_rmap_is_mergeable(&LEFT, owner, newext))
1844 2246836 : state |= RMAP_LEFT_CONTIG;
1845 : }
1846 :
1847 : /* Is there a right record that abuts our range? */
1848 45718733 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
1849 : newext, &i);
1850 45718732 : if (error)
1851 0 : goto done;
1852 45718732 : if (i) {
1853 1862789 : state |= RMAP_RIGHT_VALID;
1854 1862789 : error = xfs_rmap_get_rec(cur, &RIGHT, &i);
1855 1862790 : if (error)
1856 0 : goto done;
1857 1862790 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1858 0 : xfs_btree_mark_sick(cur);
1859 0 : error = -EFSCORRUPTED;
1860 0 : goto done;
1861 : }
1862 1862790 : if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) {
1863 0 : xfs_btree_mark_sick(cur);
1864 0 : error = -EFSCORRUPTED;
1865 0 : goto done;
1866 : }
1867 1862790 : trace_xfs_rmap_find_right_neighbor_result(cur,
1868 : RIGHT.rm_startblock, RIGHT.rm_blockcount,
1869 : RIGHT.rm_owner, RIGHT.rm_offset,
1870 : RIGHT.rm_flags);
1871 1862788 : if (xfs_rmap_is_mergeable(&RIGHT, owner, newext))
1872 1862787 : state |= RMAP_RIGHT_CONTIG;
1873 : }
1874 :
1875 : /* check that left + prev + right is not too long */
1876 45718731 : if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1877 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) ==
1878 : (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1879 382529 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) &&
1880 382529 : (unsigned long)LEFT.rm_blockcount + len +
1881 382529 : RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX)
1882 0 : state &= ~RMAP_RIGHT_CONTIG;
1883 :
1884 45718731 : trace_xfs_rmap_convert_state(cur, state, _RET_IP_);
1885 : /*
1886 : * Switch out based on the FILLING and CONTIG state bits.
1887 : */
1888 45718731 : switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1889 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) {
1890 382529 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG |
1891 : RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1892 : /*
1893 : * Setting all of a previous oldext extent to newext.
1894 : * The left and right neighbors are both contiguous with new.
1895 : */
1896 382529 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1897 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1898 : RIGHT.rm_offset, RIGHT.rm_flags);
1899 382529 : if (error)
1900 0 : goto done;
1901 382529 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1902 : PREV.rm_blockcount, PREV.rm_owner,
1903 : PREV.rm_offset, PREV.rm_flags);
1904 382529 : if (error)
1905 0 : goto done;
1906 382529 : NEW = LEFT;
1907 382529 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1908 : NEW.rm_blockcount, NEW.rm_owner,
1909 : NEW.rm_offset, NEW.rm_flags, &i);
1910 382529 : if (error)
1911 0 : goto done;
1912 382529 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1913 0 : xfs_btree_mark_sick(cur);
1914 0 : error = -EFSCORRUPTED;
1915 0 : goto done;
1916 : }
1917 382529 : NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount;
1918 382529 : error = xfs_rmap_update(cur, &NEW);
1919 382529 : if (error)
1920 0 : goto done;
1921 : break;
1922 :
1923 1441124 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
1924 : /*
1925 : * Setting all of a previous oldext extent to newext.
1926 : * The left neighbor is contiguous, the right is not.
1927 : */
1928 1441124 : error = xfs_rmap_delete(cur, PREV.rm_startblock,
1929 : PREV.rm_blockcount, PREV.rm_owner,
1930 : PREV.rm_offset, PREV.rm_flags);
1931 1441124 : if (error)
1932 0 : goto done;
1933 1441124 : NEW = LEFT;
1934 1441124 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1935 : NEW.rm_blockcount, NEW.rm_owner,
1936 : NEW.rm_offset, NEW.rm_flags, &i);
1937 1441126 : if (error)
1938 0 : goto done;
1939 1441126 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1940 0 : xfs_btree_mark_sick(cur);
1941 0 : error = -EFSCORRUPTED;
1942 0 : goto done;
1943 : }
1944 1441126 : NEW.rm_blockcount += PREV.rm_blockcount;
1945 1441126 : error = xfs_rmap_update(cur, &NEW);
1946 1441126 : if (error)
1947 0 : goto done;
1948 : break;
1949 :
1950 846641 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
1951 : /*
1952 : * Setting all of a previous oldext extent to newext.
1953 : * The right neighbor is contiguous, the left is not.
1954 : */
1955 846641 : error = xfs_rmap_delete(cur, RIGHT.rm_startblock,
1956 : RIGHT.rm_blockcount, RIGHT.rm_owner,
1957 : RIGHT.rm_offset, RIGHT.rm_flags);
1958 846640 : if (error)
1959 0 : goto done;
1960 846640 : NEW = PREV;
1961 846640 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1962 : NEW.rm_blockcount, NEW.rm_owner,
1963 : NEW.rm_offset, NEW.rm_flags, &i);
1964 846641 : if (error)
1965 0 : goto done;
1966 846641 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1967 0 : xfs_btree_mark_sick(cur);
1968 0 : error = -EFSCORRUPTED;
1969 0 : goto done;
1970 : }
1971 846641 : NEW.rm_blockcount += RIGHT.rm_blockcount;
1972 846641 : NEW.rm_flags = RIGHT.rm_flags;
1973 846641 : error = xfs_rmap_update(cur, &NEW);
1974 846640 : if (error)
1975 0 : goto done;
1976 : break;
1977 :
1978 32325621 : case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING:
1979 : /*
1980 : * Setting all of a previous oldext extent to newext.
1981 : * Neither the left nor right neighbors are contiguous with
1982 : * the new one.
1983 : */
1984 32325621 : NEW = PREV;
1985 32325621 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
1986 : NEW.rm_blockcount, NEW.rm_owner,
1987 : NEW.rm_offset, NEW.rm_flags, &i);
1988 32325623 : if (error)
1989 0 : goto done;
1990 32325623 : if (XFS_IS_CORRUPT(mp, i != 1)) {
1991 0 : xfs_btree_mark_sick(cur);
1992 0 : error = -EFSCORRUPTED;
1993 0 : goto done;
1994 : }
1995 32325623 : NEW.rm_flags = newext;
1996 32325623 : error = xfs_rmap_update(cur, &NEW);
1997 32325620 : if (error)
1998 0 : goto done;
1999 : break;
2000 :
2001 423182 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG:
2002 : /*
2003 : * Setting the first part of a previous oldext extent to newext.
2004 : * The left neighbor is contiguous.
2005 : */
2006 423182 : NEW = PREV;
2007 423182 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
2008 : NEW.rm_blockcount, NEW.rm_owner,
2009 : NEW.rm_offset, NEW.rm_flags);
2010 423182 : if (error)
2011 0 : goto done;
2012 423182 : NEW.rm_offset += len;
2013 423182 : NEW.rm_startblock += len;
2014 423182 : NEW.rm_blockcount -= len;
2015 423182 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2016 : NEW.rm_blockcount, NEW.rm_owner,
2017 : NEW.rm_offset, NEW.rm_flags);
2018 423182 : if (error)
2019 0 : goto done;
2020 423182 : NEW = LEFT;
2021 423182 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2022 : NEW.rm_blockcount, NEW.rm_owner,
2023 : NEW.rm_offset, NEW.rm_flags, &i);
2024 423182 : if (error)
2025 0 : goto done;
2026 423182 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2027 0 : xfs_btree_mark_sick(cur);
2028 0 : error = -EFSCORRUPTED;
2029 0 : goto done;
2030 : }
2031 423182 : NEW.rm_blockcount += len;
2032 423182 : error = xfs_rmap_update(cur, &NEW);
2033 423182 : if (error)
2034 0 : goto done;
2035 : break;
2036 :
2037 5646740 : case RMAP_LEFT_FILLING:
2038 : /*
2039 : * Setting the first part of a previous oldext extent to newext.
2040 : * The left neighbor is not contiguous.
2041 : */
2042 5646740 : NEW = PREV;
2043 5646740 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
2044 : NEW.rm_blockcount, NEW.rm_owner,
2045 : NEW.rm_offset, NEW.rm_flags);
2046 5646740 : if (error)
2047 0 : goto done;
2048 5646740 : NEW.rm_offset += len;
2049 5646740 : NEW.rm_startblock += len;
2050 5646740 : NEW.rm_blockcount -= len;
2051 5646740 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2052 : NEW.rm_blockcount, NEW.rm_owner,
2053 : NEW.rm_offset, NEW.rm_flags);
2054 5646740 : if (error)
2055 0 : goto done;
2056 5646740 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
2057 5646740 : if (error)
2058 0 : goto done;
2059 : break;
2060 :
2061 633620 : case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG:
2062 : /*
2063 : * Setting the last part of a previous oldext extent to newext.
2064 : * The right neighbor is contiguous with the new allocation.
2065 : */
2066 633620 : NEW = PREV;
2067 633620 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2068 : NEW.rm_blockcount, NEW.rm_owner,
2069 : NEW.rm_offset, NEW.rm_flags, &i);
2070 633619 : if (error)
2071 0 : goto done;
2072 633619 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2073 0 : xfs_btree_mark_sick(cur);
2074 0 : error = -EFSCORRUPTED;
2075 0 : goto done;
2076 : }
2077 633619 : NEW.rm_blockcount = offset - NEW.rm_offset;
2078 633619 : error = xfs_rmap_update(cur, &NEW);
2079 633619 : if (error)
2080 0 : goto done;
2081 633619 : NEW = RIGHT;
2082 633619 : error = xfs_rmap_delete(cur, NEW.rm_startblock,
2083 : NEW.rm_blockcount, NEW.rm_owner,
2084 : NEW.rm_offset, NEW.rm_flags);
2085 633620 : if (error)
2086 0 : goto done;
2087 633620 : NEW.rm_offset = offset;
2088 633620 : NEW.rm_startblock = bno;
2089 633620 : NEW.rm_blockcount += len;
2090 633620 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2091 : NEW.rm_blockcount, NEW.rm_owner,
2092 : NEW.rm_offset, NEW.rm_flags);
2093 633619 : if (error)
2094 0 : goto done;
2095 : break;
2096 :
2097 1737893 : case RMAP_RIGHT_FILLING:
2098 : /*
2099 : * Setting the last part of a previous oldext extent to newext.
2100 : * The right neighbor is not contiguous.
2101 : */
2102 1737893 : NEW = PREV;
2103 1737893 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2104 : NEW.rm_blockcount, NEW.rm_owner,
2105 : NEW.rm_offset, NEW.rm_flags, &i);
2106 1737893 : if (error)
2107 0 : goto done;
2108 1737893 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2109 0 : xfs_btree_mark_sick(cur);
2110 0 : error = -EFSCORRUPTED;
2111 0 : goto done;
2112 : }
2113 1737893 : NEW.rm_blockcount -= len;
2114 1737893 : error = xfs_rmap_update(cur, &NEW);
2115 1737893 : if (error)
2116 0 : goto done;
2117 1737893 : error = xfs_rmap_insert(cur, bno, len, owner, offset, newext);
2118 1737893 : if (error)
2119 0 : goto done;
2120 : break;
2121 :
2122 2281381 : case 0:
2123 : /*
2124 : * Setting the middle part of a previous oldext extent to
2125 : * newext. Contiguity is impossible here.
2126 : * One extent becomes three extents.
2127 : */
2128 : /* new right extent - oldext */
2129 2281381 : NEW.rm_startblock = bno + len;
2130 2281381 : NEW.rm_owner = owner;
2131 2281381 : NEW.rm_offset = new_endoff;
2132 2281381 : NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount -
2133 : new_endoff;
2134 2281381 : NEW.rm_flags = PREV.rm_flags;
2135 2281381 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2136 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
2137 : NEW.rm_flags);
2138 2281381 : if (error)
2139 0 : goto done;
2140 : /* new left extent - oldext */
2141 2281381 : NEW = PREV;
2142 2281381 : error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock,
2143 : NEW.rm_blockcount, NEW.rm_owner,
2144 : NEW.rm_offset, NEW.rm_flags, &i);
2145 2281382 : if (error)
2146 0 : goto done;
2147 2281382 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2148 0 : xfs_btree_mark_sick(cur);
2149 0 : error = -EFSCORRUPTED;
2150 0 : goto done;
2151 : }
2152 2281382 : NEW.rm_blockcount = offset - NEW.rm_offset;
2153 2281382 : error = xfs_rmap_update(cur, &NEW);
2154 2281382 : if (error)
2155 0 : goto done;
2156 : /* new middle extent - newext */
2157 2281382 : NEW.rm_startblock = bno;
2158 2281382 : NEW.rm_blockcount = len;
2159 2281382 : NEW.rm_owner = owner;
2160 2281382 : NEW.rm_offset = offset;
2161 2281382 : NEW.rm_flags = newext;
2162 2281382 : error = xfs_rmap_insert(cur, NEW.rm_startblock,
2163 : NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset,
2164 : NEW.rm_flags);
2165 2281382 : if (error)
2166 0 : goto done;
2167 : break;
2168 :
2169 0 : case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2170 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2171 : case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG:
2172 : case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG:
2173 : case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG:
2174 : case RMAP_LEFT_CONTIG:
2175 : case RMAP_RIGHT_CONTIG:
2176 : /*
2177 : * These cases are all impossible.
2178 : */
2179 0 : ASSERT(0);
2180 : }
2181 :
2182 45718731 : trace_xfs_rmap_convert_done(cur, bno, len, unwritten, oinfo);
2183 45718697 : done:
2184 45718697 : if (error)
2185 2 : trace_xfs_rmap_convert_error(cur, error, _RET_IP_);
2186 45718697 : return error;
2187 : }
2188 :
2189 : #undef NEW
2190 : #undef LEFT
2191 : #undef RIGHT
2192 : #undef PREV
2193 :
2194 : /*
2195 : * Find an extent in the rmap btree and unmap it. For rmap extent types that
2196 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2197 : * that the prev/next records in the btree might belong to another owner.
2198 : * Therefore we must use delete+insert to alter any of the key fields.
2199 : *
2200 : * For every other situation there can only be one owner for a given extent,
2201 : * so we can call the regular _free function.
2202 : */
2203 : STATIC int
2204 157624186 : xfs_rmap_unmap_shared(
2205 : struct xfs_btree_cur *cur,
2206 : xfs_agblock_t bno,
2207 : xfs_extlen_t len,
2208 : bool unwritten,
2209 : const struct xfs_owner_info *oinfo)
2210 : {
2211 157624186 : struct xfs_mount *mp = cur->bc_mp;
2212 157624186 : struct xfs_rmap_irec ltrec;
2213 157624186 : uint64_t ltoff;
2214 157624186 : int error = 0;
2215 157624186 : int i;
2216 157624186 : uint64_t owner;
2217 157624186 : uint64_t offset;
2218 157624186 : unsigned int flags;
2219 :
2220 157624186 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2221 157624186 : if (unwritten)
2222 38713836 : flags |= XFS_RMAP_UNWRITTEN;
2223 157624186 : trace_xfs_rmap_unmap(cur, bno, len, unwritten, oinfo);
2224 :
2225 : /*
2226 : * We should always have a left record because there's a static record
2227 : * for the AG headers at rm_startblock == 0 created by mkfs/growfs that
2228 : * will not ever be removed from the tree.
2229 : */
2230 157622719 : error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, flags,
2231 : <rec, &i);
2232 157621607 : if (error)
2233 272 : goto out_error;
2234 157621335 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2235 0 : xfs_btree_mark_sick(cur);
2236 0 : error = -EFSCORRUPTED;
2237 0 : goto out_error;
2238 : }
2239 157621335 : ltoff = ltrec.rm_offset;
2240 :
2241 : /* Make sure the extent we found covers the entire freeing range. */
2242 157621335 : if (XFS_IS_CORRUPT(mp,
2243 : ltrec.rm_startblock > bno ||
2244 : ltrec.rm_startblock + ltrec.rm_blockcount <
2245 : bno + len)) {
2246 0 : xfs_btree_mark_sick(cur);
2247 0 : error = -EFSCORRUPTED;
2248 0 : goto out_error;
2249 : }
2250 :
2251 : /* Make sure the owner matches what we expect to find in the tree. */
2252 157621335 : if (XFS_IS_CORRUPT(mp, owner != ltrec.rm_owner)) {
2253 0 : xfs_btree_mark_sick(cur);
2254 0 : error = -EFSCORRUPTED;
2255 0 : goto out_error;
2256 : }
2257 :
2258 : /* Make sure the unwritten flag matches. */
2259 157621335 : if (XFS_IS_CORRUPT(mp,
2260 : (flags & XFS_RMAP_UNWRITTEN) !=
2261 : (ltrec.rm_flags & XFS_RMAP_UNWRITTEN))) {
2262 0 : xfs_btree_mark_sick(cur);
2263 0 : error = -EFSCORRUPTED;
2264 0 : goto out_error;
2265 : }
2266 :
2267 : /* Check the offset. */
2268 157621335 : if (XFS_IS_CORRUPT(mp, ltrec.rm_offset > offset)) {
2269 0 : xfs_btree_mark_sick(cur);
2270 0 : error = -EFSCORRUPTED;
2271 0 : goto out_error;
2272 : }
2273 157621335 : if (XFS_IS_CORRUPT(mp, offset > ltoff + ltrec.rm_blockcount)) {
2274 0 : xfs_btree_mark_sick(cur);
2275 0 : error = -EFSCORRUPTED;
2276 0 : goto out_error;
2277 : }
2278 :
2279 157621335 : if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) {
2280 : /* Exact match, simply remove the record from rmap tree. */
2281 111798887 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2282 : ltrec.rm_blockcount, ltrec.rm_owner,
2283 : ltrec.rm_offset, ltrec.rm_flags);
2284 111800781 : if (error)
2285 2 : goto out_error;
2286 45822448 : } else if (ltrec.rm_startblock == bno) {
2287 : /*
2288 : * Overlap left hand side of extent: move the start, trim the
2289 : * length and update the current record.
2290 : *
2291 : * ltbno ltlen
2292 : * Orig: |oooooooooooooooooooo|
2293 : * Freeing: |fffffffff|
2294 : * Result: |rrrrrrrrrr|
2295 : * bno len
2296 : */
2297 :
2298 : /* Delete prev rmap. */
2299 15813058 : error = xfs_rmap_delete(cur, ltrec.rm_startblock,
2300 : ltrec.rm_blockcount, ltrec.rm_owner,
2301 : ltrec.rm_offset, ltrec.rm_flags);
2302 15813068 : if (error)
2303 0 : goto out_error;
2304 :
2305 : /* Add an rmap at the new offset. */
2306 15813068 : ltrec.rm_startblock += len;
2307 15813068 : ltrec.rm_blockcount -= len;
2308 15813068 : ltrec.rm_offset += len;
2309 15813068 : error = xfs_rmap_insert(cur, ltrec.rm_startblock,
2310 : ltrec.rm_blockcount, ltrec.rm_owner,
2311 : ltrec.rm_offset, ltrec.rm_flags);
2312 15813141 : if (error)
2313 0 : goto out_error;
2314 30009390 : } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) {
2315 : /*
2316 : * Overlap right hand side of extent: trim the length and
2317 : * update the current record.
2318 : *
2319 : * ltbno ltlen
2320 : * Orig: |oooooooooooooooooooo|
2321 : * Freeing: |fffffffff|
2322 : * Result: |rrrrrrrrrr|
2323 : * bno len
2324 : */
2325 11722290 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2326 : ltrec.rm_blockcount, ltrec.rm_owner,
2327 : ltrec.rm_offset, ltrec.rm_flags, &i);
2328 11722350 : if (error)
2329 0 : goto out_error;
2330 11722350 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2331 0 : xfs_btree_mark_sick(cur);
2332 0 : error = -EFSCORRUPTED;
2333 0 : goto out_error;
2334 : }
2335 11722350 : ltrec.rm_blockcount -= len;
2336 11722350 : error = xfs_rmap_update(cur, <rec);
2337 11722342 : if (error)
2338 0 : goto out_error;
2339 : } else {
2340 : /*
2341 : * Overlap middle of extent: trim the length of the existing
2342 : * record to the length of the new left-extent size, increment
2343 : * the insertion position so we can insert a new record
2344 : * containing the remaining right-extent space.
2345 : *
2346 : * ltbno ltlen
2347 : * Orig: |oooooooooooooooooooo|
2348 : * Freeing: |fffffffff|
2349 : * Result: |rrrrr| |rrrr|
2350 : * bno len
2351 : */
2352 18287100 : xfs_extlen_t orig_len = ltrec.rm_blockcount;
2353 :
2354 : /* Shrink the left side of the rmap */
2355 18287100 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2356 : ltrec.rm_blockcount, ltrec.rm_owner,
2357 : ltrec.rm_offset, ltrec.rm_flags, &i);
2358 18287154 : if (error)
2359 0 : goto out_error;
2360 18287154 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2361 0 : xfs_btree_mark_sick(cur);
2362 0 : error = -EFSCORRUPTED;
2363 0 : goto out_error;
2364 : }
2365 18287154 : ltrec.rm_blockcount = bno - ltrec.rm_startblock;
2366 18287154 : error = xfs_rmap_update(cur, <rec);
2367 18287180 : if (error)
2368 0 : goto out_error;
2369 :
2370 : /* Add an rmap at the new offset */
2371 18287180 : error = xfs_rmap_insert(cur, bno + len,
2372 18287180 : orig_len - len - ltrec.rm_blockcount,
2373 : ltrec.rm_owner, offset + len,
2374 : ltrec.rm_flags);
2375 18287205 : if (error)
2376 0 : goto out_error;
2377 : }
2378 :
2379 157623467 : trace_xfs_rmap_unmap_done(cur, bno, len, unwritten, oinfo);
2380 157621493 : out_error:
2381 157621493 : if (error)
2382 274 : trace_xfs_rmap_unmap_error(cur, error, _RET_IP_);
2383 157621493 : return error;
2384 : }
2385 :
2386 : /*
2387 : * Find an extent in the rmap btree and map it. For rmap extent types that
2388 : * can overlap (data fork rmaps on reflink filesystems) we must be careful
2389 : * that the prev/next records in the btree might belong to another owner.
2390 : * Therefore we must use delete+insert to alter any of the key fields.
2391 : *
2392 : * For every other situation there can only be one owner for a given extent,
2393 : * so we can call the regular _alloc function.
2394 : */
2395 : STATIC int
2396 334783104 : xfs_rmap_map_shared(
2397 : struct xfs_btree_cur *cur,
2398 : xfs_agblock_t bno,
2399 : xfs_extlen_t len,
2400 : bool unwritten,
2401 : const struct xfs_owner_info *oinfo)
2402 : {
2403 334783104 : struct xfs_mount *mp = cur->bc_mp;
2404 334783104 : struct xfs_rmap_irec ltrec;
2405 334783104 : struct xfs_rmap_irec gtrec;
2406 334783104 : int have_gt;
2407 334783104 : int have_lt;
2408 334783104 : int error = 0;
2409 334783104 : int i;
2410 334783104 : uint64_t owner;
2411 334783104 : uint64_t offset;
2412 334783104 : unsigned int flags = 0;
2413 :
2414 334783104 : xfs_owner_info_unpack(oinfo, &owner, &offset, &flags);
2415 334783104 : if (unwritten)
2416 67667029 : flags |= XFS_RMAP_UNWRITTEN;
2417 334783104 : trace_xfs_rmap_map(cur, bno, len, unwritten, oinfo);
2418 :
2419 : /* Is there a left record that abuts our range? */
2420 334780799 : error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, flags,
2421 : <rec, &have_lt);
2422 334798016 : if (error)
2423 287 : goto out_error;
2424 334797729 : if (have_lt &&
2425 59425475 : !xfs_rmap_is_mergeable(<rec, owner, flags))
2426 3933184 : have_lt = 0;
2427 :
2428 : /* Is there a right record that abuts our range? */
2429 334797729 : error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len,
2430 : flags, &have_gt);
2431 334795765 : if (error)
2432 2 : goto out_error;
2433 334795763 : if (have_gt) {
2434 4947392 : error = xfs_rmap_get_rec(cur, >rec, &have_gt);
2435 4947391 : if (error)
2436 0 : goto out_error;
2437 4947391 : if (XFS_IS_CORRUPT(mp, have_gt != 1)) {
2438 0 : xfs_btree_mark_sick(cur);
2439 0 : error = -EFSCORRUPTED;
2440 0 : goto out_error;
2441 : }
2442 4947391 : trace_xfs_rmap_find_right_neighbor_result(cur,
2443 : gtrec.rm_startblock, gtrec.rm_blockcount,
2444 : gtrec.rm_owner, gtrec.rm_offset,
2445 : gtrec.rm_flags);
2446 :
2447 4947391 : if (!xfs_rmap_is_mergeable(>rec, owner, flags))
2448 1028483 : have_gt = 0;
2449 : }
2450 :
2451 334795762 : if (have_lt &&
2452 55492309 : ltrec.rm_startblock + ltrec.rm_blockcount == bno &&
2453 17595397 : ltrec.rm_offset + ltrec.rm_blockcount == offset) {
2454 : /*
2455 : * Left edge contiguous, merge into left record.
2456 : *
2457 : * ltbno ltlen
2458 : * orig: |ooooooooo|
2459 : * adding: |aaaaaaaaa|
2460 : * result: |rrrrrrrrrrrrrrrrrrr|
2461 : * bno len
2462 : */
2463 17595396 : ltrec.rm_blockcount += len;
2464 17595396 : if (have_gt &&
2465 2028131 : bno + len == gtrec.rm_startblock &&
2466 2028131 : offset + len == gtrec.rm_offset) {
2467 : /*
2468 : * Right edge also contiguous, delete right record
2469 : * and merge into left record.
2470 : *
2471 : * ltbno ltlen gtbno gtlen
2472 : * orig: |ooooooooo| |ooooooooo|
2473 : * adding: |aaaaaaaaa|
2474 : * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr|
2475 : */
2476 2028131 : ltrec.rm_blockcount += gtrec.rm_blockcount;
2477 2028131 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2478 : gtrec.rm_blockcount, gtrec.rm_owner,
2479 : gtrec.rm_offset, gtrec.rm_flags);
2480 2028131 : if (error)
2481 0 : goto out_error;
2482 : }
2483 :
2484 : /* Point the cursor back to the left record and update. */
2485 17595396 : error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock,
2486 : ltrec.rm_blockcount, ltrec.rm_owner,
2487 : ltrec.rm_offset, ltrec.rm_flags, &i);
2488 17595398 : if (error)
2489 0 : goto out_error;
2490 17595398 : if (XFS_IS_CORRUPT(mp, i != 1)) {
2491 0 : xfs_btree_mark_sick(cur);
2492 0 : error = -EFSCORRUPTED;
2493 0 : goto out_error;
2494 : }
2495 :
2496 17595398 : error = xfs_rmap_update(cur, <rec);
2497 17595396 : if (error)
2498 0 : goto out_error;
2499 317200366 : } else if (have_gt &&
2500 1890778 : bno + len == gtrec.rm_startblock &&
2501 1890778 : offset + len == gtrec.rm_offset) {
2502 : /*
2503 : * Right edge contiguous, merge into right record.
2504 : *
2505 : * gtbno gtlen
2506 : * Orig: |ooooooooo|
2507 : * adding: |aaaaaaaaa|
2508 : * Result: |rrrrrrrrrrrrrrrrrrr|
2509 : * bno len
2510 : */
2511 : /* Delete the old record. */
2512 1890778 : error = xfs_rmap_delete(cur, gtrec.rm_startblock,
2513 : gtrec.rm_blockcount, gtrec.rm_owner,
2514 : gtrec.rm_offset, gtrec.rm_flags);
2515 1890778 : if (error)
2516 0 : goto out_error;
2517 :
2518 : /* Move the start and re-add it. */
2519 1890778 : gtrec.rm_startblock = bno;
2520 1890778 : gtrec.rm_blockcount += len;
2521 1890778 : gtrec.rm_offset = offset;
2522 1890778 : error = xfs_rmap_insert(cur, gtrec.rm_startblock,
2523 : gtrec.rm_blockcount, gtrec.rm_owner,
2524 : gtrec.rm_offset, gtrec.rm_flags);
2525 1890778 : if (error)
2526 0 : goto out_error;
2527 : } else {
2528 : /*
2529 : * No contiguous edge with identical owner, insert
2530 : * new record at current cursor position.
2531 : */
2532 315309588 : error = xfs_rmap_insert(cur, bno, len, owner, offset, flags);
2533 315312135 : if (error)
2534 165 : goto out_error;
2535 : }
2536 :
2537 334798144 : trace_xfs_rmap_map_done(cur, bno, len, unwritten, oinfo);
2538 334793704 : out_error:
2539 334793704 : if (error)
2540 454 : trace_xfs_rmap_map_error(cur, error, _RET_IP_);
2541 334793704 : return error;
2542 : }
2543 :
2544 : /* Insert a raw rmap into the rmapbt. */
2545 : int
2546 130567343 : xfs_rmap_map_raw(
2547 : struct xfs_btree_cur *cur,
2548 : struct xfs_rmap_irec *rmap)
2549 : {
2550 130567343 : struct xfs_owner_info oinfo;
2551 :
2552 130567343 : xfs_owner_info_pack(&oinfo, rmap->rm_owner, rmap->rm_offset,
2553 : rmap->rm_flags);
2554 :
2555 130567343 : if ((rmap->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK |
2556 101114201 : XFS_RMAP_UNWRITTEN)) ||
2557 101114201 : XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
2558 31754316 : return xfs_rmap_map(cur, rmap->rm_startblock,
2559 : rmap->rm_blockcount,
2560 31754316 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2561 : &oinfo);
2562 :
2563 98813027 : return xfs_rmap_map_shared(cur, rmap->rm_startblock,
2564 : rmap->rm_blockcount,
2565 98813027 : rmap->rm_flags & XFS_RMAP_UNWRITTEN,
2566 : &oinfo);
2567 : }
2568 :
2569 : struct xfs_rmap_query_range_info {
2570 : xfs_rmap_query_range_fn fn;
2571 : void *priv;
2572 : };
2573 :
2574 : /* Format btree record and pass to our callback. */
2575 : STATIC int
2576 >21772*10^7 : xfs_rmap_query_range_helper(
2577 : struct xfs_btree_cur *cur,
2578 : const union xfs_btree_rec *rec,
2579 : void *priv)
2580 : {
2581 >21772*10^7 : struct xfs_rmap_query_range_info *query = priv;
2582 >21772*10^7 : struct xfs_rmap_irec irec;
2583 >21772*10^7 : xfs_failaddr_t fa;
2584 :
2585 >21772*10^7 : fa = xfs_rmap_btrec_to_irec(rec, &irec);
2586 >21665*10^7 : if (!fa)
2587 >21666*10^7 : fa = xfs_rmap_check_irec(cur, &irec);
2588 >21614*10^7 : if (fa)
2589 0 : return xfs_rmap_complain_bad_rec(cur, fa, &irec);
2590 :
2591 >21614*10^7 : return query->fn(cur, &irec, query->priv);
2592 : }
2593 :
2594 : /* Find all rmaps between two keys. */
2595 : int
2596 1531032973 : xfs_rmap_query_range(
2597 : struct xfs_btree_cur *cur,
2598 : const struct xfs_rmap_irec *low_rec,
2599 : const struct xfs_rmap_irec *high_rec,
2600 : xfs_rmap_query_range_fn fn,
2601 : void *priv)
2602 : {
2603 1531032973 : union xfs_btree_irec low_brec = { .r = *low_rec };
2604 1531032973 : union xfs_btree_irec high_brec = { .r = *high_rec };
2605 1531032973 : struct xfs_rmap_query_range_info query = { .priv = priv, .fn = fn };
2606 :
2607 1531032973 : return xfs_btree_query_range(cur, &low_brec, &high_brec,
2608 : xfs_rmap_query_range_helper, &query);
2609 : }
2610 :
2611 : /* Find all rmaps. */
2612 : int
2613 32531870 : xfs_rmap_query_all(
2614 : struct xfs_btree_cur *cur,
2615 : xfs_rmap_query_range_fn fn,
2616 : void *priv)
2617 : {
2618 32531870 : struct xfs_rmap_query_range_info query;
2619 :
2620 32531870 : query.priv = priv;
2621 32531870 : query.fn = fn;
2622 32531870 : return xfs_btree_query_all(cur, xfs_rmap_query_range_helper, &query);
2623 : }
2624 :
2625 : /* Clean up after calling xfs_rmap_finish_one. */
2626 : void
2627 728929140 : xfs_rmap_finish_one_cleanup(
2628 : struct xfs_trans *tp,
2629 : struct xfs_btree_cur *rcur,
2630 : int error)
2631 : {
2632 728929140 : struct xfs_buf *agbp = NULL;
2633 :
2634 728929140 : if (rcur == NULL)
2635 : return;
2636 728927831 : if (rcur->bc_btnum == XFS_BTNUM_RMAP)
2637 444450934 : agbp = rcur->bc_ag.agbp;
2638 728927831 : xfs_btree_del_cursor(rcur, error);
2639 728953242 : if (error && agbp)
2640 1698 : xfs_trans_brelse(tp, agbp);
2641 : }
2642 :
2643 : /* Commit an rmap operation into the ondisk tree. */
2644 : int
2645 838033247 : __xfs_rmap_finish_intent(
2646 : struct xfs_btree_cur *rcur,
2647 : enum xfs_rmap_intent_type op,
2648 : xfs_agblock_t bno,
2649 : xfs_extlen_t len,
2650 : const struct xfs_owner_info *oinfo,
2651 : bool unwritten)
2652 : {
2653 838033247 : switch (op) {
2654 196586964 : case XFS_RMAP_ALLOC:
2655 : case XFS_RMAP_MAP:
2656 196586964 : return xfs_rmap_map(rcur, bno, len, unwritten, oinfo);
2657 235978953 : case XFS_RMAP_MAP_SHARED:
2658 235978953 : return xfs_rmap_map_shared(rcur, bno, len, unwritten, oinfo);
2659 173338803 : case XFS_RMAP_FREE:
2660 : case XFS_RMAP_UNMAP:
2661 173338803 : return xfs_rmap_unmap(rcur, bno, len, unwritten, oinfo);
2662 157624213 : case XFS_RMAP_UNMAP_SHARED:
2663 157624213 : return xfs_rmap_unmap_shared(rcur, bno, len, unwritten, oinfo);
2664 28785599 : case XFS_RMAP_CONVERT:
2665 28785599 : return xfs_rmap_convert(rcur, bno, len, !unwritten, oinfo);
2666 45718715 : case XFS_RMAP_CONVERT_SHARED:
2667 45718715 : return xfs_rmap_convert_shared(rcur, bno, len, !unwritten,
2668 45718715 : oinfo);
2669 0 : default:
2670 0 : ASSERT(0);
2671 0 : return -EFSCORRUPTED;
2672 : }
2673 : }
2674 :
2675 : /* Does this btree cursor match the given group object? */
2676 : static inline bool
2677 : xfs_rmap_is_wrong_cursor(
2678 : struct xfs_btree_cur *cur,
2679 : struct xfs_rmap_intent *ri)
2680 : {
2681 108262737 : if (cur->bc_btnum == XFS_BTNUM_RTRMAP)
2682 44227668 : return cur->bc_ino.rtg != ri->ri_rtg;
2683 64035069 : return cur->bc_ag.pag != ri->ri_pag;
2684 : }
2685 :
2686 : /*
2687 : * Process one of the deferred rmap operations. We pass back the
2688 : * btree cursor to maintain our lock on the rmapbt between calls.
2689 : * This saves time and eliminates a buffer deadlock between the
2690 : * superblock and the AGF because we'll always grab them in the same
2691 : * order.
2692 : */
2693 : int
2694 837225398 : xfs_rmap_finish_one(
2695 : struct xfs_trans *tp,
2696 : struct xfs_rmap_intent *ri,
2697 : struct xfs_btree_cur **pcur)
2698 : {
2699 837225398 : struct xfs_owner_info oinfo;
2700 837225398 : struct xfs_mount *mp = tp->t_mountp;
2701 837225398 : struct xfs_btree_cur *rcur;
2702 837225398 : struct xfs_buf *agbp = NULL;
2703 837225398 : xfs_agblock_t bno;
2704 837225398 : bool unwritten;
2705 837225398 : int error = 0;
2706 :
2707 837225398 : trace_xfs_rmap_deferred(mp, ri);
2708 :
2709 837214735 : if (ri->ri_realtime) {
2710 328666232 : xfs_rgnumber_t rgno;
2711 :
2712 328666232 : bno = xfs_rtb_to_rgbno(mp, ri->ri_bmap.br_startblock, &rgno);
2713 : } else {
2714 508548503 : bno = XFS_FSB_TO_AGBNO(mp, ri->ri_bmap.br_startblock);
2715 : }
2716 :
2717 837215216 : if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_RMAP_FINISH_ONE))
2718 : return -EIO;
2719 :
2720 : /*
2721 : * If we haven't gotten a cursor or the cursor AG doesn't match
2722 : * the startblock, get one now.
2723 : */
2724 837221260 : rcur = *pcur;
2725 945483997 : if (rcur != NULL && xfs_rmap_is_wrong_cursor(rcur, ri)) {
2726 4722 : xfs_rmap_finish_one_cleanup(tp, rcur, 0);
2727 4722 : rcur = NULL;
2728 4722 : *pcur = NULL;
2729 : }
2730 837221260 : if (rcur == NULL) {
2731 728961206 : if (ri->ri_realtime) {
2732 284439125 : xfs_rtgroup_lock(tp, ri->ri_rtg, XFS_RTGLOCK_RMAP);
2733 284440531 : rcur = xfs_rtrmapbt_init_cursor(mp, tp, ri->ri_rtg,
2734 284440531 : ri->ri_rtg->rtg_rmapip);
2735 284440406 : rcur->bc_ino.flags = 0;
2736 : } else {
2737 : /*
2738 : * Refresh the freelist before we start changing the
2739 : * rmapbt, because a shape change could cause us to
2740 : * allocate blocks.
2741 : */
2742 444522081 : error = xfs_free_extent_fix_freelist(tp, ri->ri_pag,
2743 : &agbp);
2744 444535260 : if (error) {
2745 1280 : xfs_ag_mark_sick(ri->ri_pag, XFS_SICK_AG_AGFL);
2746 1280 : return error;
2747 : }
2748 444533980 : if (XFS_IS_CORRUPT(tp->t_mountp, !agbp)) {
2749 0 : xfs_ag_mark_sick(ri->ri_pag, XFS_SICK_AG_AGFL);
2750 0 : return -EFSCORRUPTED;
2751 : }
2752 :
2753 444533980 : rcur = xfs_rmapbt_init_cursor(mp, tp, agbp, ri->ri_pag);
2754 : }
2755 : }
2756 837240995 : *pcur = rcur;
2757 :
2758 837240995 : xfs_rmap_ino_owner(&oinfo, ri->ri_owner, ri->ri_whichfork,
2759 : ri->ri_bmap.br_startoff);
2760 837240995 : unwritten = ri->ri_bmap.br_state == XFS_EXT_UNWRITTEN;
2761 :
2762 837240995 : error = __xfs_rmap_finish_intent(rcur, ri->ri_type, bno,
2763 837240995 : ri->ri_bmap.br_blockcount, &oinfo, unwritten);
2764 837171789 : if (error)
2765 : return error;
2766 :
2767 837170087 : if (ri->ri_realtime)
2768 328665515 : xfs_rmap_update_hook(tp, NULL, ri->ri_rtg, ri->ri_type, bno,
2769 328665515 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2770 : else
2771 508504572 : xfs_rmap_update_hook(tp, ri->ri_pag, NULL, ri->ri_type, bno,
2772 508504572 : ri->ri_bmap.br_blockcount, unwritten, &oinfo);
2773 : return 0;
2774 : }
2775 :
2776 : /*
2777 : * Don't defer an rmap if we aren't an rmap filesystem.
2778 : */
2779 : static bool
2780 : xfs_rmap_update_is_needed(
2781 : struct xfs_mount *mp,
2782 : int whichfork)
2783 : {
2784 827841586 : return xfs_has_rmapbt(mp) && whichfork != XFS_COW_FORK;
2785 : }
2786 :
2787 : /*
2788 : * Record a rmap intent; the list is kept sorted first by AG and then by
2789 : * increasing age.
2790 : */
2791 : static void
2792 837004106 : __xfs_rmap_add(
2793 : struct xfs_trans *tp,
2794 : enum xfs_rmap_intent_type type,
2795 : uint64_t owner,
2796 : bool isrt,
2797 : int whichfork,
2798 : struct xfs_bmbt_irec *bmap)
2799 : {
2800 837004106 : struct xfs_rmap_intent *ri;
2801 837004106 : enum xfs_defer_ops_type optype;
2802 :
2803 837004106 : ri = kmem_cache_alloc(xfs_rmap_intent_cache, GFP_NOFS | __GFP_NOFAIL);
2804 837087859 : INIT_LIST_HEAD(&ri->ri_list);
2805 837087859 : ri->ri_type = type;
2806 837087859 : ri->ri_owner = owner;
2807 837087859 : ri->ri_whichfork = whichfork;
2808 837087859 : ri->ri_bmap = *bmap;
2809 837087859 : ri->ri_realtime = isrt;
2810 :
2811 : /*
2812 : * Deferred rmap updates for the realtime and data sections must use
2813 : * separate transactions to finish deferred work because updates to
2814 : * realtime metadata files can lock AGFs to allocate btree blocks and
2815 : * we don't want that mixing with the AGF locks taken to finish data
2816 : * section updates.
2817 : */
2818 837087859 : if (isrt)
2819 : optype = XFS_DEFER_OPS_TYPE_RMAP_RT;
2820 : else
2821 508405651 : optype = XFS_DEFER_OPS_TYPE_RMAP;
2822 :
2823 837087859 : trace_xfs_rmap_defer(tp->t_mountp, ri);
2824 :
2825 837050491 : xfs_rmap_update_get_group(tp->t_mountp, ri);
2826 837192751 : xfs_defer_add(tp, optype, &ri->ri_list);
2827 837085804 : }
2828 :
2829 : /* Map an extent into a file. */
2830 : void
2831 455464049 : xfs_rmap_map_extent(
2832 : struct xfs_trans *tp,
2833 : struct xfs_inode *ip,
2834 : int whichfork,
2835 : struct xfs_bmbt_irec *PREV)
2836 : {
2837 455464049 : enum xfs_rmap_intent_type type = XFS_RMAP_MAP;
2838 455464049 : bool isrt = xfs_ifork_is_realtime(ip, whichfork);
2839 :
2840 910889778 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2841 : return;
2842 :
2843 427068119 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2844 235690711 : type = XFS_RMAP_MAP_SHARED;
2845 :
2846 427068119 : __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV);
2847 : }
2848 :
2849 : /* Unmap an extent out of a file. */
2850 : void
2851 331445396 : xfs_rmap_unmap_extent(
2852 : struct xfs_trans *tp,
2853 : struct xfs_inode *ip,
2854 : int whichfork,
2855 : struct xfs_bmbt_irec *PREV)
2856 : {
2857 331445396 : enum xfs_rmap_intent_type type = XFS_RMAP_UNMAP;
2858 331445396 : bool isrt = xfs_ifork_is_realtime(ip, whichfork);
2859 :
2860 662819876 : if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork))
2861 : return;
2862 :
2863 309176210 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2864 157375664 : type = XFS_RMAP_UNMAP_SHARED;
2865 :
2866 309176210 : __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV);
2867 : }
2868 :
2869 : /*
2870 : * Convert a data fork extent from unwritten to real or vice versa.
2871 : *
2872 : * Note that tp can be NULL here as no transaction is used for COW fork
2873 : * unwritten conversion.
2874 : */
2875 : void
2876 91333179 : xfs_rmap_convert_extent(
2877 : struct xfs_mount *mp,
2878 : struct xfs_trans *tp,
2879 : struct xfs_inode *ip,
2880 : int whichfork,
2881 : struct xfs_bmbt_irec *PREV)
2882 : {
2883 91333179 : enum xfs_rmap_intent_type type = XFS_RMAP_CONVERT;
2884 91333179 : bool isrt = xfs_ifork_is_realtime(ip, whichfork);
2885 :
2886 182662790 : if (!xfs_rmap_update_is_needed(mp, whichfork))
2887 : return;
2888 :
2889 74358920 : if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip))
2890 45597523 : type = XFS_RMAP_CONVERT_SHARED;
2891 :
2892 74358920 : __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV);
2893 : }
2894 :
2895 : /* Schedule the creation of an rmap for non-file data. */
2896 : void
2897 5065664 : xfs_rmap_alloc_extent(
2898 : struct xfs_trans *tp,
2899 : bool isrt,
2900 : xfs_fsblock_t fsbno,
2901 : xfs_extlen_t len,
2902 : uint64_t owner)
2903 : {
2904 5065664 : struct xfs_bmbt_irec bmap;
2905 :
2906 5065664 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2907 0 : return;
2908 :
2909 5065664 : bmap.br_startblock = fsbno;
2910 5065664 : bmap.br_blockcount = len;
2911 5065664 : bmap.br_startoff = 0;
2912 5065664 : bmap.br_state = XFS_EXT_NORM;
2913 :
2914 5065664 : __xfs_rmap_add(tp, XFS_RMAP_ALLOC, owner, isrt, XFS_DATA_FORK, &bmap);
2915 : }
2916 :
2917 : /* Schedule the deletion of an rmap for non-file data. */
2918 : void
2919 21339015 : xfs_rmap_free_extent(
2920 : struct xfs_trans *tp,
2921 : bool isrt,
2922 : xfs_fsblock_t fsbno,
2923 : xfs_extlen_t len,
2924 : uint64_t owner)
2925 : {
2926 21339015 : struct xfs_bmbt_irec bmap;
2927 :
2928 21339015 : if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK))
2929 0 : return;
2930 :
2931 21339015 : bmap.br_startblock = fsbno;
2932 21339015 : bmap.br_blockcount = len;
2933 21339015 : bmap.br_startoff = 0;
2934 21339015 : bmap.br_state = XFS_EXT_NORM;
2935 :
2936 21339015 : __xfs_rmap_add(tp, XFS_RMAP_FREE, owner, isrt, XFS_DATA_FORK, &bmap);
2937 : }
2938 :
2939 : /* Compare rmap records. Returns -1 if a < b, 1 if a > b, and 0 if equal. */
2940 : int
2941 2433462262 : xfs_rmap_compare(
2942 : const struct xfs_rmap_irec *a,
2943 : const struct xfs_rmap_irec *b)
2944 : {
2945 2433462262 : __u64 oa;
2946 2433462262 : __u64 ob;
2947 :
2948 2433462262 : oa = xfs_rmap_irec_offset_pack(a);
2949 2433462262 : ob = xfs_rmap_irec_offset_pack(b);
2950 :
2951 2433462262 : if (a->rm_startblock < b->rm_startblock)
2952 : return -1;
2953 2433362475 : else if (a->rm_startblock > b->rm_startblock)
2954 : return 1;
2955 2226267806 : else if (a->rm_owner < b->rm_owner)
2956 : return -1;
2957 1135876552 : else if (a->rm_owner > b->rm_owner)
2958 : return 1;
2959 1119464862 : else if (oa < ob)
2960 : return -1;
2961 43254538 : else if (oa > ob)
2962 : return 1;
2963 : else
2964 21127 : return 0;
2965 : }
2966 :
2967 : /*
2968 : * Scan the physical storage part of the keyspace of the reverse mapping index
2969 : * and tell us if the area has no records, is fully mapped by records, or is
2970 : * partially filled.
2971 : */
2972 : int
2973 481802783 : xfs_rmap_has_records(
2974 : struct xfs_btree_cur *cur,
2975 : xfs_agblock_t bno,
2976 : xfs_extlen_t len,
2977 : enum xbtree_recpacking *outcome)
2978 : {
2979 481802783 : union xfs_btree_key mask = {
2980 : .rmap.rm_startblock = cpu_to_be32(-1U),
2981 : };
2982 481802783 : union xfs_btree_irec low;
2983 481802783 : union xfs_btree_irec high;
2984 :
2985 481802783 : memset(&low, 0, sizeof(low));
2986 481802783 : low.r.rm_startblock = bno;
2987 481802783 : memset(&high, 0xFF, sizeof(high));
2988 481802783 : high.r.rm_startblock = bno + len - 1;
2989 :
2990 481802783 : return xfs_btree_has_records(cur, &low, &high, &mask, outcome);
2991 : }
2992 :
2993 : struct xfs_rmap_ownercount {
2994 : /* Owner that we're looking for. */
2995 : struct xfs_rmap_irec good;
2996 :
2997 : /* rmap search keys */
2998 : struct xfs_rmap_irec low;
2999 : struct xfs_rmap_irec high;
3000 :
3001 : struct xfs_rmap_matches *results;
3002 :
3003 : /* Stop early if we find a nonmatch? */
3004 : bool stop_on_nonmatch;
3005 : };
3006 :
3007 : /* Does this rmap represent space that can have multiple owners? */
3008 : static inline bool
3009 : xfs_rmap_shareable(
3010 : struct xfs_mount *mp,
3011 : const struct xfs_rmap_irec *rmap)
3012 : {
3013 84987633 : if (!xfs_has_reflink(mp))
3014 : return false;
3015 169973378 : if (XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner))
3016 : return false;
3017 83901476 : if (rmap->rm_flags & (XFS_RMAP_ATTR_FORK |
3018 : XFS_RMAP_BMBT_BLOCK))
3019 8114075 : return false;
3020 : return true;
3021 : }
3022 :
3023 : static inline void
3024 1020551406 : xfs_rmap_ownercount_init(
3025 : struct xfs_rmap_ownercount *roc,
3026 : xfs_agblock_t bno,
3027 : xfs_extlen_t len,
3028 : const struct xfs_owner_info *oinfo,
3029 : struct xfs_rmap_matches *results)
3030 : {
3031 1020551406 : memset(roc, 0, sizeof(*roc));
3032 1020551406 : roc->results = results;
3033 :
3034 1020551406 : roc->low.rm_startblock = bno;
3035 1020551406 : memset(&roc->high, 0xFF, sizeof(roc->high));
3036 1020551406 : roc->high.rm_startblock = bno + len - 1;
3037 :
3038 1020551406 : memset(results, 0, sizeof(*results));
3039 1020551406 : roc->good.rm_startblock = bno;
3040 1020551406 : roc->good.rm_blockcount = len;
3041 1020551406 : roc->good.rm_owner = oinfo->oi_owner;
3042 1020551406 : roc->good.rm_offset = oinfo->oi_offset;
3043 1020551406 : if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK)
3044 2395019 : roc->good.rm_flags |= XFS_RMAP_ATTR_FORK;
3045 1020551406 : if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK)
3046 12457949 : roc->good.rm_flags |= XFS_RMAP_BMBT_BLOCK;
3047 1020551406 : }
3048 :
3049 : /* Figure out if this is a match for the owner. */
3050 : STATIC int
3051 1079717298 : xfs_rmap_count_owners_helper(
3052 : struct xfs_btree_cur *cur,
3053 : const struct xfs_rmap_irec *rec,
3054 : void *priv)
3055 : {
3056 1079717298 : struct xfs_rmap_ownercount *roc = priv;
3057 1079717298 : struct xfs_rmap_irec check = *rec;
3058 1079717298 : unsigned int keyflags;
3059 1079717298 : bool filedata;
3060 1079717298 : int64_t delta;
3061 :
3062 1079717298 : filedata = !XFS_RMAP_NON_INODE_OWNER(check.rm_owner) &&
3063 592866998 : !(check.rm_flags & XFS_RMAP_BMBT_BLOCK);
3064 :
3065 : /* Trim the part of check that comes before the comparison range. */
3066 1079717298 : delta = (int64_t)roc->good.rm_startblock - check.rm_startblock;
3067 1079717298 : if (delta > 0) {
3068 347409836 : check.rm_startblock += delta;
3069 347409836 : check.rm_blockcount -= delta;
3070 347409836 : if (filedata)
3071 3982682 : check.rm_offset += delta;
3072 : }
3073 :
3074 : /* Trim the part of check that comes after the comparison range. */
3075 1079717298 : delta = (check.rm_startblock + check.rm_blockcount) -
3076 1079717298 : (roc->good.rm_startblock + roc->good.rm_blockcount);
3077 1079717298 : if (delta > 0)
3078 413150452 : check.rm_blockcount -= delta;
3079 :
3080 : /* Don't care about unwritten status for establishing ownership. */
3081 1079717298 : keyflags = check.rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK);
3082 :
3083 1079717298 : if (check.rm_startblock == roc->good.rm_startblock &&
3084 1020095718 : check.rm_blockcount == roc->good.rm_blockcount &&
3085 1020095718 : check.rm_owner == roc->good.rm_owner &&
3086 994708008 : check.rm_offset == roc->good.rm_offset &&
3087 994718506 : keyflags == roc->good.rm_flags) {
3088 994729665 : roc->results->matches++;
3089 : } else {
3090 84987633 : roc->results->non_owner_matches++;
3091 169975266 : if (xfs_rmap_shareable(cur->bc_mp, &roc->good) ^
3092 : xfs_rmap_shareable(cur->bc_mp, &check))
3093 75787624 : roc->results->bad_non_owner_matches++;
3094 : }
3095 :
3096 1079717298 : if (roc->results->non_owner_matches && roc->stop_on_nonmatch)
3097 0 : return -ECANCELED;
3098 :
3099 : return 0;
3100 : }
3101 :
3102 : /* Count the number of owners and non-owners of this range of blocks. */
3103 : int
3104 1013450236 : xfs_rmap_count_owners(
3105 : struct xfs_btree_cur *cur,
3106 : xfs_agblock_t bno,
3107 : xfs_extlen_t len,
3108 : const struct xfs_owner_info *oinfo,
3109 : struct xfs_rmap_matches *results)
3110 : {
3111 1013450236 : struct xfs_rmap_ownercount roc;
3112 1013450236 : int error;
3113 :
3114 1013450236 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, results);
3115 1013077795 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
3116 : xfs_rmap_count_owners_helper, &roc);
3117 1014141729 : if (error)
3118 : return error;
3119 :
3120 : /*
3121 : * There can't be any non-owner rmaps that conflict with the given
3122 : * owner if we didn't find any rmaps matching the owner.
3123 : */
3124 1014141729 : if (!results->matches)
3125 26228880 : results->bad_non_owner_matches = 0;
3126 :
3127 : return 0;
3128 : }
3129 :
3130 : /*
3131 : * Given an extent and some owner info, can we find records overlapping
3132 : * the extent whose owner info does not match the given owner?
3133 : */
3134 : int
3135 7380426 : xfs_rmap_has_other_keys(
3136 : struct xfs_btree_cur *cur,
3137 : xfs_agblock_t bno,
3138 : xfs_extlen_t len,
3139 : const struct xfs_owner_info *oinfo,
3140 : bool *has_other)
3141 : {
3142 7380426 : struct xfs_rmap_matches res;
3143 7380426 : struct xfs_rmap_ownercount roc;
3144 7380426 : int error;
3145 :
3146 7380426 : xfs_rmap_ownercount_init(&roc, bno, len, oinfo, &res);
3147 7379696 : roc.stop_on_nonmatch = true;
3148 :
3149 7379696 : error = xfs_rmap_query_range(cur, &roc.low, &roc.high,
3150 : xfs_rmap_count_owners_helper, &roc);
3151 7381175 : if (error == -ECANCELED) {
3152 0 : *has_other = true;
3153 0 : return 0;
3154 : }
3155 7381175 : if (error)
3156 : return error;
3157 :
3158 7381175 : *has_other = false;
3159 7381175 : return 0;
3160 : }
3161 :
3162 : const struct xfs_owner_info XFS_RMAP_OINFO_SKIP_UPDATE = {
3163 : .oi_owner = XFS_RMAP_OWN_NULL,
3164 : };
3165 : const struct xfs_owner_info XFS_RMAP_OINFO_ANY_OWNER = {
3166 : .oi_owner = XFS_RMAP_OWN_UNKNOWN,
3167 : };
3168 : const struct xfs_owner_info XFS_RMAP_OINFO_FS = {
3169 : .oi_owner = XFS_RMAP_OWN_FS,
3170 : };
3171 : const struct xfs_owner_info XFS_RMAP_OINFO_LOG = {
3172 : .oi_owner = XFS_RMAP_OWN_LOG,
3173 : };
3174 : const struct xfs_owner_info XFS_RMAP_OINFO_AG = {
3175 : .oi_owner = XFS_RMAP_OWN_AG,
3176 : };
3177 : const struct xfs_owner_info XFS_RMAP_OINFO_INOBT = {
3178 : .oi_owner = XFS_RMAP_OWN_INOBT,
3179 : };
3180 : const struct xfs_owner_info XFS_RMAP_OINFO_INODES = {
3181 : .oi_owner = XFS_RMAP_OWN_INODES,
3182 : };
3183 : const struct xfs_owner_info XFS_RMAP_OINFO_REFC = {
3184 : .oi_owner = XFS_RMAP_OWN_REFC,
3185 : };
3186 : const struct xfs_owner_info XFS_RMAP_OINFO_COW = {
3187 : .oi_owner = XFS_RMAP_OWN_COW,
3188 : };
3189 :
3190 : int __init
3191 59 : xfs_rmap_intent_init_cache(void)
3192 : {
3193 59 : xfs_rmap_intent_cache = kmem_cache_create("xfs_rmap_intent",
3194 : sizeof(struct xfs_rmap_intent),
3195 : 0, 0, NULL);
3196 :
3197 59 : return xfs_rmap_intent_cache != NULL ? 0 : -ENOMEM;
3198 : }
3199 :
3200 : void
3201 58 : xfs_rmap_intent_destroy_cache(void)
3202 : {
3203 58 : kmem_cache_destroy(xfs_rmap_intent_cache);
3204 58 : xfs_rmap_intent_cache = NULL;
3205 58 : }
|
