Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0+
2 : /*
3 : * Copyright (C) 2016 Oracle. All Rights Reserved.
4 : * Author: Darrick J. Wong <darrick.wong@oracle.com>
5 : */
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_shared.h"
9 : #include "xfs_format.h"
10 : #include "xfs_log_format.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_mount.h"
13 : #include "xfs_defer.h"
14 : #include "xfs_trans.h"
15 : #include "xfs_buf_item.h"
16 : #include "xfs_inode.h"
17 : #include "xfs_inode_item.h"
18 : #include "xfs_trace.h"
19 : #include "xfs_icache.h"
20 : #include "xfs_log.h"
21 : #include "xfs_rmap.h"
22 : #include "xfs_refcount.h"
23 : #include "xfs_bmap.h"
24 : #include "xfs_alloc.h"
25 : #include "xfs_buf.h"
26 : #include "xfs_da_format.h"
27 : #include "xfs_da_btree.h"
28 : #include "xfs_attr.h"
29 : #include "xfs_swapext.h"
30 :
31 : static struct kmem_cache *xfs_defer_pending_cache;
32 :
33 : /*
34 : * Deferred Operations in XFS
35 : *
36 : * Due to the way locking rules work in XFS, certain transactions (block
37 : * mapping and unmapping, typically) have permanent reservations so that
38 : * we can roll the transaction to adhere to AG locking order rules and
39 : * to unlock buffers between metadata updates. Prior to rmap/reflink,
40 : * the mapping code had a mechanism to perform these deferrals for
41 : * extents that were going to be freed; this code makes that facility
42 : * more generic.
43 : *
44 : * When adding the reverse mapping and reflink features, it became
45 : * necessary to perform complex remapping multi-transactions to comply
46 : * with AG locking order rules, and to be able to spread a single
47 : * refcount update operation (an operation on an n-block extent can
48 : * update as many as n records!) among multiple transactions. XFS can
49 : * roll a transaction to facilitate this, but using this facility
50 : * requires us to log "intent" items in case log recovery needs to
51 : * redo the operation, and to log "done" items to indicate that redo
52 : * is not necessary.
53 : *
54 : * Deferred work is tracked in xfs_defer_pending items. Each pending
55 : * item tracks one type of deferred work. Incoming work items (which
56 : * have not yet had an intent logged) are attached to a pending item
57 : * on the dop_intake list, where they wait for the caller to finish
58 : * the deferred operations.
59 : *
60 : * Finishing a set of deferred operations is an involved process. To
61 : * start, we define "rolling a deferred-op transaction" as follows:
62 : *
63 : * > For each xfs_defer_pending item on the dop_intake list,
64 : * - Sort the work items in AG order. XFS locking
65 : * order rules require us to lock buffers in AG order.
66 : * - Create a log intent item for that type.
67 : * - Attach it to the pending item.
68 : * - Move the pending item from the dop_intake list to the
69 : * dop_pending list.
70 : * > Roll the transaction.
71 : *
72 : * NOTE: To avoid exceeding the transaction reservation, we limit the
73 : * number of items that we attach to a given xfs_defer_pending.
74 : *
75 : * The actual finishing process looks like this:
76 : *
77 : * > For each xfs_defer_pending in the dop_pending list,
78 : * - Roll the deferred-op transaction as above.
79 : * - Create a log done item for that type, and attach it to the
80 : * log intent item.
81 : * - For each work item attached to the log intent item,
82 : * * Perform the described action.
83 : * * Attach the work item to the log done item.
84 : * * If the result of doing the work was -EAGAIN, ->finish work
85 : * wants a new transaction. See the "Requesting a Fresh
86 : * Transaction while Finishing Deferred Work" section below for
87 : * details.
88 : *
89 : * The key here is that we must log an intent item for all pending
90 : * work items every time we roll the transaction, and that we must log
91 : * a done item as soon as the work is completed. With this mechanism
92 : * we can perform complex remapping operations, chaining intent items
93 : * as needed.
94 : *
95 : * Requesting a Fresh Transaction while Finishing Deferred Work
96 : *
97 : * If ->finish_item decides that it needs a fresh transaction to
98 : * finish the work, it must ask its caller (xfs_defer_finish) for a
99 : * continuation. The most likely cause of this circumstance are the
100 : * refcount adjust functions deciding that they've logged enough items
101 : * to be at risk of exceeding the transaction reservation.
102 : *
103 : * To get a fresh transaction, we want to log the existing log done
104 : * item to prevent the log intent item from replaying, immediately log
105 : * a new log intent item with the unfinished work items, roll the
106 : * transaction, and re-call ->finish_item wherever it left off. The
107 : * log done item and the new log intent item must be in the same
108 : * transaction or atomicity cannot be guaranteed; defer_finish ensures
109 : * that this happens.
110 : *
111 : * This requires some coordination between ->finish_item and
112 : * defer_finish. Upon deciding to request a new transaction,
113 : * ->finish_item should update the current work item to reflect the
114 : * unfinished work. Next, it should reset the log done item's list
115 : * count to the number of items finished, and return -EAGAIN.
116 : * defer_finish sees the -EAGAIN, logs the new log intent item
117 : * with the remaining work items, and leaves the xfs_defer_pending
118 : * item at the head of the dop_work queue. Then it rolls the
119 : * transaction and picks up processing where it left off. It is
120 : * required that ->finish_item must be careful to leave enough
121 : * transaction reservation to fit the new log intent item.
122 : *
123 : * This is an example of remapping the extent (E, E+B) into file X at
124 : * offset A and dealing with the extent (C, C+B) already being mapped
125 : * there:
126 : * +-------------------------------------------------+
127 : * | Unmap file X startblock C offset A length B | t0
128 : * | Intent to reduce refcount for extent (C, B) |
129 : * | Intent to remove rmap (X, C, A, B) |
130 : * | Intent to free extent (D, 1) (bmbt block) |
131 : * | Intent to map (X, A, B) at startblock E |
132 : * +-------------------------------------------------+
133 : * | Map file X startblock E offset A length B | t1
134 : * | Done mapping (X, E, A, B) |
135 : * | Intent to increase refcount for extent (E, B) |
136 : * | Intent to add rmap (X, E, A, B) |
137 : * +-------------------------------------------------+
138 : * | Reduce refcount for extent (C, B) | t2
139 : * | Done reducing refcount for extent (C, 9) |
140 : * | Intent to reduce refcount for extent (C+9, B-9) |
141 : * | (ran out of space after 9 refcount updates) |
142 : * +-------------------------------------------------+
143 : * | Reduce refcount for extent (C+9, B+9) | t3
144 : * | Done reducing refcount for extent (C+9, B-9) |
145 : * | Increase refcount for extent (E, B) |
146 : * | Done increasing refcount for extent (E, B) |
147 : * | Intent to free extent (C, B) |
148 : * | Intent to free extent (F, 1) (refcountbt block) |
149 : * | Intent to remove rmap (F, 1, REFC) |
150 : * +-------------------------------------------------+
151 : * | Remove rmap (X, C, A, B) | t4
152 : * | Done removing rmap (X, C, A, B) |
153 : * | Add rmap (X, E, A, B) |
154 : * | Done adding rmap (X, E, A, B) |
155 : * | Remove rmap (F, 1, REFC) |
156 : * | Done removing rmap (F, 1, REFC) |
157 : * +-------------------------------------------------+
158 : * | Free extent (C, B) | t5
159 : * | Done freeing extent (C, B) |
160 : * | Free extent (D, 1) |
161 : * | Done freeing extent (D, 1) |
162 : * | Free extent (F, 1) |
163 : * | Done freeing extent (F, 1) |
164 : * +-------------------------------------------------+
165 : *
166 : * If we should crash before t2 commits, log recovery replays
167 : * the following intent items:
168 : *
169 : * - Intent to reduce refcount for extent (C, B)
170 : * - Intent to remove rmap (X, C, A, B)
171 : * - Intent to free extent (D, 1) (bmbt block)
172 : * - Intent to increase refcount for extent (E, B)
173 : * - Intent to add rmap (X, E, A, B)
174 : *
175 : * In the process of recovering, it should also generate and take care
176 : * of these intent items:
177 : *
178 : * - Intent to free extent (C, B)
179 : * - Intent to free extent (F, 1) (refcountbt block)
180 : * - Intent to remove rmap (F, 1, REFC)
181 : *
182 : * Note that the continuation requested between t2 and t3 is likely to
183 : * reoccur.
184 : */
185 :
186 : static const struct xfs_defer_op_type *defer_op_types[] = {
187 : [XFS_DEFER_OPS_TYPE_BMAP] = &xfs_bmap_update_defer_type,
188 : [XFS_DEFER_OPS_TYPE_REFCOUNT] = &xfs_refcount_update_defer_type,
189 : [XFS_DEFER_OPS_TYPE_REFCOUNT_RT] = &xfs_refcount_update_defer_type,
190 : [XFS_DEFER_OPS_TYPE_RMAP] = &xfs_rmap_update_defer_type,
191 : [XFS_DEFER_OPS_TYPE_RMAP_RT] = &xfs_rmap_update_defer_type,
192 : [XFS_DEFER_OPS_TYPE_FREE] = &xfs_extent_free_defer_type,
193 : [XFS_DEFER_OPS_TYPE_FREE_RT] = &xfs_extent_free_defer_type,
194 : [XFS_DEFER_OPS_TYPE_AGFL_FREE] = &xfs_agfl_free_defer_type,
195 : [XFS_DEFER_OPS_TYPE_ATTR] = &xfs_attr_defer_type,
196 : [XFS_DEFER_OPS_TYPE_SWAPEXT] = &xfs_swapext_defer_type,
197 : };
198 :
199 : /*
200 : * Ensure there's a log intent item associated with this deferred work item if
201 : * the operation must be restarted on crash. Returns 1 if there's a log item;
202 : * 0 if there isn't; or a negative errno.
203 : */
204 : static int
205 2023387214 : xfs_defer_create_intent(
206 : struct xfs_trans *tp,
207 : struct xfs_defer_pending *dfp,
208 : bool sort)
209 : {
210 2023387214 : const struct xfs_defer_op_type *ops = defer_op_types[dfp->dfp_type];
211 2023313931 : struct xfs_log_item *lip;
212 :
213 2023313931 : if (dfp->dfp_intent)
214 : return 1;
215 :
216 2023766841 : lip = ops->create_intent(tp, &dfp->dfp_work, dfp->dfp_count, sort);
217 2024154491 : if (!lip)
218 : return 0;
219 1689804342 : if (IS_ERR(lip))
220 0 : return PTR_ERR(lip);
221 :
222 1689804342 : dfp->dfp_intent = lip;
223 1689804342 : return 1;
224 : }
225 :
226 : /*
227 : * For each pending item in the intake list, log its intent item and the
228 : * associated extents, then add the entire intake list to the end of
229 : * the pending list.
230 : *
231 : * Returns 1 if at least one log item was associated with the deferred work;
232 : * 0 if there are no log items; or a negative errno.
233 : */
234 : static int
235 2023585362 : xfs_defer_create_intents(
236 : struct xfs_trans *tp)
237 : {
238 2023585362 : struct xfs_defer_pending *dfp;
239 2023585362 : int ret = 0;
240 :
241 3904834376 : list_for_each_entry(dfp, &tp->t_dfops, dfp_list) {
242 1880692231 : int ret2;
243 :
244 1880692231 : trace_xfs_defer_create_intent(tp->t_mountp, dfp);
245 1880164603 : ret2 = xfs_defer_create_intent(tp, dfp, true);
246 1881249014 : if (ret2 < 0)
247 0 : return ret2;
248 1881249014 : ret |= ret2;
249 : }
250 : return ret;
251 : }
252 :
253 : /* Abort all the intents that were committed. */
254 : STATIC void
255 4551 : xfs_defer_trans_abort(
256 : struct xfs_trans *tp,
257 : struct list_head *dop_pending)
258 : {
259 4551 : struct xfs_defer_pending *dfp;
260 4551 : const struct xfs_defer_op_type *ops;
261 :
262 4551 : trace_xfs_defer_trans_abort(tp, _RET_IP_);
263 :
264 : /* Abort intent items that don't have a done item. */
265 12070 : list_for_each_entry(dfp, dop_pending, dfp_list) {
266 7519 : ops = defer_op_types[dfp->dfp_type];
267 7519 : trace_xfs_defer_pending_abort(tp->t_mountp, dfp);
268 7520 : if (dfp->dfp_intent && !dfp->dfp_done) {
269 3183 : ops->abort_intent(dfp->dfp_intent);
270 3183 : dfp->dfp_intent = NULL;
271 : }
272 : }
273 4551 : }
274 :
275 : /*
276 : * Capture resources that the caller said not to release ("held") when the
277 : * transaction commits. Caller is responsible for zero-initializing @dres.
278 : */
279 : static int
280 1946819206 : xfs_defer_save_resources(
281 : struct xfs_defer_resources *dres,
282 : struct xfs_trans *tp)
283 : {
284 1946819206 : struct xfs_buf_log_item *bli;
285 1946819206 : struct xfs_inode_log_item *ili;
286 1946819206 : struct xfs_log_item *lip;
287 :
288 1946819206 : BUILD_BUG_ON(NBBY * sizeof(dres->dr_ordered) < XFS_DEFER_OPS_NR_BUFS);
289 :
290 10729811041 : list_for_each_entry(lip, &tp->t_items, li_trans) {
291 8782624992 : switch (lip->li_type) {
292 3103866778 : case XFS_LI_BUF:
293 3103866778 : bli = container_of(lip, struct xfs_buf_log_item,
294 : bli_item);
295 3103866778 : if (bli->bli_flags & XFS_BLI_HOLD) {
296 8694142 : if (dres->dr_bufs >= XFS_DEFER_OPS_NR_BUFS) {
297 0 : ASSERT(0);
298 0 : return -EFSCORRUPTED;
299 : }
300 8694142 : if (bli->bli_flags & XFS_BLI_ORDERED)
301 128135 : dres->dr_ordered |=
302 128135 : (1U << dres->dr_bufs);
303 : else
304 8566007 : xfs_trans_dirty_buf(tp, bli->bli_buf);
305 8697761 : dres->dr_bp[dres->dr_bufs++] = bli->bli_buf;
306 : }
307 : break;
308 2976034615 : case XFS_LI_INODE:
309 2976034615 : ili = container_of(lip, struct xfs_inode_log_item,
310 : ili_item);
311 2976034615 : if (ili->ili_lock_flags == 0) {
312 2473739076 : if (dres->dr_inos >= XFS_DEFER_OPS_NR_INODES) {
313 0 : ASSERT(0);
314 0 : return -EFSCORRUPTED;
315 : }
316 2473739076 : xfs_trans_log_inode(tp, ili->ili_inode,
317 : XFS_ILOG_CORE);
318 2474267216 : dres->dr_ip[dres->dr_inos++] = ili->ili_inode;
319 : }
320 : break;
321 : default:
322 : break;
323 : }
324 : }
325 :
326 : return 0;
327 : }
328 :
329 : /* Attach the held resources to the transaction. */
330 : static void
331 1947516609 : xfs_defer_restore_resources(
332 : struct xfs_trans *tp,
333 : struct xfs_defer_resources *dres)
334 : {
335 1947516609 : unsigned short i;
336 :
337 : /* Rejoin the joined inodes. */
338 4422216613 : for (i = 0; i < dres->dr_inos; i++)
339 2474677084 : xfs_trans_ijoin(tp, dres->dr_ip[i], 0);
340 :
341 : /* Rejoin the buffers and dirty them so the log moves forward. */
342 1956236726 : for (i = 0; i < dres->dr_bufs; i++) {
343 8696008 : xfs_trans_bjoin(tp, dres->dr_bp[i]);
344 8699376 : if (dres->dr_ordered & (1U << i))
345 128131 : xfs_trans_ordered_buf(tp, dres->dr_bp[i]);
346 8699380 : xfs_trans_bhold(tp, dres->dr_bp[i]);
347 : }
348 1947540718 : }
349 :
350 : /* Roll a transaction so we can do some deferred op processing. */
351 : STATIC int
352 1946957007 : xfs_defer_trans_roll(
353 : struct xfs_trans **tpp)
354 : {
355 1946957007 : struct xfs_defer_resources dres = { };
356 1946957007 : int error;
357 :
358 1946957007 : error = xfs_defer_save_resources(&dres, *tpp);
359 1947205823 : if (error)
360 : return error;
361 :
362 1947221175 : trace_xfs_defer_trans_roll(*tpp, _RET_IP_);
363 :
364 : /*
365 : * Roll the transaction. Rolling always given a new transaction (even
366 : * if committing the old one fails!) to hand back to the caller, so we
367 : * join the held resources to the new transaction so that we always
368 : * return with the held resources joined to @tpp, no matter what
369 : * happened.
370 : */
371 1946853665 : error = xfs_trans_roll(tpp);
372 :
373 1947541648 : xfs_defer_restore_resources(*tpp, &dres);
374 :
375 1947533759 : if (error)
376 246 : trace_xfs_defer_trans_roll_error(*tpp, error);
377 : return error;
378 : }
379 :
380 : /*
381 : * Free up any items left in the list.
382 : */
383 : static void
384 9475 : xfs_defer_cancel_list(
385 : struct xfs_mount *mp,
386 : struct list_head *dop_list)
387 : {
388 9475 : struct xfs_defer_pending *dfp;
389 9475 : struct xfs_defer_pending *pli;
390 9475 : struct list_head *pwi;
391 9475 : struct list_head *n;
392 9475 : const struct xfs_defer_op_type *ops;
393 :
394 : /*
395 : * Free the pending items. Caller should already have arranged
396 : * for the intent items to be released.
397 : */
398 17379 : list_for_each_entry_safe(dfp, pli, dop_list, dfp_list) {
399 7903 : ops = defer_op_types[dfp->dfp_type];
400 7903 : trace_xfs_defer_cancel_list(mp, dfp);
401 7903 : list_del(&dfp->dfp_list);
402 11521 : list_for_each_safe(pwi, n, &dfp->dfp_work) {
403 3617 : list_del(pwi);
404 3617 : dfp->dfp_count--;
405 3617 : trace_xfs_defer_cancel_item(mp, dfp, pwi);
406 3617 : ops->cancel_item(pwi);
407 : }
408 7904 : ASSERT(dfp->dfp_count == 0);
409 7904 : kmem_cache_free(xfs_defer_pending_cache, dfp);
410 : }
411 9476 : }
412 :
413 : /*
414 : * Prevent a log intent item from pinning the tail of the log by logging a
415 : * done item to release the intent item; and then log a new intent item.
416 : * The caller should provide a fresh transaction and roll it after we're done.
417 : */
418 : static int
419 1767532702 : xfs_defer_relog(
420 : struct xfs_trans **tpp,
421 : struct list_head *dfops)
422 : {
423 1767532702 : struct xlog *log = (*tpp)->t_mountp->m_log;
424 1767532702 : struct xfs_defer_pending *dfp;
425 1767532702 : xfs_lsn_t threshold_lsn = NULLCOMMITLSN;
426 :
427 :
428 1767532702 : ASSERT((*tpp)->t_flags & XFS_TRANS_PERM_LOG_RES);
429 :
430 4561704238 : list_for_each_entry(dfp, dfops, dfp_list) {
431 : /*
432 : * If the log intent item for this deferred op is not a part of
433 : * the current log checkpoint, relog the intent item to keep
434 : * the log tail moving forward. We're ok with this being racy
435 : * because an incorrect decision means we'll be a little slower
436 : * at pushing the tail.
437 : */
438 5495699690 : if (dfp->dfp_intent == NULL ||
439 2700643683 : xfs_log_item_in_current_chkpt(dfp->dfp_intent))
440 2794042587 : continue;
441 :
442 : /*
443 : * Figure out where we need the tail to be in order to maintain
444 : * the minimum required free space in the log. Only sample
445 : * the log threshold once per call.
446 : */
447 1013420 : if (threshold_lsn == NULLCOMMITLSN) {
448 963444 : threshold_lsn = xlog_grant_push_threshold(log, 0);
449 963446 : if (threshold_lsn == NULLCOMMITLSN)
450 : break;
451 : }
452 128948 : if (XFS_LSN_CMP(dfp->dfp_intent->li_lsn, threshold_lsn) >= 0)
453 7226 : continue;
454 :
455 121722 : trace_xfs_defer_relog_intent((*tpp)->t_mountp, dfp);
456 121722 : XFS_STATS_INC((*tpp)->t_mountp, defer_relog);
457 121722 : dfp->dfp_intent = xfs_trans_item_relog(dfp->dfp_intent, *tpp);
458 : }
459 :
460 1767526267 : if ((*tpp)->t_flags & XFS_TRANS_DIRTY)
461 138884 : return xfs_defer_trans_roll(tpp);
462 : return 0;
463 : }
464 :
465 : /*
466 : * Log an intent-done item for the first pending intent, and finish the work
467 : * items.
468 : */
469 : static int
470 2024528828 : xfs_defer_finish_one(
471 : struct xfs_trans *tp,
472 : struct xfs_defer_pending *dfp)
473 : {
474 2024528828 : const struct xfs_defer_op_type *ops = defer_op_types[dfp->dfp_type];
475 2024490543 : struct xfs_btree_cur *state = NULL;
476 2024490543 : struct list_head *li, *n;
477 2024490543 : int error;
478 :
479 2024490543 : trace_xfs_defer_pending_finish(tp->t_mountp, dfp);
480 :
481 2024245042 : dfp->dfp_done = ops->create_done(tp, dfp->dfp_intent, dfp->dfp_count);
482 4026566370 : list_for_each_safe(li, n, &dfp->dfp_work) {
483 2144837259 : list_del(li);
484 2144802573 : dfp->dfp_count--;
485 2144802573 : trace_xfs_defer_finish_item(tp->t_mountp, dfp, li);
486 2144672746 : error = ops->finish_item(tp, dfp->dfp_done, li, &state);
487 2145077300 : if (error == -EAGAIN) {
488 142910601 : int ret;
489 :
490 : /*
491 : * Caller wants a fresh transaction; put the work item
492 : * back on the list and log a new log intent item to
493 : * replace the old one. See "Requesting a Fresh
494 : * Transaction while Finishing Deferred Work" above.
495 : */
496 142910601 : list_add(li, &dfp->dfp_work);
497 142903349 : dfp->dfp_count++;
498 142903349 : dfp->dfp_done = NULL;
499 142903349 : dfp->dfp_intent = NULL;
500 142903349 : ret = xfs_defer_create_intent(tp, dfp, false);
501 142929164 : if (ret < 0)
502 0 : error = ret;
503 : }
504 :
505 2145095863 : if (error)
506 142934225 : goto out;
507 : }
508 :
509 : /* Done with the dfp, free it. */
510 1881729111 : list_del(&dfp->dfp_list);
511 1881669450 : kmem_cache_free(xfs_defer_pending_cache, dfp);
512 1881792875 : tp->t_dfops_nr--;
513 1881792875 : tp->t_dfops_finished++;
514 2024727100 : out:
515 2024727100 : if (ops->finish_cleanup)
516 1020098891 : ops->finish_cleanup(tp, state, error);
517 2024822004 : return error;
518 : }
519 :
520 : /*
521 : * Finish all the pending work. This involves logging intent items for
522 : * any work items that wandered in since the last transaction roll (if
523 : * one has even happened), rolling the transaction, and finishing the
524 : * work items in the first item on the logged-and-pending list.
525 : *
526 : * If an inode is provided, relog it to the new transaction.
527 : */
528 : int
529 1453185709 : xfs_defer_finish_noroll(
530 : struct xfs_trans **tp)
531 : {
532 1453185709 : struct xfs_defer_pending *dfp = NULL;
533 1453185709 : int error = 0;
534 1453185709 : LIST_HEAD(dop_pending);
535 :
536 1453185709 : ASSERT((*tp)->t_flags & XFS_TRANS_PERM_LOG_RES);
537 :
538 1453185709 : trace_xfs_defer_finish(*tp, _RET_IP_);
539 :
540 : /* Until we run out of pending work to finish... */
541 3477096990 : while (!list_empty(&dop_pending) || !list_empty(&(*tp)->t_dfops)) {
542 : /*
543 : * Deferred items that are created in the process of finishing
544 : * other deferred work items should be queued at the head of
545 : * the pending list, which puts them ahead of the deferred work
546 : * that was created by the caller. This keeps the number of
547 : * pending work items to a minimum, which decreases the amount
548 : * of time that any one intent item can stick around in memory,
549 : * pinning the log tail.
550 : */
551 2023039793 : int has_intents = xfs_defer_create_intents(*tp);
552 :
553 2024052393 : list_splice_init(&(*tp)->t_dfops, &dop_pending);
554 :
555 2024052393 : (*tp)->t_dfops_nr_max = max((*tp)->t_dfops_nr,
556 : (*tp)->t_dfops_nr_max);
557 :
558 2024052393 : if (has_intents < 0) {
559 0 : error = has_intents;
560 0 : goto out_shutdown;
561 : }
562 2024052393 : if (has_intents || dfp) {
563 1766943088 : error = xfs_defer_trans_roll(tp);
564 1767549530 : if (error)
565 240 : goto out_shutdown;
566 :
567 : /* Relog intent items to keep the log moving. */
568 1767549290 : error = xfs_defer_relog(tp, &dop_pending);
569 1767510920 : if (error)
570 0 : goto out_shutdown;
571 : }
572 :
573 2024620225 : dfp = list_first_entry(&dop_pending, struct xfs_defer_pending,
574 : dfp_list);
575 2024620225 : error = xfs_defer_finish_one(*tp, dfp);
576 2024622352 : if (error && error != -EAGAIN)
577 4311 : goto out_shutdown;
578 : }
579 :
580 1453453972 : trace_xfs_defer_finish_done(*tp, _RET_IP_);
581 1453486985 : return 0;
582 :
583 4551 : out_shutdown:
584 4551 : xfs_defer_trans_abort(*tp, &dop_pending);
585 4551 : xfs_force_shutdown((*tp)->t_mountp, SHUTDOWN_CORRUPT_INCORE);
586 4550 : trace_xfs_defer_finish_error(*tp, error);
587 4550 : xfs_defer_cancel_list((*tp)->t_mountp, &dop_pending);
588 4551 : (*tp)->t_dfops_nr = 0;
589 4551 : xfs_defer_cancel(*tp);
590 4551 : return error;
591 : }
592 :
593 : int
594 195415785 : xfs_defer_finish(
595 : struct xfs_trans **tp)
596 : {
597 195415785 : int error;
598 :
599 : /*
600 : * Finish and roll the transaction once more to avoid returning to the
601 : * caller with a dirty transaction.
602 : */
603 195415785 : error = xfs_defer_finish_noroll(tp);
604 195341798 : if (error)
605 : return error;
606 195340279 : if ((*tp)->t_flags & XFS_TRANS_DIRTY) {
607 179828154 : error = xfs_defer_trans_roll(tp);
608 179853446 : if (error) {
609 6 : xfs_force_shutdown((*tp)->t_mountp,
610 : SHUTDOWN_CORRUPT_INCORE);
611 6 : return error;
612 : }
613 : }
614 :
615 : /* Reset LOWMODE now that we've finished all the dfops. */
616 195365565 : ASSERT(list_empty(&(*tp)->t_dfops));
617 195365565 : (*tp)->t_flags &= ~XFS_TRANS_LOWMODE;
618 195365565 : return 0;
619 : }
620 :
621 : void
622 4925 : xfs_defer_cancel(
623 : struct xfs_trans *tp)
624 : {
625 4925 : struct xfs_mount *mp = tp->t_mountp;
626 :
627 4925 : trace_xfs_defer_cancel(tp, _RET_IP_);
628 4925 : xfs_defer_cancel_list(mp, &tp->t_dfops);
629 4925 : tp->t_dfops_nr = 0;
630 4925 : }
631 :
632 : /* Add an item for later deferred processing. */
633 : void
634 2001830274 : xfs_defer_add(
635 : struct xfs_trans *tp,
636 : enum xfs_defer_ops_type type,
637 : struct list_head *li)
638 : {
639 2001830274 : struct xfs_defer_pending *dfp = NULL;
640 2001830274 : const struct xfs_defer_op_type *ops = defer_op_types[type];
641 :
642 2001502786 : ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES);
643 2001502786 : BUILD_BUG_ON(ARRAY_SIZE(defer_op_types) != XFS_DEFER_OPS_TYPE_MAX);
644 :
645 : /*
646 : * Add the item to a pending item at the end of the intake list.
647 : * If the last pending item has the same type, reuse it. Else,
648 : * create a new pending item at the end of the intake list.
649 : */
650 2001502786 : if (!list_empty(&tp->t_dfops)) {
651 607982432 : dfp = list_last_entry(&tp->t_dfops,
652 : struct xfs_defer_pending, dfp_list);
653 607982432 : if (dfp->dfp_type != type ||
654 151863868 : (ops->max_items && dfp->dfp_count >= ops->max_items))
655 : dfp = NULL;
656 : }
657 120393431 : if (!dfp) {
658 1881109355 : dfp = kmem_cache_zalloc(xfs_defer_pending_cache,
659 : GFP_NOFS | __GFP_NOFAIL);
660 1881253199 : dfp->dfp_type = type;
661 1881253199 : dfp->dfp_intent = NULL;
662 1881253199 : dfp->dfp_done = NULL;
663 1881253199 : dfp->dfp_count = 0;
664 1881253199 : INIT_LIST_HEAD(&dfp->dfp_work);
665 1881253199 : list_add_tail(&dfp->dfp_list, &tp->t_dfops);
666 1880985697 : tp->t_dfops_nr++;
667 : }
668 :
669 2001379128 : list_add_tail(li, &dfp->dfp_work);
670 2001630255 : trace_xfs_defer_add_item(tp->t_mountp, dfp, li);
671 2001539416 : dfp->dfp_count++;
672 2001539416 : }
673 :
674 : /*
675 : * Move deferred ops from one transaction to another and reset the source to
676 : * initial state. This is primarily used to carry state forward across
677 : * transaction rolls with pending dfops.
678 : */
679 : void
680 1962348096 : xfs_defer_move(
681 : struct xfs_trans *dtp,
682 : struct xfs_trans *stp)
683 : {
684 1962348096 : list_splice_init(&stp->t_dfops, &dtp->t_dfops);
685 1962348096 : dtp->t_dfops_nr += stp->t_dfops_nr;
686 1962348096 : dtp->t_dfops_nr_max = stp->t_dfops_nr_max;
687 1962348096 : dtp->t_dfops_finished = stp->t_dfops_finished;
688 1962348096 : stp->t_dfops_nr = 0;
689 1962348096 : stp->t_dfops_nr_max = 0;
690 1962348096 : stp->t_dfops_finished = 0;
691 :
692 : /*
693 : * Low free space mode was historically controlled by a dfops field.
694 : * This meant that low mode state potentially carried across multiple
695 : * transaction rolls. Transfer low mode on a dfops move to preserve
696 : * that behavior.
697 : */
698 1962348096 : dtp->t_flags |= (stp->t_flags & XFS_TRANS_LOWMODE);
699 1962348096 : stp->t_flags &= ~XFS_TRANS_LOWMODE;
700 1962348096 : }
701 :
702 : /*
703 : * Prepare a chain of fresh deferred ops work items to be completed later. Log
704 : * recovery requires the ability to put off until later the actual finishing
705 : * work so that it can process unfinished items recovered from the log in
706 : * correct order.
707 : *
708 : * Create and log intent items for all the work that we're capturing so that we
709 : * can be assured that the items will get replayed if the system goes down
710 : * before log recovery gets a chance to finish the work it put off. The entire
711 : * deferred ops state is transferred to the capture structure and the
712 : * transaction is then ready for the caller to commit it. If there are no
713 : * intent items to capture, this function returns NULL.
714 : *
715 : * If capture_ip is not NULL, the capture structure will obtain an extra
716 : * reference to the inode.
717 : */
718 : static struct xfs_defer_capture *
719 5074 : xfs_defer_ops_capture(
720 : struct xfs_trans *tp)
721 : {
722 5074 : struct xfs_defer_capture *dfc;
723 5074 : unsigned short i;
724 5074 : int error;
725 :
726 5074 : if (list_empty(&tp->t_dfops))
727 : return NULL;
728 :
729 1399 : error = xfs_defer_create_intents(tp);
730 1399 : if (error < 0)
731 0 : return ERR_PTR(error);
732 :
733 : /* Create an object to capture the defer ops. */
734 1399 : dfc = kmem_zalloc(sizeof(*dfc), KM_NOFS);
735 1399 : INIT_LIST_HEAD(&dfc->dfc_list);
736 1399 : INIT_LIST_HEAD(&dfc->dfc_dfops);
737 :
738 : /* Move the dfops chain and transaction state to the capture struct. */
739 1399 : list_splice_init(&tp->t_dfops, &dfc->dfc_dfops);
740 1399 : dfc->dfc_tpflags = tp->t_flags & XFS_TRANS_LOWMODE;
741 1399 : tp->t_flags &= ~XFS_TRANS_LOWMODE;
742 :
743 : /* Capture the remaining block reservations along with the dfops. */
744 1399 : dfc->dfc_blkres = tp->t_blk_res - tp->t_blk_res_used;
745 1399 : dfc->dfc_rtxres = tp->t_rtx_res - tp->t_rtx_res_used;
746 :
747 : /* Preserve the log reservation size. */
748 1399 : dfc->dfc_logres = tp->t_log_res;
749 :
750 1399 : error = xfs_defer_save_resources(&dfc->dfc_held, tp);
751 1399 : if (error) {
752 : /*
753 : * Resource capture should never fail, but if it does, we
754 : * still have to shut down the log and release things
755 : * properly.
756 : */
757 0 : xfs_force_shutdown(tp->t_mountp, SHUTDOWN_CORRUPT_INCORE);
758 : }
759 :
760 : /*
761 : * Grab extra references to the inodes and buffers because callers are
762 : * expected to release their held references after we commit the
763 : * transaction.
764 : */
765 2280 : for (i = 0; i < dfc->dfc_held.dr_inos; i++) {
766 881 : ASSERT(xfs_isilocked(dfc->dfc_held.dr_ip[i], XFS_ILOCK_EXCL));
767 881 : ihold(VFS_I(dfc->dfc_held.dr_ip[i]));
768 : }
769 :
770 1399 : for (i = 0; i < dfc->dfc_held.dr_bufs; i++)
771 0 : xfs_buf_hold(dfc->dfc_held.dr_bp[i]);
772 :
773 : return dfc;
774 : }
775 :
776 : /* Release all resources that we used to capture deferred ops. */
777 : void
778 0 : xfs_defer_ops_capture_free(
779 : struct xfs_mount *mp,
780 : struct xfs_defer_capture *dfc)
781 : {
782 0 : unsigned short i;
783 :
784 0 : xfs_defer_cancel_list(mp, &dfc->dfc_dfops);
785 :
786 0 : for (i = 0; i < dfc->dfc_held.dr_bufs; i++)
787 0 : xfs_buf_relse(dfc->dfc_held.dr_bp[i]);
788 :
789 0 : for (i = 0; i < dfc->dfc_held.dr_inos; i++)
790 0 : xfs_irele(dfc->dfc_held.dr_ip[i]);
791 :
792 0 : kmem_free(dfc);
793 0 : }
794 :
795 : /*
796 : * Capture any deferred ops and commit the transaction. This is the last step
797 : * needed to finish a log intent item that we recovered from the log. If any
798 : * of the deferred ops operate on an inode, the caller must pass in that inode
799 : * so that the reference can be transferred to the capture structure. The
800 : * caller must hold ILOCK_EXCL on the inode, and must unlock it before calling
801 : * xfs_defer_ops_continue.
802 : */
803 : int
804 5074 : xfs_defer_ops_capture_and_commit(
805 : struct xfs_trans *tp,
806 : struct list_head *capture_list)
807 : {
808 5074 : struct xfs_mount *mp = tp->t_mountp;
809 5074 : struct xfs_defer_capture *dfc;
810 5074 : int error;
811 :
812 : /* If we don't capture anything, commit transaction and exit. */
813 5074 : dfc = xfs_defer_ops_capture(tp);
814 5074 : if (IS_ERR(dfc)) {
815 0 : xfs_trans_cancel(tp);
816 0 : return PTR_ERR(dfc);
817 : }
818 5074 : if (!dfc)
819 3675 : return xfs_trans_commit(tp);
820 :
821 : /* Commit the transaction and add the capture structure to the list. */
822 1399 : error = xfs_trans_commit(tp);
823 1399 : if (error) {
824 0 : xfs_defer_ops_capture_free(mp, dfc);
825 0 : return error;
826 : }
827 :
828 1399 : list_add_tail(&dfc->dfc_list, capture_list);
829 1399 : return 0;
830 : }
831 :
832 : /*
833 : * Attach a chain of captured deferred ops to a new transaction and free the
834 : * capture structure. If an inode was captured, it will be passed back to the
835 : * caller with ILOCK_EXCL held and joined to the transaction with lockflags==0.
836 : * The caller now owns the inode reference.
837 : */
838 : void
839 1399 : xfs_defer_ops_continue(
840 : struct xfs_defer_capture *dfc,
841 : struct xfs_trans *tp,
842 : struct xfs_defer_resources *dres)
843 : {
844 1399 : unsigned int i;
845 :
846 1399 : ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES);
847 1399 : ASSERT(!(tp->t_flags & XFS_TRANS_DIRTY));
848 :
849 : /* Lock the captured resources to the new transaction. */
850 1399 : if (dfc->dfc_held.dr_inos > 2) {
851 0 : xfs_sort_inodes(dfc->dfc_held.dr_ip, dfc->dfc_held.dr_inos);
852 0 : xfs_lock_inodes(dfc->dfc_held.dr_ip, dfc->dfc_held.dr_inos,
853 : XFS_ILOCK_EXCL);
854 1399 : } else if (dfc->dfc_held.dr_inos == 2)
855 25 : xfs_lock_two_inodes(dfc->dfc_held.dr_ip[0], XFS_ILOCK_EXCL,
856 : dfc->dfc_held.dr_ip[1], XFS_ILOCK_EXCL);
857 1374 : else if (dfc->dfc_held.dr_inos == 1)
858 831 : xfs_ilock(dfc->dfc_held.dr_ip[0], XFS_ILOCK_EXCL);
859 :
860 1399 : for (i = 0; i < dfc->dfc_held.dr_bufs; i++)
861 0 : xfs_buf_lock(dfc->dfc_held.dr_bp[i]);
862 :
863 : /* Join the captured resources to the new transaction. */
864 1399 : xfs_defer_restore_resources(tp, &dfc->dfc_held);
865 2798 : memcpy(dres, &dfc->dfc_held, sizeof(struct xfs_defer_resources));
866 1399 : dres->dr_bufs = 0;
867 :
868 : /* Move captured dfops chain and state to the transaction. */
869 1399 : list_splice_init(&dfc->dfc_dfops, &tp->t_dfops);
870 1399 : tp->t_flags |= dfc->dfc_tpflags;
871 :
872 1399 : kmem_free(dfc);
873 1399 : }
874 :
875 : /* Release the resources captured and continued during recovery. */
876 : void
877 1399 : xfs_defer_resources_rele(
878 : struct xfs_defer_resources *dres)
879 : {
880 1399 : unsigned short i;
881 :
882 2280 : for (i = 0; i < dres->dr_inos; i++) {
883 881 : xfs_iunlock(dres->dr_ip[i], XFS_ILOCK_EXCL);
884 881 : xfs_irele(dres->dr_ip[i]);
885 881 : dres->dr_ip[i] = NULL;
886 : }
887 :
888 1399 : for (i = 0; i < dres->dr_bufs; i++) {
889 0 : xfs_buf_relse(dres->dr_bp[i]);
890 0 : dres->dr_bp[i] = NULL;
891 : }
892 :
893 1399 : dres->dr_inos = 0;
894 1399 : dres->dr_bufs = 0;
895 1399 : dres->dr_ordered = 0;
896 1399 : }
897 :
898 : static inline int __init
899 59 : xfs_defer_init_cache(void)
900 : {
901 59 : xfs_defer_pending_cache = kmem_cache_create("xfs_defer_pending",
902 : sizeof(struct xfs_defer_pending),
903 : 0, 0, NULL);
904 :
905 59 : return xfs_defer_pending_cache != NULL ? 0 : -ENOMEM;
906 : }
907 :
908 : static inline void
909 : xfs_defer_destroy_cache(void)
910 : {
911 58 : kmem_cache_destroy(xfs_defer_pending_cache);
912 58 : xfs_defer_pending_cache = NULL;
913 : }
914 :
915 : /* Set up caches for deferred work items. */
916 : int __init
917 59 : xfs_defer_init_item_caches(void)
918 : {
919 59 : int error;
920 :
921 59 : error = xfs_defer_init_cache();
922 59 : if (error)
923 : return error;
924 59 : error = xfs_rmap_intent_init_cache();
925 59 : if (error)
926 0 : goto err;
927 59 : error = xfs_refcount_intent_init_cache();
928 59 : if (error)
929 0 : goto err;
930 59 : error = xfs_bmap_intent_init_cache();
931 59 : if (error)
932 0 : goto err;
933 59 : error = xfs_extfree_intent_init_cache();
934 59 : if (error)
935 0 : goto err;
936 59 : error = xfs_attr_intent_init_cache();
937 59 : if (error)
938 0 : goto err;
939 59 : error = xfs_swapext_intent_init_cache();
940 59 : if (error)
941 0 : goto err;
942 :
943 : return 0;
944 0 : err:
945 0 : xfs_defer_destroy_item_caches();
946 0 : return error;
947 : }
948 :
949 : /* Destroy all the deferred work item caches, if they've been allocated. */
950 : void
951 58 : xfs_defer_destroy_item_caches(void)
952 : {
953 58 : xfs_swapext_intent_destroy_cache();
954 58 : xfs_attr_intent_destroy_cache();
955 58 : xfs_extfree_intent_destroy_cache();
956 58 : xfs_bmap_intent_destroy_cache();
957 58 : xfs_refcount_intent_destroy_cache();
958 58 : xfs_rmap_intent_destroy_cache();
959 58 : xfs_defer_destroy_cache();
960 58 : }
|
